Merge branch 'test/unit_test_multi_devices_case_pass_param_by_signals' into 'master'

unit-test-app: support passing parameter with signals

See merge request idf/esp-idf!3960

components/esp32/test/test_wifi.c

@@ -1,6 +1,7 @@
 /*
  Tests for the Wi-Fi
 */
+#include "string.h"
 #include "esp_system.h"
 #include "unity.h"
 #include "esp_system.h"
@@ -11,12 +12,22 @@
 #include "nvs_flash.h"
 #include "test_utils.h"
 #include "freertos/task.h"
+#include "freertos/event_groups.h"
 
 static const char* TAG = "test_wifi";
 
 #define DEFAULT_SSID "TEST_SSID"
 #define DEFAULT_PWD "TEST_PASS"
 
+#define GOT_IP_EVENT        0x00000001
+#define DISCONNECT_EVENT    0x00000002
+
+#define EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT 0x00000001
+
+static uint32_t wifi_event_handler_flag;
+
+static EventGroupHandle_t wifi_events;
+
 static esp_err_t event_handler(void *ctx, system_event_t *event)
 {
     printf("ev_handle_called.\n");
@@ -30,10 +41,18 @@ static esp_err_t event_handler(void *ctx, system_event_t *event)
             ESP_LOGI(TAG, "SYSTEM_EVENT_STA_GOT_IP");
             ESP_LOGI(TAG, "got ip:%s\n",
             ip4addr_ntoa(&event->event_info.got_ip.ip_info.ip));
+            if (wifi_events) {
+                xEventGroupSetBits(wifi_events, GOT_IP_EVENT);
+            }
             break;
         case SYSTEM_EVENT_STA_DISCONNECTED:
             ESP_LOGI(TAG, "SYSTEM_EVENT_STA_DISCONNECTED");
-            TEST_ESP_OK(esp_wifi_connect());
+            if (! (EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT & wifi_event_handler_flag) ) {
+                TEST_ESP_OK(esp_wifi_connect());
+            }
+            if (wifi_events) {
+                xEventGroupSetBits(wifi_events, DISCONNECT_EVENT);
+            }
             break;
         default:
             break;
@@ -129,8 +148,8 @@ static void start_wifi_as_softap(void)
     cfg.nvs_enable = false;
 
     wifi_config_t w_config = {
-        .ap.ssid = "default_ssid",
-        .ap.password = "default_password",
+        .ap.ssid = DEFAULT_SSID,
+        .ap.password = DEFAULT_PWD,
         .ap.ssid_len = 0,
         .ap.channel = 1,
         .ap.authmode = WIFI_AUTH_WPA2_PSK,
@@ -139,21 +158,60 @@ static void start_wifi_as_softap(void)
         .ap.beacon_interval = 100,
     };
 
+    TEST_ESP_OK(esp_event_loop_init(event_handler, NULL));
+
+    // can't deinit event loop, need to reset leak check
+    unity_reset_leak_checks();
+
+    if (wifi_events == NULL) {
+        wifi_events = xEventGroupCreate();
+    }
+
     TEST_ESP_OK(esp_wifi_init(&cfg));
     TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_AP));
     TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_AP, &w_config));
     TEST_ESP_OK(esp_wifi_start());
+}
+
+static void start_wifi_as_sta(void)
+{
+    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
+    cfg.nvs_enable = false;
+
+    // do not auto connect
+    wifi_event_handler_flag |= EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT;
+    TEST_ESP_OK(esp_event_loop_init(event_handler, NULL));
+
+    // can't deinit event loop, need to reset leak check
+    unity_reset_leak_checks();
+
+    if (wifi_events == NULL) {
+        wifi_events = xEventGroupCreate();
+    } else {
+        xEventGroupClearBits(wifi_events, 0x00ffffff);
+    }
+
+    TEST_ESP_OK(esp_wifi_init(&cfg));
+    TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_STA));
+    TEST_ESP_OK(esp_wifi_start());
 
 }
 
 static void stop_wifi(void)
 {
+    printf("stop wifi\n");
     TEST_ESP_OK(esp_wifi_stop());
     TEST_ESP_OK(esp_wifi_deinit());
+    if (wifi_events) {
+        vEventGroupDelete(wifi_events);
+        wifi_events = NULL;
+    }
+    vTaskDelay(1000/portTICK_PERIOD_MS);
 }
 
 static void receive_ds2ds_packet(void)
 {
+    test_case_uses_tcpip();
     start_wifi_as_softap();
     unity_wait_for_signal("sender ready");
     unity_send_signal("receiver ready");
@@ -161,7 +219,6 @@ static void receive_ds2ds_packet(void)
     // wait for sender to send packets
     vTaskDelay(1000/portTICK_PERIOD_MS);
     stop_wifi();
-
 }
 
 static const char ds2ds_pdu[] = {
@@ -174,6 +231,7 @@ static const char ds2ds_pdu[] = {
 
 static void send_ds2ds_packet(void)
 {
+    test_case_uses_tcpip();
     start_wifi_as_softap();
     unity_send_signal("sender ready");
     unity_wait_for_signal("receiver ready");
@@ -187,3 +245,70 @@ static void send_ds2ds_packet(void)
 }
 
 TEST_CASE_MULTIPLE_DEVICES("receive ds2ds packet without exception", "[wifi][test_env=UT_T2_1]", receive_ds2ds_packet, send_ds2ds_packet);
+
+static void wifi_connect_by_bssid(uint8_t *bssid)
+{
+    EventBits_t bits;
+
+    wifi_config_t w_config = {
+        .sta.ssid = DEFAULT_SSID,
+        .sta.password = DEFAULT_PWD,
+        .sta.bssid_set = true,
+    };
+
+    memcpy(w_config.sta.bssid, bssid, 6);
+
+    TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_STA, &w_config));
+    TEST_ESP_OK(esp_wifi_connect());
+    bits = xEventGroupWaitBits(wifi_events, GOT_IP_EVENT, 1, 0, 5000/portTICK_RATE_MS);
+    TEST_ASSERT(bits == GOT_IP_EVENT);
+}
+
+static void test_wifi_connection_sta(void)
+{
+    char mac_str[19];
+    uint8_t mac[6];
+    EventBits_t bits;
+
+    test_case_uses_tcpip();
+
+    start_wifi_as_sta();
+
+    unity_wait_for_signal_param("SoftAP mac", mac_str, 19);
+
+    TEST_ASSERT_TRUE(unity_util_convert_mac_from_string(mac_str, mac));
+
+    wifi_connect_by_bssid(mac);
+
+    unity_send_signal("STA connected");
+
+    bits = xEventGroupWaitBits(wifi_events, DISCONNECT_EVENT, 1, 0, 60000 / portTICK_RATE_MS);
+    // disconnect event not triggered
+    printf("wait finish\n");
+    TEST_ASSERT(bits == 0);
+
+    stop_wifi();
+}
+
+static void test_wifi_connection_softap(void)
+{
+    char mac_str[19] = {0};
+    uint8_t mac[6];
+
+    test_case_uses_tcpip();
+
+    start_wifi_as_softap();
+
+    TEST_ESP_OK(esp_wifi_get_mac(ESP_IF_WIFI_AP, mac));
+    sprintf(mac_str, MACSTR, MAC2STR(mac));
+
+    unity_send_signal_param("SoftAP mac", mac_str);
+
+    unity_wait_for_signal("STA connected");
+
+    vTaskDelay(60000 / portTICK_PERIOD_MS);
+
+    stop_wifi();
+}
+
+TEST_CASE_MULTIPLE_DEVICES("test wifi retain connection for 60s", "[wifi][test_env=UT_T2_1][timeout=90]", test_wifi_connection_sta, test_wifi_connection_softap);

docs/en/api-guides/unit-tests.rst

@@ -75,13 +75,27 @@ As the secnario in the above example, slave should get GPIO level after master s
 DUT1 (master) console::
 
     Waiting for signal: [output high level]!
-    Please press "Enter" key to once any board send this signal.
+    Please press "Enter" key once any board send this signal.
 
 DUT2 (slave) console::
 
     Send signal: [output high level]!
 
-Once the signal is set from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level.
+Once the signal is sent from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level.
+
+Signals can also be used to pass parameters between multiple devices. For example, DUT1 want to know the MAC address of DUT2, so it can connect to DUT2.
+In this case, ``unity_wait_for_signal_param`` and ``unity_send_signal_param`` can be used:
+
+DUT1 console::
+
+    Waiting for signal: [dut2 mac address]!
+    Please input parameter value from any board send this signal and press "Enter" key.
+
+DUT2 console:: 
+
+    Send signal: [dut2 mac address][10:20:30:40:50:60]!
+
+Once the signal is sent from DUT2, you need to input ``10:20:30:40:50:60`` on DUT1 and press "Enter". Then DUT1 will get the MAC address string of DUT2 and unblocks from ``unity_wait_for_signal_param``, start to connect to DUT2.
 
 
 Add multiple stages test cases
@@ -165,13 +179,13 @@ Normal case will print the case name and description. Master slave cases will al
 
 Test cases can be run by inputting one of the following:
 
-- Test case name in quotation marks to run a single test case 
+- Test case name in quotation marks (for example, ``"esp_ota_begin() verifies arguments"``) to run a single test case.
 
-- Test case index to run a single test case
+- Test case index (for example, ``1``) to run a single test case.
 
-- Module name in square brackets to run all test cases for a specific module
+- Module name in square brackets (for example, ``[cxx]``) to run all test cases for a specific module.
 
-- An asterisk to run all test cases
+- An asterisk (``*``) to run all test cases
 
 ``[multi_device]`` and ``[multi_stage]`` tags tell the test runner whether a test case is a multiple devices or multiple stages test case.
 These tags are automatically added by ```TEST_CASE_MULTIPLE_STAGES`` and ``TEST_CASE_MULTIPLE_DEVICES`` macros.

docs/zh_CN/api-guides/unit-tests.rst

@@ -109,6 +109,21 @@ DUT2（slave）终端：
 一旦 DUT2 发送了该信号，您需要在 DUT2 的终端输入回车，然后 DUT1 会从
 ``unity_wait_for_signal`` 函数中解除阻塞，并开始更改 GPIO 的电平。
 
+信号也可以用来在不同 DUT 之间传递参数。例如，DUT1 希望能够拿到 DUT2 的 MAC 地址，来进行蓝牙连接。
+这时，我们可以使用 ``unity_wait_for_signal_param`` 以及 ``unity_send_signal_param``。
+
+DUT1 终端::
+
+    Waiting for signal: [dut2 mac address]!
+    Please input parameter value from any board send this signal and press "Enter" key.
+    
+
+DUT2 终端:: 
+
+    Send signal: [dut2 mac address][10:20:30:40:50:60]!
+
+一旦 DUT2 发送信号，您需要在 DUT1 输入 ``10:20:30:40:50:60`` 及回车，然后 DUT1 会从 ``unity_wait_for_signal_param`` 中获取到蓝牙地址的字符串，并解除阻塞开始蓝牙连接。
+
 
 添加多阶段测试用例
 ------------------
@@ -203,13 +218,13 @@ DUT2（slave）终端：
 
 可以输入以下任意一项来运行测试用例：
 
--  引号中的测试用例的名字，运行单个测试用例。
+-  引号中的测试用例的名字（例如 ``"esp_ota_begin() verifies arguments"``），运行单个测试用例。
 
--  测试用例的序号，运行单个测试用例。
+-  测试用例的序号（例如 ``1``），运行单个测试用例。
 
--  方括号中的模块名字，运行指定模块所有的测试用例。
+-  方括号中的模块名字（例如 ``[cxx]``），运行指定模块所有的测试用例。
 
--  星号，运行所有测试用例。
+-  星号 (``*``)，运行所有测试用例。
 
 ``[multi_device]`` 和 ``[multi_stage]``
 标签告诉测试运行者该用例是多设备测试还是多阶段测试。这些标签由

tools/unit-test-app/components/test_utils/include/test_utils.h

@@ -107,9 +107,8 @@ void unity_reset_leak_checks(void);
  */
 void test_case_uses_tcpip(void);
 
-
 /**
- * @brief wait for signals.
+ * @brief wait for signals with parameters.
  *
  * for multiple devices test cases, DUT might need to wait for other DUTs before continue testing.
  * As all DUTs are independent, need user (or test script) interaction to make test synchronized.
@@ -136,13 +135,64 @@ void test_case_uses_tcpip(void);
  *
  * Then we press Enter key on DUT1's console, DUT1 starts to read input and then test success.
  *
+ * Another example, we have 2 DUTs in multiple devices test, and DUT1 need to get DUT2's mac address to perform BT connection.
+ * DUT1 should call `unity_wait_for_signal_param("dut2 mac address", mac, 19);` to wait for DUT2's mac address.
+ * DUT2 should call `unity_send_signal_param("dut2 mac address", "10:20:30:40:50:60");` to send to DUT1 its mac address.
+ * According to the console logs:
+ *
+ * DUT1 console:
+ *
+ * ```
+ *     Waiting for signal: [dut2 mac address]!
+ *     Please input parameter value from any board send this signal and press "Enter" key.
+ * ```
+ *
+ * DUT2 console:
+ *
+ * ```
+ *     Send signal: [dut2 mac address][10:20:30:40:50:60]!
+ * ```
+ *
  * @param signal_name signal name which DUT expected to wait before proceed testing
+ * @param parameter_buf buffer to receive parameter
+ * @param buf_len length of parameter_buf.
+ *                Currently we have a limitation that it will write 1 extra byte at the end of string.
+ *                We need to use a buffer with 2 bytes longer than actual string length.
  */
-void unity_wait_for_signal(const char* signal_name);
+void unity_wait_for_signal_param(const char* signal_name, char *parameter_buf, uint8_t buf_len);
 
 /**
- * @brief DUT send signal.
+ * @brief wait for signals.
+ *
+ * @param signal_name signal name which DUT expected to wait before proceed testing
+ */
+static inline void unity_wait_for_signal(const char* signal_name)
+{
+    unity_wait_for_signal_param(signal_name, NULL, 0);
+}
+
+/**
+ * @brief DUT send signal and pass parameter to other devices.
+ *
+ * @param signal_name signal name which DUT send once it finished preparing.
+ * @param parameter a string to let remote device to receive.
+ */
+void unity_send_signal_param(const char* signal_name, const char *parameter);
+
+/**
+ * @brief DUT send signal with parameter.
  *
  * @param signal_name signal name which DUT send once it finished preparing.
  */
-void unity_send_signal(const char* signal_name);
+static inline void unity_send_signal(const char* signal_name)
+{
+    unity_send_signal_param(signal_name, NULL);
+}
+
+/**
+ * @brief convert mac string to mac address
+ *
+ * @param mac_str MAC address string with format "xx:xx:xx:xx:xx:xx"
+ * @param[out] mac_addr store converted MAC address
+ */
+bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr);

tools/unit-test-app/components/test_utils/test_utils.c

@@ -31,20 +31,6 @@ const esp_partition_t *get_test_data_partition()
     return result;
 }
 
-// wait user to send "Enter" key
-static void wait_user_control()
-{
-    char sign[5] = {0};
-    while(strlen(sign) == 0)
-    {
-        /* Flush anything already in the RX buffer */
-        while(uart_rx_one_char((uint8_t *) sign) == OK) {
-        }
-        /* Read line */
-        UartRxString((uint8_t*) sign, sizeof(sign) - 1);
-    }
-}
-
 void test_case_uses_tcpip()
 {
     // Can be called more than once, does nothing on subsequent calls
@@ -71,16 +57,60 @@ void test_case_uses_tcpip()
     unity_reset_leak_checks();
 }
 
+// wait user to send "Enter" key or input parameter
+static void wait_user_control(char* parameter_buf, uint8_t buf_len)
+{
+    char *buffer = parameter_buf;
+    char sign[5];
+    uint8_t buffer_len = buf_len - 1;
+
+    if (parameter_buf == NULL) {
+        buffer = sign;
+        buffer_len = sizeof(sign) - 1;
+    }
+    // workaround that unity_gets (UartRxString) will not set '\0' correctly
+    bzero(buffer, buffer_len);
+
+    unity_gets(buffer, buffer_len);
+}
+
 // signal functions, used for sync between unity DUTs for multiple devices cases
-void unity_wait_for_signal(const char* signal_name)
+void unity_wait_for_signal_param(const char* signal_name, char* parameter_buf, uint8_t buf_len)
 {
-    printf("Waiting for signal: [%s]!\n"
-            "Please press \"Enter\" key to once any board send this signal.\n", signal_name);
-    wait_user_control();
+    printf("Waiting for signal: [%s]!\n", signal_name);
+    if (parameter_buf == NULL) {
+        printf("Please press \"Enter\" key once any board send this signal.\n");
+    } else {
+        printf("Please input parameter value from any board send this signal and press \"Enter\" key.\n");
+    }
+    wait_user_control(parameter_buf, buf_len);
 }
 
-void unity_send_signal(const char* signal_name)
+void unity_send_signal_param(const char* signal_name, const char *parameter)
 {
-    printf("Send signal: [%s]!\n", signal_name);
+    if (parameter == NULL) {
+        printf("Send signal: [%s]!\n", signal_name);
+    } else {
+        printf("Send signal: [%s][%s]!\n", signal_name, parameter);
+    }
+}
+
+bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr)
+{
+    uint8_t loop = 0;
+    uint8_t tmp = 0;
+    const char *start;
+    char *stop;
+
+    for (loop = 0; loop < 6; loop++) {
+        start = mac_str + loop * 3;
+        tmp = strtol(start, &stop, 16);
+        if (stop - start == 2 && (*stop == ':' || (*stop == 0 && loop == 5))) {
+            mac_addr[loop] = tmp;
+        } else {
+            return false;
+        }
+    }
+    return true;
 }
 

tools/unit-test-app/unit_test.py

@@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2
 DEFAULT_TIMEOUT = 20
 
 DUT_STARTUP_CHECK_RETRY_COUNT = 5
-TEST_HISTROY_CHECK_TIMEOUT = 1
+TEST_HISTORY_CHECK_TIMEOUT = 1
+
+
+class TestCaseFailed(AssertionError):
+    pass
 
 
 def format_test_case_config(test_case_data):
@@ -163,7 +167,7 @@ def reset_dut(dut):
     for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
         dut.write("-")
         try:
-            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
+            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
             break
         except ExpectTimeout:
             pass
@@ -171,7 +175,7 @@ def reset_dut(dut):
         raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
 
 
-def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
+def run_one_normal_case(dut, one_case, junit_test_case):
 
     reset_dut(dut)
 
@@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
         if result:
             Utility.console_log("Success: " + one_case["name"], color="green")
         else:
-            failed_cases.append(one_case["name"])
             Utility.console_log("Failed: " + one_case["name"], color="red")
             junit_test_case.add_failure_info(output)
+            raise TestCaseFailed()
 
     def handle_exception_reset(data):
         """
@@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
                            timeout=one_case["timeout"])
         except ExpectTimeout:
             Utility.console_log("Timeout in expect", color="orange")
-            junit_test_case.add_error_info("timeout")
+            junit_test_case.add_failure_info("timeout")
             one_case_finish(False)
             break
 
@@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data):
             # create junit report test case
             junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
             try:
-                run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
+                run_one_normal_case(dut, one_case, junit_test_case)
+            except TestCaseFailed:
+                failed_cases.append(one_case["name"])
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     # raise exception if any case fails
@@ -300,8 +306,8 @@ def run_unit_test_cases(env, extra_data):
 
 class Handler(threading.Thread):
 
-    WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)\]!')
-    SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!')
+    WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)]!')
+    SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[([^]]+)](\[([^]]+)])?!')
     FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 
     def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout):
@@ -348,15 +354,23 @@ class Handler(threading.Thread):
                     Utility.console_log("Timeout in device for function: %s" % self.child_case_name, color="orange")
                     break
                 with self.lock:
-                    if expected_signal in self.sent_signal_list:
-                        self.dut.write(" ")
-                        self.sent_signal_list.remove(expected_signal)
-                        break
-                time.sleep(0.01)
+                    for sent_signal in self.sent_signal_list:
+                        if expected_signal == sent_signal["name"]:
+                            self.dut.write(sent_signal["parameter"])
+                            self.sent_signal_list.remove(sent_signal)
+                            break
+                    else:
+                        time.sleep(0.01)
+                        continue
+                    break
 
         def device_send_action(data):
             with self.lock:
-                self.sent_signal_list.append(data[0].encode('utf-8'))
+                self.sent_signal_list.append({
+                    "name": data[0].encode('utf-8'),
+                    "parameter": "" if data[2] is None else data[2].encode('utf-8')
+                    # no parameter means we only write EOL to DUT
+                })
 
         def handle_device_test_finish(data):
             """ test finished without reset """
@@ -468,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data):
             try:
                 run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
                                               one_case.get('app_bin'), junit_test_case)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     if failed_cases:
@@ -480,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data):
         raise AssertionError("Unit Test Failed")
 
 
-def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
+def run_one_multiple_stage_case(dut, one_case, junit_test_case):
     reset_dut(dut)
 
     dut.start_capture_raw_data()
@@ -515,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
                     err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
                                                                                            exception_reset_list)
                     Utility.console_log(err_msg, color="orange")
-                    junit_test_case.add_error_info(err_msg)
+                    junit_test_case.add_failure_info(err_msg)
             else:
                 # we allow omit reset in multi stage cases
                 result = True
@@ -530,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
             if result:
                 Utility.console_log("Success: " + one_case["name"], color="green")
             else:
-                failed_cases.append(one_case["name"])
                 Utility.console_log("Failed: " + one_case["name"], color="red")
                 junit_test_case.add_failure_info(output)
+                raise TestCaseFailed()
             stage_finish.append("break")
 
         def handle_exception_reset(data):
@@ -612,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data):
         for one_case in case_config[ut_config]:
             junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
             try:
-                run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
+                run_one_multiple_stage_case(dut, one_case, junit_test_case)
+            except TestCaseFailed:
+                failed_cases.append(one_case["name"])
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     # raise exception if any case fails