tiny-test-fw: add build config and target options

tools/tiny-test-fw/App.py

@@ -38,9 +38,11 @@ class BaseApp(object):
     Also implements some common methods.
 
     :param app_path: the path for app.
+    :param config_name: app configuration to be tested
+    :param target: build target
     """
 
-    def __init__(self, app_path):
+    def __init__(self, app_path, config_name=None, target=None):
         pass
 
     @classmethod

tools/tiny-test-fw/DUT.py

@@ -275,6 +275,7 @@ class BaseDUT(object):
     DEFAULT_EXPECT_TIMEOUT = 10
     MAX_EXPECT_FAILURES_TO_SAVED = 10
     RECV_THREAD_CLS = RecvThread
+    TARGET = None
     """ DUT subclass can specify RECV_THREAD_CLS to do add some extra stuff when receive data.
     For example, DUT can implement exception detect & analysis logic in receive thread subclass. """
     LOG_THREAD = _LogThread()
@@ -377,15 +378,14 @@ class BaseDUT(object):
 
     # methods that need to be overwritten by Tool
     @classmethod
-    def confirm_dut(cls, port, app, **kwargs):
+    def confirm_dut(cls, port, **kwargs):
         """
         confirm if it's a DUT, usually used by auto detecting DUT in by Env config.
 
         subclass (tool) must overwrite this method.
 
         :param port: comport
-        :param app: app instance
-        :return: True or False
+        :return: tuple of result (bool), and target (str)
         """
         pass
 

tools/tiny-test-fw/Env.py

@@ -62,7 +62,7 @@ class Env(object):
         self.lock = threading.RLock()
 
     @_synced
-    def get_dut(self, dut_name, app_path, dut_class=None, app_class=None, **dut_init_args):
+    def get_dut(self, dut_name, app_path, dut_class=None, app_class=None, app_config_name=None, **dut_init_args):
         """
         get_dut(dut_name, app_path, dut_class=None, app_class=None)
 
@@ -70,6 +70,7 @@ class Env(object):
         :param app_path: application path, app instance will use this path to process application info
         :param dut_class: dut class, if not specified will use default dut class of env
         :param app_class: app class, if not specified will use default app of env
+        :param app_config_name: app build config
         :keyword dut_init_args: extra kwargs used when creating DUT instance
         :return: dut instance
         """
@@ -80,7 +81,7 @@ class Env(object):
                 dut_class = self.default_dut_cls
             if app_class is None:
                 app_class = self.app_cls
-            app_inst = app_class(app_path)
+            detected_target = None
             try:
                 port = self.config.get_variable(dut_name)
             except ValueError:
@@ -89,10 +90,19 @@ class Env(object):
                 available_ports = dut_class.list_available_ports()
                 for port in available_ports:
                     if port not in allocated_ports:
-                        if dut_class.confirm_dut(port, app_inst):
+                        result, detected_target = dut_class.confirm_dut(port)
+                        if result:
                             break
                 else:
                     port = None
+
+            app_target = dut_class.TARGET
+            if not app_target:
+                app_target = detected_target
+            if not app_target:
+                raise ValueError("DUT class doesn't specify the target, and autodetection failed")
+            app_inst = app_class(app_path, app_config_name, app_target)
+
             if port:
                 try:
                     dut_config = self.get_variable(dut_name + "_port_config")

tools/tiny-test-fw/IDF/IDFApp.py

@@ -29,10 +29,12 @@ class IDFApp(App.BaseApp):
     IDF_DOWNLOAD_CONFIG_FILE = "download.config"
     IDF_FLASH_ARGS_FILE = "flasher_args.json"
 
-    def __init__(self, app_path):
+    def __init__(self, app_path, config_name=None, target=None):
         super(IDFApp, self).__init__(app_path)
+        self.config_name = config_name
+        self.target = target
         self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path)
+        self.binary_path = self.get_binary_path(app_path, config_name)
         self.elf_file = self._get_elf_file_path(self.binary_path)
         assert os.path.exists(self.binary_path)
         sdkconfig_dict = self.get_sdkconfig()
@@ -87,13 +89,14 @@ class IDFApp(App.BaseApp):
                     d[configs[0]] = configs[1].rstrip()
         return d
 
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         """
         get binary path according to input app_path.
 
         subclass must overwrite this method.
 
         :param app_path: path of application
+        :param config_name: name of the application build config
         :return: abs app binary path
         """
         pass
@@ -206,59 +209,65 @@ class Example(IDFApp):
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # build folder of example path
         path = os.path.join(self.idf_path, app_path, "build")
-        if not os.path.exists(path):
-            # search for CI build folders
-            app = os.path.basename(app_path)
-            example_path = os.path.join(self.idf_path, "build_examples", "example_builds")
-            # example_path has subdirectories named after targets. So we need to look into only the right
-            # subdirectory. Currently, the target is not known at this moment.
-            for dirpath, dirnames, files in os.walk(example_path):
-                if dirnames:
-                    if dirnames[0] == app:
-                        path = os.path.join(example_path, dirpath, dirnames[0], "build")
-                        break
-            else:
-                raise OSError("Failed to find example binary")
-        return path
+        if os.path.exists(path):
+            return path
+
+        if not config_name:
+            config_name = "default"
+
+        # Search for CI build folders.
+        # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
+        # (see tools/ci/build_examples_cmake.sh)
+        # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
+        app_path_underscored = app_path.replace(os.path.sep, "_")
+        example_path = os.path.join(self.idf_path, "build_examples")
+        for dirpath in os.listdir(example_path):
+            if os.path.basename(dirpath) == app_path_underscored:
+                path = os.path.join(example_path, dirpath, config_name, self.target, "build")
+                return path
+
+        raise OSError("Failed to find example binary")
 
 
 class UT(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         """
-        :param app_path: app path or app config
+        :param app_path: app path
+        :param config_name: config name
         :return: binary path
         """
-        if not app_path:
-            app_path = "default"
+        if not config_name:
+            config_name = "default"
 
         path = os.path.join(self.idf_path, app_path)
-        if not os.path.exists(path):
-            while True:
-                # try to get by config
-                if app_path == "default":
-                    # it's default config, we first try to get form build folder of unit-test-app
-                    path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
-                    if os.path.exists(path):
-                        # found, use bin in build path
-                        break
-                # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
-                path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", app_path)
-                if os.path.exists(path):
-                    break
-                raise OSError("Failed to get unit-test-app binary path")
-        return path
+        default_build_path = os.path.join(path, "build")
+        if os.path.exists(default_build_path):
+            return path
+
+        # first try to get from build folder of unit-test-app
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
+        if os.path.exists(path):
+            # found, use bin in build path
+            return path
+
+        # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", config_name)
+        if os.path.exists(path):
+            return path
+
+        raise OSError("Failed to get unit-test-app binary path")
 
 
 class SSC(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # TODO: to implement SSC get binary path
         return app_path
 
 
 class AT(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # TODO: to implement AT get binary path
         return app_path

tools/tiny-test-fw/IDF/IDFDUT.py

@@ -152,7 +152,6 @@ class IDFDUT(DUT.SerialDUT):
     # if need to erase NVS partition in start app
     ERASE_NVS = True
     RECV_THREAD_CLS = IDFRecvThread
-    TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
 
     def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
         super(IDFDUT, self).__init__(name, port, log_file, app, **kwargs)
@@ -174,6 +173,7 @@ class IDFDUT(DUT.SerialDUT):
         :param port: serial port as string
         :return: MAC address or None
         """
+        esp = None
         try:
             esp = cls._get_rom()(port)
             esp.connect()
@@ -181,12 +181,13 @@ class IDFDUT(DUT.SerialDUT):
         except RuntimeError:
             return None
         finally:
-            # do hard reset after use esptool
-            esp.hard_reset()
-            esp._port.close()
+            if esp:
+                # do hard reset after use esptool
+                esp.hard_reset()
+                esp._port.close()
 
     @classmethod
-    def confirm_dut(cls, port, app, **kwargs):
+    def confirm_dut(cls, port, **kwargs):
         inst = None
         try:
             expected_rom_class = cls._get_rom()
@@ -199,9 +200,9 @@ class IDFDUT(DUT.SerialDUT):
             inst = esptool.ESPLoader.detect_chip(port)
             if expected_rom_class and type(inst) != expected_rom_class:
                 raise RuntimeError("Target not expected")
-            return inst.read_mac() is not None
+            return inst.read_mac() is not None, get_target_by_rom_class(type(inst))
         except(esptool.FatalError, RuntimeError):
-            return False
+            return False, None
         finally:
             if inst is not None:
                 inst._port.close()
@@ -415,18 +416,31 @@ class IDFDUT(DUT.SerialDUT):
 
 
 class ESP32DUT(IDFDUT):
+    TARGET = "esp32"
+    TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP32ROM
 
 
 class ESP32S2DUT(IDFDUT):
+    TARGET = "esp32s2beta"
+    TOOLCHAIN_PREFIX = "xtensa-esp32s2-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP32S2ROM
 
 
 class ESP8266DUT(IDFDUT):
+    TARGET = "esp8266"
+    TOOLCHAIN_PREFIX = "xtensa-lx106-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP8266ROM
+
+
+def get_target_by_rom_class(cls):
+    for c in [ESP32DUT, ESP32S2DUT, ESP8266DUT]:
+        if c._get_rom() == cls:
+            return c.TARGET
+    return None

tools/tiny-test-fw/IDF/__init__.py

@@ -25,7 +25,7 @@ def format_case_id(chip, case_name):
 
 
 def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
-                     level="example", erase_nvs=True, **kwargs):
+                     level="example", erase_nvs=True, config_name=None, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
@@ -36,6 +36,7 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", e
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
     :param erase_nvs: if need to erase_nvs in DUT.start_app()
+    :param config_name: if specified, name of the app configuration
     :param kwargs: other keyword args
     :return: test method
     """

tools/unit-test-app/unit_test.py

@@ -56,6 +56,7 @@ FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 END_LIST_STR = r'\r?\nEnter test for running'
 TEST_PATTERN = re.compile(r'\((\d+)\)\s+"([^"]+)" ([^\r\n]+)\r?\n(' + END_LIST_STR + r')?')
 TEST_SUBMENU_PATTERN = re.compile(r'\s+\((\d+)\)\s+"[^"]+"\r?\n(?=(?=\()|(' + END_LIST_STR + r'))')
+UT_APP_PATH = "tools/unit-test-app"
 
 SIMPLE_TEST_ID = 0
 MULTI_STAGE_ID = 1
@@ -284,7 +285,7 @@ def run_unit_test_cases(env, extra_data):
 
     for ut_config in case_config:
         Utility.console_log("Running unit test for config: " + ut_config, "O")
-        dut = env.get_dut("unit-test-app", app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         if len(case_config[ut_config]) > 0:
             replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin'))
         dut.start_app()
@@ -423,7 +424,7 @@ def get_dut(duts, env, name, ut_config, app_bin=None):
     if name in duts:
         dut = duts[name]
     else:
-        dut = env.get_dut(name, app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut(name, app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         duts[name] = dut
         replace_app_bin(dut, "unit-test-app", app_bin)
         dut.start_app()  # download bin to board
@@ -638,7 +639,7 @@ def run_multiple_stage_cases(env, extra_data):
 
     for ut_config in case_config:
         Utility.console_log("Running unit test for config: " + ut_config, "O")
-        dut = env.get_dut("unit-test-app", app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         if len(case_config[ut_config]) > 0:
             replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin'))
         dut.start_app()
@@ -671,7 +672,7 @@ def detect_update_unit_test_info(env, extra_data, app_bin):
     case_config = format_test_case_config(extra_data)
 
     for ut_config in case_config:
-        dut = env.get_dut("unit-test-app", app_path=ut_config)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config)
         replace_app_bin(dut, "unit-test-app", app_bin)
         dut.start_app()