Add multi target support for search examples

drop keyword `dut`, use `target` instead to assign`dut_class` to `Env`

tools/ci/python_packages/tiny_test_fw/TinyFW.py

@@ -63,7 +63,6 @@ class DefaultEnvConfig(object):
 set_default_config = DefaultEnvConfig.set_default_config
 get_default_config = DefaultEnvConfig.get_default_config
 
-
 MANDATORY_INFO = {
     "execution_time": 1,
     "env_tag": "default",
@@ -158,6 +157,7 @@ def test_method(**kwargs):
                                    In some cases, one test function might test many test cases.
                                    If this flag is set, test case can update junit report by its own.
     """
+
     def test(test_func):
 
         case_info = MANDATORY_INFO.copy()
@@ -181,6 +181,27 @@ def test_method(**kwargs):
                     env_config[key] = kwargs[key]
 
             env_config.update(overwrite)
+
+            # FIXME: CI need more variable here. add `if CI_TARGET: ...` later with CI.
+            target = env_config['target'] if 'target' in env_config else kwargs['target']
+            dut_dict = kwargs['dut_dict']
+            if isinstance(target, list):
+                target = target[0]
+            elif isinstance(target, str):
+                target = target
+            else:
+                raise TypeError('keyword targets can only be list or str')
+            if target not in dut_dict:
+                raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys()))
+
+            dut = dut_dict[target]
+            try:
+                # try to config the default behavior of erase nvs
+                dut.ERASE_NVS = kwargs['erase_nvs']
+            except AttributeError:
+                pass
+
+            env_config['dut'] = dut
             env_inst = Env.Env(**env_config)
 
             # prepare for xunit test results
@@ -227,4 +248,5 @@ def test_method(**kwargs):
         handle_test.case_info = case_info
         handle_test.test_method = True
         return handle_test
+
     return test

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -42,9 +42,14 @@ class Search(object):
                     continue
         except ImportError as e:
             print("ImportError: \r\n\tFile:" + file_name + "\r\n\tError:" + str(e))
-        for i, test_function in enumerate(test_functions):
+
+        test_functions_out = []
+        for case in test_functions:
+            test_functions_out += cls.replicate_case(case)
+
+        for i, test_function in enumerate(test_functions_out):
             print("\t{}. ".format(i + 1) + test_function.case_info["name"])
-        return test_functions
+        return test_functions_out
 
     @classmethod
     def _search_test_case_files(cls, test_case, file_pattern):
@@ -77,17 +82,26 @@ class Search(object):
             if isinstance(case.case_info[key], (list, tuple)):
                 replicate_config.append(key)
 
-        def _replicate_for_key(case_list, replicate_key, replicate_list):
+        def _replicate_for_key(cases, replicate_key, replicate_list):
+            def deepcopy_func(f, name=None):
+                fn = types.FunctionType(f.__code__, f.__globals__, name if name else f.__name__,
+                                        f.__defaults__, f.__closure__)
+                fn.__dict__.update(copy.deepcopy(f.__dict__))
+                return fn
+
             case_out = []
-            for _case in case_list:
+            for inner_case in cases:
                 for value in replicate_list:
-                    new_case = copy.deepcopy(_case)
+                    new_case = deepcopy_func(inner_case)
                     new_case.case_info[replicate_key] = value
                     case_out.append(new_case)
             return case_out
 
         replicated_cases = [case]
-        for key in replicate_config:
+        while replicate_config:
+            if not replicate_config:
+                break
+            key = replicate_config.pop()
             replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
 
         return replicated_cases
@@ -104,8 +118,4 @@ class Search(object):
         test_cases = []
         for test_case_file in test_case_files:
             test_cases += cls._search_cases_from_file(test_case_file)
-        # handle replicate cases
-        test_case_out = []
-        for case in test_cases:
-            test_case_out += cls.replicate_case(case)
-        return test_case_out
+        return test_cases

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -29,7 +29,7 @@ IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
 
 
 class ExampleGroup(CIAssignTest.Group):
-    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
+    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"]
     BUILD_LOCAL_DIR = "build_examples"
     BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
 
@@ -61,7 +61,7 @@ def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=Exa
     artifact_index_list = []
 
     def format_build_log_path():
-        parallel = job_info["parallel_num"]    # Could be None if "parallel_num" not defined for the job
+        parallel = job_info["parallel_num"]  # Could be None if "parallel_num" not defined for the job
         return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
 
     for build_job_name in case_group.BUILD_JOB_NAMES:
@@ -99,7 +99,7 @@ if __name__ == '__main__':
                         help="file name pattern used to find Python test case files")
     parser.add_argument('--custom-group',
                         help='select custom-group for the test cases, if other than ExampleTest',
-                        choices=['example','test-apps'], default='example')
+                        choices=['example', 'test-apps'], default='example')
 
     args = parser.parse_args()
 

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -19,19 +19,24 @@ from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: ex
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
 from .DebugUtils import OCDProcess, GDBProcess, TelnetProcess, CustomProcess  # noqa: export DebugUtils for users
 
+# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue.
+TARGET_DUT_CLS_DICT = {
+    'ESP32': ESP32DUT,
+    'ESP32S2': ESP32S2DUT,
+}
+
 
 def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
-def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
+def idf_example_test(app=Example, target="ESP32", module="examples", execution_time=1,
                      level="example", erase_nvs=True, config_name=None, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -40,31 +45,24 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", e
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test
 
 
-def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", execution_time=1,
+def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1,
                   level="unit", erase_nvs=True, **kwargs):
     """
     decorator for testing idf unit tests (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -72,32 +70,24 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test
 
 
-def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
+def idf_custom_test(app=TestApp, target="ESP32", module="misc", execution_time=1,
                     level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
-
     """
     decorator for idf custom tests (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -107,18 +97,12 @@ def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execut
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test