Merge branch 'feature/ci_support_test_one_case_multiple_times' into 'master'

CI: support test one case multiple times by @bot

See merge request idf/esp-idf!2692

tools/tiny-test-fw/CIAssignUnitTest.py

@@ -109,16 +109,22 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
         with open(test_case_path, "r") as f:
             raw_data = yaml.load(f)
         test_cases = raw_data["test cases"]
+        # filter keys are lower case. Do map lower case keys with original keys.
+        try:
+            key_mapping = {x.lower(): x for x in test_cases[0].keys()}
+        except IndexError:
+            key_mapping = dict()
         if case_filter:
             for key in case_filter:
                 filtered_cases = []
                 for case in test_cases:
                     try:
+                        mapped_key = key_mapping[key]
                         # bot converts string to lower case
-                        if isinstance(case[key], str):
-                            _value = case[key].lower()
+                        if isinstance(case[mapped_key], str):
+                            _value = case[mapped_key].lower()
                         else:
-                            _value = case[key]
+                            _value = case[mapped_key]
                         if _value in case_filter[key]:
                             filtered_cases.append(case)
                     except KeyError:

tools/tiny-test-fw/TinyFW.py

@@ -130,7 +130,7 @@ def test_method(**kwargs):
         test_func_file_name = frame[1][1]
 
         case_info = MANDATORY_INFO.copy()
-        case_info["name"] = test_func.__name__
+        case_info["name"] = case_info["ID"] = test_func.__name__
         case_info.update(kwargs)
 
         @functools.wraps(test_func)

tools/tiny-test-fw/Utility/CIAssignTest.py

@@ -188,6 +188,16 @@ class AssignTest(object):
             bot_filter = dict()
         return bot_filter
 
+    def _apply_bot_test_count(self):
+        """
+        Bot could also pass test count.
+        If filtered cases need to be tested for several times, then we do duplicate them here.
+        """
+        test_count = os.getenv("BOT_TEST_COUNT")
+        if test_count:
+            test_count = int(test_count)
+            self.test_cases *= test_count
+
     def assign_cases(self):
         """
         separate test cases to groups and assign test cases to CI jobs.
@@ -198,6 +208,7 @@ class AssignTest(object):
         failed_to_assign = []
         case_filter = self._apply_bot_filter()
         self.test_cases = self._search_cases(self.test_case_path, case_filter)
+        self._apply_bot_test_count()
         test_groups = self._group_cases()
         for group in test_groups:
             for job in self.jobs:

tools/tiny-test-fw/Utility/CaseConfig.py

@@ -68,11 +68,15 @@ def _convert_to_lower_case(item):
 def _filter_one_case(test_method, case_filter):
     """ Apply filter for one case (the filter logic is the same as described in ``filter_test_cases``) """
     filter_result = True
-    for key in case_filter:
+    # filter keys are lower case. Do map lower case keys with original keys.
+    key_mapping = {x.lower(): x for x in test_method.case_info.keys()}
+
+    for orig_key in case_filter:
+        key = key_mapping[orig_key]
         if key in test_method.case_info:
             # the filter key is both in case and filter
             # we need to check if they match
-            filter_item = _convert_to_lower_case(case_filter[key])
+            filter_item = _convert_to_lower_case(case_filter[orig_key])
             accepted_item = _convert_to_lower_case(test_method.case_info[key])
 
             if isinstance(filter_item, (tuple, list)) \

tools/unit-test-app/tools/UnitTestParser.py

@@ -179,28 +179,11 @@ class Parser(object):
         """
         prop = self.parse_case_properities(description)
 
-        idf_path = os.getenv("IDF_PATH")
-
-        # use relative file path to IDF_PATH, to make sure file path is consist
-        relative_file_path = os.path.relpath(file_name, idf_path)
-
-        file_name_hash = int(hashlib.sha256(relative_file_path).hexdigest(), base=16) % 1000
-
-        if file_name_hash in self.file_name_cache:
-            self.file_name_cache[file_name_hash] += 1
-        else:
-            self.file_name_cache[file_name_hash] = 1
-
-        tc_id = "UT_%s_%s_%03d%02d" % (self.module_map[prop["module"]]['module abbr'],
-                                       self.module_map[prop["module"]]['sub module abbr'],
-                                       file_name_hash,
-                                       self.file_name_cache[file_name_hash])
-
         test_case = deepcopy(TEST_CASE_PATTERN)
         test_case.update({"config": config_name,
                           "module": self.module_map[prop["module"]]['module'],
                           "CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
-                          "ID": tc_id,
+                          "ID": name,
                           "test point 2": prop["module"],
                           "steps": name,
                           "test environment": prop["test_env"],