move ci_target_check to ttfw_idf, move ci_target_filter to AssignTest

mark `supported_in_ci` for AssignTest filter.

tools/ci/python_packages/tiny_test_fw/TinyFW.py

@@ -166,21 +166,6 @@ def test_method(**kwargs):
         case_info["name"] = case_info["ID"] = test_func.__name__
         case_info["junit_report_by_case"] = False
 
-        def _filter_ci_target(target, ci_target):
-            if not ci_target:
-                return target
-            if isinstance(target, str):
-                if isinstance(ci_target, str) and target == ci_target:
-                    return ci_target
-            else:
-                if isinstance(ci_target, str) and ci_target in target:
-                    return ci_target
-                elif isinstance(ci_target, list) and set(ci_target).issubset(set(target)):
-                    return ci_target
-            raise ValueError('ci_target must be a subset of target')
-
-        if os.getenv('CI_JOB_NAME') and 'ci_target' in kwargs:
-            kwargs['target'] = _filter_ci_target(kwargs['target'], kwargs['ci_target'])
         case_info.update(kwargs)
 
         @functools.wraps(test_func)
@@ -230,7 +215,7 @@ def test_method(**kwargs):
 
             dut_dict = kwargs['dut_dict']
             if target not in dut_dict:
-                raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys()))
+                raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_dict.keys()))
 
             dut = dut_dict[target]
             try:

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -143,6 +143,7 @@ class AssignTest(object):
     DEFAULT_FILTER = {
         "category": "function",
         "ignore": False,
+        "supported_in_ci": True,
     }
 
     def __init__(self, test_case_path, ci_config_file, case_group=Group):

tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py

@@ -127,7 +127,6 @@ def filter_test_cases(test_methods, case_filter):
         * user case filter is ``chip: ["esp32", "esp32c"]``, case attribute is ``chip: "esp32"``
         * user case filter is ``chip: "esp32"``, case attribute is ``chip: "esp32"``
 
-
     :param test_methods: a list of test methods functions
     :param case_filter: case filter
     :return: filtered test methods

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -106,6 +106,16 @@ class Search(object):
             key = replicate_config.pop()
             replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
 
+        # mark the cases with targets not in ci_target
+        for case in replicated_cases:
+            ci_target = case.case_info['ci_target']
+            if isinstance(ci_target, str):
+                ci_target = [ci_target]
+            if not ci_target or case.case_info['target'] in ci_target:
+                case.case_info['supported_in_ci'] = True
+            else:
+                case.case_info['supported_in_ci'] = False
+
         return replicated_cases
 
     @classmethod

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -11,9 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
 import os
 import re
 
+from typing import Union
+
 from tiny_test_fw import TinyFW, Utility
 from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: export all Apps for users
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
@@ -30,6 +33,28 @@ def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
+def upper_list(text):  # type: (Union[str, unicode, list]) -> list
+    if isinstance(text, basestring):  # It's not working in python3
+        res = [text.upper()]
+    else:
+        res = [item.upper() for item in text]
+    return res
+
+
+def ci_target_check(func):
+    @functools.wraps(func)
+    def wrapper(**kwargs):
+        target = upper_list(kwargs.get('target', []))
+        ci_target = upper_list(kwargs.get('ci_target', []))
+        if not set(ci_target).issubset(set(target)):
+            raise ValueError('ci_target must be a subset of target')
+
+        return func(**kwargs)
+
+    return wrapper
+
+
+@ci_target_check
 def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1,
                      level="example", erase_nvs=True, config_name=None, **kwargs):
     """
@@ -48,7 +73,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
@@ -58,6 +83,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
     return test
 
 
+@ci_target_check
 def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1,
                   level="unit", erase_nvs=True, **kwargs):
     """
@@ -75,7 +101,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
@@ -85,6 +111,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
     return test
 
 
+@ci_target_check
 def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
                     level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
     """
@@ -104,7 +131,7 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)