tiny-test-fw: fix misc bugs:

1. configs are not functional
    * decorator will be executed when search case, need to set default config before search case.
2. fix DUT encode/decode errors
    * python3 serial don't support write string/unicode, need to convert to bytes first.
    * python2 string could failed to encode/decode non-acsii chars
3. fix bug that log folder not created
4. fix bug that test suite name is not correctly passed:
    * the keyward arg should be `test_suite_name` not `test_name`
5. fix bug that test stopped on failed case
6. fix DUT `read` don't return all data cache
    * `_DataCache.get_data` should first copy all data from queue to data cache and then return to user.
7. fix bug that `expect_all` failed even all expected item passed
8. optimize error info for expect
    * print pattern for regular expression when match failed
9. fix bug that set default config doesn't work

tools/tiny-test-fw/App.py

@@ -78,9 +78,12 @@ class BaseApp(object):
         if not test_suite_name:
             test_suite_name = os.path.splitext(os.path.basename(sys.modules['__main__'].__file__))[0]
         sdk_path = cls.get_sdk_path()
-        return os.path.join(sdk_path, "TEST_LOGS",
-                            test_suite_name +
-                            time.strftime("_%m%d_%H_%M_%S", time.localtime(LOG_FOLDER_TIMESTAMP)))
+        log_folder = os.path.join(sdk_path, "TEST_LOGS",
+                                  test_suite_name +
+                                  time.strftime("_%m%d_%H_%M_%S", time.localtime(LOG_FOLDER_TIMESTAMP)))
+        if not os.path.exists(log_folder):
+            os.makedirs(log_folder)
+        return log_folder
 
     def process_app_info(self):
         """

tools/tiny-test-fw/DUT.py

@@ -85,6 +85,14 @@ def _decode_data(data):
     return data
 
 
+def _pattern_to_string(pattern):
+    try:
+        ret = "RegEx: " + pattern.pattern
+    except AttributeError:
+        ret = pattern
+    return ret
+
+
 class _DataCache(_queue.Queue):
     """
     Data cache based on Queue. Allow users to process data cache based on bytes instead of Queue."
@@ -94,6 +102,21 @@ class _DataCache(_queue.Queue):
         _queue.Queue.__init__(self, maxsize=maxsize)
         self.data_cache = str()
 
+    def _move_from_queue_to_cache(self):
+        """
+        move all of the available data in the queue to cache
+
+        :return: True if moved any item from queue to data cache, else False
+        """
+        ret = False
+        while True:
+            try:
+                self.data_cache += _decode_data(self.get(0))
+                ret = True
+            except _queue.Empty:
+                break
+        return ret
+
     def get_data(self, timeout=0):
         """
         get a copy of data from cache.
@@ -105,12 +128,16 @@ class _DataCache(_queue.Queue):
         if timeout < 0:
             timeout = 0
 
-        try:
-            data = self.get(timeout=timeout)
-            self.data_cache += _decode_data(data)
-        except _queue.Empty:
-            # don't do anything when on update for cache
-            pass
+        ret = self._move_from_queue_to_cache()
+
+        if not ret:
+            # we only wait for new data if we can't provide a new data_cache
+            try:
+                data = self.get(timeout=timeout)
+                self.data_cache += _decode_data(data)
+            except _queue.Empty:
+                # don't do anything when on update for cache
+                pass
         return copy.deepcopy(self.data_cache)
 
     def flush(self, index=0xFFFFFFFF):
@@ -417,7 +444,7 @@ class BaseDUT(object):
             data = self.data_cache.get_data(time.time() + timeout - start_time)
 
         if ret is None:
-            raise ExpectTimeout(self.name + ": " + str(pattern))
+            raise ExpectTimeout(self.name + ": " + _pattern_to_string(pattern))
         return ret
 
     def _expect_multi(self, expect_all, expect_item_list, timeout):
@@ -457,12 +484,11 @@ class BaseDUT(object):
                     if expect_item["ret"] is not None:
                         # match succeed for one item
                         matched_expect_items.append(expect_item)
-                        break
 
             # if expect all, then all items need to be matched,
             # else only one item need to matched
             if expect_all:
-                match_succeed = (matched_expect_items == expect_items)
+                match_succeed = len(matched_expect_items) == len(expect_items)
             else:
                 match_succeed = True if matched_expect_items else False
 
@@ -482,7 +508,7 @@ class BaseDUT(object):
             # flush already matched data
             self.data_cache.flush(slice_index)
         else:
-            raise ExpectTimeout(self.name + ": " + str(expect_items))
+            raise ExpectTimeout(self.name + ": " + str([_pattern_to_string(x) for x in expect_items]))
 
     @_expect_lock
     def expect_any(self, *expect_items, **timeout):

tools/tiny-test-fw/Env.py

@@ -49,12 +49,12 @@ class Env(object):
                  dut=None,
                  env_tag=None,
                  env_config_file=None,
-                 test_name=None,
+                 test_suite_name=None,
                  **kwargs):
         self.app_cls = app
         self.default_dut_cls = dut
         self.config = EnvConfig.Config(env_config_file, env_tag)
-        self.log_path = self.app_cls.get_log_folder(test_name)
+        self.log_path = self.app_cls.get_log_folder(test_suite_name)
         if not os.path.exists(self.log_path):
             os.makedirs(self.log_path)
 

tools/tiny-test-fw/EnvConfig.py

@@ -53,7 +53,7 @@ class Config(object):
         try:
             with open(config_file) as f:
                 configs = yaml.load(f)[env_name]
-        except (OSError, TypeError):
+        except (OSError, TypeError, IOError):
             configs = dict()
         return configs
 

tools/tiny-test-fw/Runner.py

@@ -40,18 +40,22 @@ class Runner(threading.Thread):
     def __init__(self, test_case, case_config, env_config_file=None):
         super(Runner, self).__init__()
         self.setDaemon(True)
-        test_methods = SearchCases.Search.search_test_cases(test_case)
-        self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config)
-        self.test_result = True
         if case_config:
             test_suite_name = os.path.splitext(os.path.basename(case_config))[0]
         else:
             test_suite_name = "TestRunner"
         TinyFW.set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name)
+        test_methods = SearchCases.Search.search_test_cases(test_case)
+        self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config)
+        self.test_result = []
 
     def run(self):
         for case in self.test_cases:
-            self.test_result = self.test_result and case.run()
+            result = case.run()
+            self.test_result.append(result)
+    
+    def get_test_result(self):
+        return self.test_result and all(self.test_result)
 
 
 if __name__ == '__main__':
@@ -76,5 +80,5 @@ if __name__ == '__main__':
         except KeyboardInterrupt:
             print("exit by Ctrl-C")
             break
-    if not runner.test_result:
+    if not runner.get_test_result():
         sys.exit(1)

tools/tiny-test-fw/TinyFW.py

@@ -132,12 +132,6 @@ def test_method(**kwargs):
         case_info["name"] = test_func.__name__
         case_info.update(kwargs)
 
-        # create env instance
-        env_config = DefaultEnvConfig.get_default_config()
-        for key in kwargs:
-            if key in env_config:
-                env_config[key] = kwargs[key]
-
         @functools.wraps(test_func)
         def handle_test(extra_data=None, **overwrite):
             """
@@ -147,6 +141,12 @@ def test_method(**kwargs):
             :param overwrite: args that runner or main want to overwrite
             :return: None
             """
+            # create env instance
+            env_config = DefaultEnvConfig.get_default_config()
+            for key in kwargs:
+                if key in env_config:
+                    env_config[key] = kwargs[key]
+
             env_config.update(overwrite)
             env_inst = Env.Env(**env_config)
             # prepare for xunit test results

tools/tiny-test-fw/docs/index.rst

@@ -69,7 +69,7 @@ Let's first check a simple simple::
 
 
     if __name__ == '__main__':
-        TinyFW.set_default_config(config_file="EnvConfigTemplate.yml")
+        TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml")
         test_examples_protocol_https_request()
 
 

tools/tiny-test-fw/example.py

@@ -47,5 +47,5 @@ def test_examples_protocol_https_request(env, extra_data):
 
 
 if __name__ == '__main__':
-    TinyFW.set_default_config(config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT)
+    TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT)
     test_examples_protocol_https_request()