remove build_or_not attr.

tools/build_apps.py

@@ -77,10 +77,9 @@ def main():
     setup_logging(args)
 
     build_items = [BuildItem.from_json(line) for line in args.build_list]
-
     if not build_items:
-        logging.error("Empty build list!")
-        raise SystemExit(1)
+        logging.warning("Empty build list")
+        SystemExit(0)
 
     num_builds = len(build_items)
     num_jobs = args.parallel_count
@@ -109,10 +108,6 @@ def main():
 
     failed_builds = []
     for build_info in builds_for_current_job:
-        if not build_info.build:
-            logging.info("Skip building app {}".format(build_info.app_dir))
-            continue
-
         logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
         build_system_class = BUILD_SYSTEMS[build_info.build_system]
         try:

tools/ci/check_build_warnings.py

@@ -70,16 +70,12 @@ def main():
     setup_logging(args)
 
     build_items = [BuildItem.from_json(line) for line in args.build_list]
-
     if not build_items:
-        logging.error("Empty build list!")
-        raise SystemExit(1)
+        logging.warning("Empty build list")
+        SystemExit(0)
 
     found_warnings = 0
     for build_item in build_items:
-        if not build_item.build:
-            logging.debug("Skip checking build log for app {}".format(build_item.app_dir))
-            continue
         if not build_item.build_log_path:
             logging.debug("No log file for {}".format(build_item.work_dir))
             continue

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -1,6 +1,7 @@
 import argparse
 import errno
 import json
+import logging
 import os
 import re
 from collections import defaultdict
@@ -14,74 +15,91 @@ VALID_TARGETS = [
     'esp32s2',
 ]
 
-SPECIAL_REFS = [
-    'master',
-    re.compile(r'^release/v'),
-    re.compile(r'^v\d+\.\d+'),
+TEST_LABELS = {
+    'example_test': 'BOT_LABEL_EXAMPLE_TEST',
+    'test_apps': 'BOT_LABEL_CUSTOM_TEST',
+}
+
+BUILD_ALL_LABELS = [
+    'BOT_LABEL_BUILD_ALL_APPS',
+    'BOT_LABEL_REGULAR_TEST',
 ]
 
 
-def _judge_build_all(args_build_all):
-    if args_build_all:
-        return True
-    if os.getenv('BUILD_ALL_APPS'):
-        return True
+def _has_build_all_label():
+    for label in BUILD_ALL_LABELS:
+        if os.getenv(label):
+            return True
+    return False
 
-    ref = os.getenv('CI_COMMIT_REF_NAME')
-    pipeline_src = os.getenv('CI_PIPELINE_SOURCE')
-    if not ref or not pipeline_src:
-        return False
 
-    # scheduled pipeline will build all
-    if pipeline_src == 'schedule':
-        return True
+def _judge_build_or_not(action, build_all):  # type: (str, bool) -> (bool, bool)
+    """
+    :return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps)
+    """
+    if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')):
+        logging.info('Build all apps')
+        return True, True
 
-    # master, release/v..., v1.2.3..., and will build all
-    for special_ref in SPECIAL_REFS:
-        if isinstance(special_ref, re._pattern_type):
-            if special_ref.match(ref):
-                return True
-        else:
-            if ref == special_ref:
-                return True
-    return False
+    if os.getenv(TEST_LABELS[action]):
+        logging.info('Build test cases apps')
+        return True, False
+    else:
+        logging.info('Skip all')
+        return False, False
+
+
+def output_json(apps_dict_list, target, build_system, output_dir):
+    output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system))
+    with open(output_path, 'w') as fw:
+        fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list])
 
 
 def main():
     parser = argparse.ArgumentParser(description='Scan the required build tests')
-    actions = parser.add_subparsers(dest='action')
-
-    common = argparse.ArgumentParser(add_help=False)
-    common.add_argument('paths',
+    parser.add_argument('test_type',
+                        choices=TEST_LABELS.keys(),
+                        help='Scan test type')
+    parser.add_argument('paths',
                         nargs='+',
-                        help="One or more app paths")
-    common.add_argument('-b', '--build-system',
+                        help='One or more app paths')
+    parser.add_argument('-b', '--build-system',
                         choices=BUILD_SYSTEMS.keys(),
                         default=BUILD_SYSTEM_CMAKE)
-    common.add_argument('-c', '--ci-config-file',
+    parser.add_argument('-c', '--ci-config-file',
                         required=True,
                         help="gitlab ci config target-test file")
-    common.add_argument('-o', '--output-path',
+    parser.add_argument('-o', '--output-path',
                         required=True,
                         help="output path of the scan result")
-    common.add_argument("--exclude",
+    parser.add_argument("--exclude",
                         action="append",
-                        help="Ignore specified directory. Can be used multiple times.")
-    common.add_argument('--preserve', action="store_true",
+                        help='Ignore specified directory. Can be used multiple times.')
+    parser.add_argument('--preserve', action="store_true",
                         help='add this flag to preserve artifacts for all apps')
-    common.add_argument('--build-all', action="store_true",
+    parser.add_argument('--build-all', action="store_true",
                         help='add this flag to build all apps')
 
-    actions.add_parser('example_test', parents=[common])
-    actions.add_parser('test_apps', parents=[common])
-
     args = parser.parse_args()
+    build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
+
+    if not os.path.exists(args.output_path):
+        try:
+            os.makedirs(args.output_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+    if (not build_standalone_apps) and (not build_test_case_apps):
+        for target in VALID_TARGETS:
+            output_json([], target, args.build_system, args.output_path)
+            SystemExit(0)
 
     test_cases = []
-    for path in args.paths:
-        if args.action == 'example_test':
+    for path in set(args.paths):
+        if args.test_type == 'example_test':
             assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
-        elif args.action == 'test_apps':
+        elif args.test_type == 'test_apps':
             CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
             assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
         else:
@@ -89,13 +107,6 @@ def main():
 
         test_cases.extend(assign.search_cases())
 
-    if not os.path.exists(args.output_path):
-        try:
-            os.makedirs(args.output_path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise e
-
     '''
     {
         <target>: {
@@ -113,32 +124,37 @@ def main():
     build_system = args.build_system.lower()
     build_system_class = BUILD_SYSTEMS[build_system]
 
-    for target in VALID_TARGETS:
-        target_dict = scan_info_dict[target]
-        test_case_apps = target_dict['test_case_apps'] = set()
-        for case in test_cases:
-            app_dir = case.case_info['app_dir']
-            app_target = case.case_info['target']
-            if app_target.lower() != target.lower():
-                continue
-            test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
-            exclude_apps.append(app_dir)
-
-    for target in VALID_TARGETS:
-        target_dict = scan_info_dict[target]
-        standalone_apps = target_dict['standalone_apps'] = set()
-        for path in args.paths:
-            standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
+    if build_test_case_apps:
+        for target in VALID_TARGETS:
+            target_dict = scan_info_dict[target]
+            test_case_apps = target_dict['test_case_apps'] = set()
+            for case in test_cases:
+                app_dir = case.case_info['app_dir']
+                app_target = case.case_info['target']
+                if app_target.lower() != target.lower():
+                    continue
+                test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
+                exclude_apps.append(app_dir)
+    else:
+        for target in VALID_TARGETS:
+            scan_info_dict[target]['test_case_apps'] = set()
+
+    if build_standalone_apps:
+        for target in VALID_TARGETS:
+            target_dict = scan_info_dict[target]
+            standalone_apps = target_dict['standalone_apps'] = set()
+            for path in args.paths:
+                standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
+    else:
+        for target in VALID_TARGETS:
+            scan_info_dict[target]['standalone_apps'] = set()
 
-    build_all = _judge_build_all(args.build_all)
     test_case_apps_preserve_default = True if build_system == 'cmake' else False
-
     for target in VALID_TARGETS:
         apps = []
         for app_dir in scan_info_dict[target]['test_case_apps']:
             apps.append({
                 'app_dir': app_dir,
-                'build': True,
                 'build_system': args.build_system,
                 'target': target,
                 'preserve': args.preserve or test_case_apps_preserve_default
@@ -146,10 +162,9 @@ def main():
         for app_dir in scan_info_dict[target]['standalone_apps']:
             apps.append({
                 'app_dir': app_dir,
-                'build': build_all if build_system == 'cmake' else True,
                 'build_system': args.build_system,
                 'target': target,
-                'preserve': (args.preserve and build_all) if build_system == 'cmake' else False
+                'preserve': args.preserve
             })
         output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
         with open(output_path, 'w') as fw:

tools/find_apps.py

@@ -49,8 +49,8 @@ def dict_from_sdkconfig(path):
 
 
 def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
-                        build_system, config_rules, build_or_not=True, preserve_artifacts=True):
-    # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem]
+                        build_system, config_rules, preserve_artifacts=True):
+    # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
     """
     Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
     :param app_path: app directory (can be / usually will be a relative path)
@@ -63,7 +63,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
                        a different CONFIG_IDF_TARGET value.
     :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
     :param config_rules: mapping of sdkconfig file name patterns to configuration names
-    :param build_or_not: determine if it will build in build_apps.py
     :param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
     :return: list of BuildItems representing build configuration of the app
     """
@@ -109,7 +108,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
                     sdkconfig_path,
                     config_name,
                     build_system,
-                    build_or_not,
                     preserve_artifacts,
                 ))
 
@@ -125,7 +123,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
                 None,
                 default_config_name,
                 build_system,
-                build_or_not,
                 preserve_artifacts,
             )
         ]
@@ -296,10 +293,10 @@ def main():
         apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
 
     if not apps:
-        logging.critical("No apps found")
-        raise SystemExit(1)
-    logging.info("Found {} apps".format(len(apps)))
+        logging.warning("No apps found")
+        SystemExit(0)
 
+    logging.info("Found {} apps".format(len(apps)))
     apps.sort(key=lambda x: x["app_dir"])
 
     # Find compatible configurations of each app, collect them as BuildItems
@@ -314,7 +311,6 @@ def main():
             args.target or app["target"],
             args.build_system or app["build_system"],
             config_rules,
-            app["build"],
             app["preserve"],
         )
     logging.info("Found {} builds".format(len(build_items)))

tools/find_build_apps/common.py

@@ -71,7 +71,6 @@ class BuildItem(object):
             sdkconfig_path,
             config_name,
             build_system,
-            build_or_not,
             preserve_artifacts,
     ):
         # These internal variables store the paths with environment variables and placeholders;
@@ -86,7 +85,6 @@ class BuildItem(object):
         self.target = target
         self.build_system = build_system
 
-        self.build = build_or_not
         self.preserve = preserve_artifacts
 
         self._app_name = os.path.basename(os.path.normpath(app_path))
@@ -160,7 +158,6 @@ class BuildItem(object):
             "config": self.config_name,
             "target": self.target,
             "verbose": self.verbose,
-            "build": self.build,
             "preserve": self.preserve,
         })
 
@@ -179,7 +176,6 @@ class BuildItem(object):
             config_name=d["config"],
             target=d["target"],
             build_system=d["build_system"],
-            build_or_not=d["build"],
             preserve_artifacts=d["preserve"]
         )
         result.verbose = d["verbose"]