tools: Add pararrel build support

Signed-off-by: Huaqi Fang <578567190@qq.com>

.gitignore

@@ -38,3 +38,4 @@ doc/build/
 _build
 __pycache__
 *.pyc
+logs*

tools/scripts/nsdk_cli/nsdk_bench.py

@@ -26,6 +26,7 @@ class nsdk_bench(nsdk_runner):
         global_build_config = config.get("build_config", dict())
         global_build_configs = config.get("build_configs", dict())
         global_target = config.get("build_target", "all")
+        global_parallel = config.get("parallel", "")
         rootdirs = config.get("appdirs", [])
         ignored_rootdirs = config.get("appdirs_ignore", [])
         if (isinstance(rootdirs, list) and isinstance(ignored_rootdirs, list)) == False:
@@ -66,6 +67,7 @@ class nsdk_bench(nsdk_runner):
                 app_buildcfg = copy.deepcopy(global_build_config)
                 app_buildcfgs = copy.deepcopy(global_build_configs)
                 app_buildtarget = copy.deepcopy(global_target)
+                app_parallel = copy.deepcopy(global_parallel)
                 found_cfg = find_local_appconfig(appdir, appconfigs)
                 if found_cfg:
                     appcfg = appconfigs[found_cfg]
@@ -78,10 +80,13 @@ class nsdk_bench(nsdk_runner):
                         app_buildcfgs.update(appcfg.get("build_configs", dict()))
                         if "build_target" in appcfg:
                             app_buildtarget = appcfg["build_target"]
+                        if "parallel" in appcfg:
+                            app_parallel = appcfg["parallel"]
                     else: # if merge_global is false, then use app config only
                         app_buildcfg = appcfg.get("build_config", dict())
                         app_buildcfgs = appcfg.get("build_configs", dict())
                         app_buildtarget = appcfg.get("build_target", "all")
+                        app_parallel = appcfg.get("parallel", "")
 
                 app_allconfigs = {"configs": {}}
                 if len(app_buildcfgs) == 0:
@@ -95,7 +100,7 @@ class nsdk_bench(nsdk_runner):
                     percfg_appbuildcfg = copy.deepcopy(app_buildcfg)
                     percfg_appbuildcfg.update(app_buildcfgs[cfgname])
                     app_allconfigs["configs"][cfgname] = {"build_config": percfg_appbuildcfg, "build_target": app_buildtarget, \
-                            "logs": {"build": applogfile}}
+                            "parallel": app_parallel, "logs": {"build": applogfile}}
 
                 apps_config[appdir] = copy.deepcopy(app_allconfigs)
 
@@ -113,6 +118,7 @@ class nsdk_bench(nsdk_runner):
         global_build_config = config.get("build_config", dict())
         global_build_configs = config.get("build_configs", dict())
         global_target = config.get("build_target", "clean all")
+        global_parallel = config.get("parallel", "")
         global_run_config = config.get("run_config", dict())
         global_checks = config.get("checks", None)
         rootdirs = config.get("appdirs", [])
@@ -155,6 +161,7 @@ class nsdk_bench(nsdk_runner):
                 app_buildcfg = copy.deepcopy(global_build_config)
                 app_buildcfgs = copy.deepcopy(global_build_configs)
                 app_buildtarget = copy.deepcopy(global_target)
+                app_parallel = copy.deepcopy(global_parallel)
                 app_runcfg = copy.deepcopy(global_run_config)
                 app_checks = copy.deepcopy(global_checks)
                 found_cfg = find_local_appconfig(appdir, appconfigs)
@@ -170,10 +177,13 @@ class nsdk_bench(nsdk_runner):
                         app_checks.update(appcfg.get("checks", dict()))
                         if "build_target" in appcfg:
                             app_buildtarget = appcfg["build_target"]
+                        if "parallel" in appcfg:
+                            app_parallel = appcfg["parallel"]
                     else: # if merge_global is false, then use app config only
                         app_buildcfg = appcfg.get("build_config", dict())
                         app_buildcfgs = appcfg.get("build_configs", dict())
                         app_buildtarget = appcfg.get("build_target", "clean all")
+                        app_parallel = appcfg.get("parallel", "")
                         app_runcfg = appcfg.get("run_config", dict())
                         app_checks = appcfg.get("checks", dict())
 
@@ -192,7 +202,7 @@ class nsdk_bench(nsdk_runner):
                     percfg_appbuildcfg = copy.deepcopy(app_buildcfg)
                     percfg_appbuildcfg.update(app_buildcfgs[cfgname])
                     app_allconfigs["configs"][cfgname] = {"build_config": percfg_appbuildcfg, "build_target": app_buildtarget, \
-                            "run_config": app_runcfg, "checks": app_checks, \
+                            "parallel": app_parallel, "run_config": app_runcfg, "checks": app_checks, \
                             "logs": {"build": app_buildlogfile, "run": app_runlogfile}}
 
                 apps_config[appdir] = copy.deepcopy(app_allconfigs)
@@ -212,10 +222,11 @@ def merge_config(appcfg, hwcfg):
     if isinstance(appcfg, dict) == False and isinstance(hwcfg, dict) == True:
         return hwcfg
     merged_appcfg = copy.deepcopy(appcfg)
-    merged_appcfg.update(hwcfg)
+    #merged_appcfg.update(hwcfg) # this one don't update the dict recursively
+    dict_merge(merged_appcfg, hwcfg)
     return merged_appcfg
 
-def merge_cmd_config(config, serport, baudrate, make_options):
+def merge_cmd_config(config, serport, baudrate, make_options, parallel=None):
     if isinstance(config, dict) == False:
         return None
     new_config = copy.deepcopy(config)
@@ -237,6 +248,8 @@ def merge_cmd_config(config, serport, baudrate, make_options):
                 new_config["run_config"]["hardware"]["baudrate"] = int(baudrate)
             else:
                 new_config["run_config"]["hardware"] = {"serport": "/dev/ttyUSB1", "baudrate": int(baudrate)}
+    if parallel is not None:
+        new_config["parallel"] = parallel
     if make_options:
         opt_splits=make_options.strip().split()
         passed_buildcfg = dict()
@@ -263,6 +276,7 @@ if __name__ == '__main__':
     parser.add_argument('--serport', help="Serial port for monitor, if not specified, it will use the specified by appcfg and hwcfg")
     parser.add_argument('--baudrate', help="Serial port baudrate for monitor, it will use the specified by appcfg and hwcfg")
     parser.add_argument('--make_options', help="Extra make options passed to overwrite default build configuration passed via appcfg and hwcfg")
+    parser.add_argument('--parallel', help="parallel value, such as -j4 or -j or -j8, default None")
     parser.add_argument('--run', action='store_true', help="If specified, will do run not build process")
     parser.add_argument('--verbose', action='store_true', help="If specified, will show detailed build/run messsage")
     args = parser.parse_args()
@@ -278,13 +292,14 @@ if __name__ == '__main__':
     # Merge appcfg and hwcfg, hwcfg has higher priority
     config = merge_config(appcfg, hwcfg)
     # Merge options passed by serport, baudrate, make_options
-    config = merge_cmd_config(config, args.serport, args.baudrate, args.make_options)
+    config = merge_cmd_config(config, args.serport, args.baudrate, args.make_options, args.parallel)
 
     nsdk_ext = nsdk_bench()
     if args.run:
         cmdsts, result = nsdk_ext.run_apps(config, args.verbose, args.logdir, False)
     else:
         cmdsts, result = nsdk_ext.build_apps(config, args.verbose, args.logdir, False)
+
     runtime = round(time.time() - start_time, 2)
     print("Applications specified in %s build or run status: %s, time cost %s seconds" % (args.appcfg, cmdsts, runtime))
     ret = check_expected(result, config, args.run)
@@ -294,14 +309,18 @@ if __name__ == '__main__':
         # Generate build or run report
         rptfile = os.path.join(args.logdir, "report.md")
         generate_report(config, result, rptfile, args.logdir, args.run)
-        print("App, Case, build, run, total, text, data, bss")
+        csvfile = os.path.join(args.logdir, "result.csv")
+        csvstrings = "App, Case, build, run, total, text, data, bss\n"
         for app in result:
             size_of_all_cases = result[app]
             for case in size_of_all_cases:
                 size = size_of_all_cases[case]["size"]
                 app_status = size_of_all_cases[case]["status"]
-                print("%s, %s, %s, %s, %d, %d, %d, %d" % (app, case, app_status["build"], app_status.get("run", False), \
-                    size["total"], size["text"], size["data"], size["bss"]))
+                csvstrings += "%s, %s, %s, %s, %d, %d, %d, %d\n" % (app, case, app_status["build"], app_status.get("run", False), \
+                    size["total"], size["text"], size["data"], size["bss"])
+        with open(csvfile, "w") as cf:
+            cf.write(csvstrings)
+        print(csvstrings)
     # Exit with ret value
     if ret:
         sys.exit(0)

tools/scripts/nsdk_cli/nsdk_builder.py

@@ -48,17 +48,35 @@ class nsdk_builder(object):
         build_objects["verilog"] = find_app_object("*.verilog")
         return build_objects
 
-    def build_target_only(self, appdir, make_options="", target="clean", show_output=True, logfile=None):
+    def build_target_only(self, appdir, make_options="", target="clean", show_output=True, logfile=None, parallel=""):
         if self.is_app(appdir) == False:
             return COMMAND_NOTAPP, 0
 
-        build_cmd = "make -C %s %s %s" % (appdir, make_options, target)
-        if not ((show_output == False) and (target == "info")):
-            print("Build application %s, with target: %s" % (appdir, target))
-            print("Build command: %s" % (build_cmd))
-        ret, ticks = run_command(build_cmd, show_output, logfile=logfile)
-        if not ((show_output == False) and (target == "info")):
-            print("Build command return value: %s" % (ret))
+        # Parallel must start with -j
+        if isinstance(parallel, str):
+            parallel = parallel.strip()
+            if parallel != "" and parallel.startswith("-j") == False:
+                parallel = ""
+        else:
+            parallel = ""
+        if parallel != "": # need to split targets
+            build_targets = target.strip().split()
+        else:
+            build_targets = [target]
+        if os.path.isfile(logfile):
+            os.remove(logfile)
+        total_ticks = 0
+        for btg in build_targets:
+            build_cmd = "make %s -C %s %s %s" % (parallel, appdir, make_options, btg)
+            if not ((show_output == False) and (btg == "info")):
+                print("Build application %s, with target: %s" % (appdir, btg))
+                print("Build command: %s" % (build_cmd))
+            ret, ticks = run_command(build_cmd, show_output, logfile=logfile, append=True)
+            if not ((show_output == False) and (btg == "info")):
+                print("Build command return value: %s" % (ret))
+            total_ticks += ticks
+            if ret != 0: # if one target failed, then stop
+                break
 
         return ret, ticks
 
@@ -83,11 +101,11 @@ class nsdk_builder(object):
         os.remove(infolog)
         return build_info
 
-    def build_target(self, appdir, make_options="", target="clean", show_output=True, logfile=None):
+    def build_target(self, appdir, make_options="", target="clean", show_output=True, logfile=None, parallel=""):
         if self.is_app(appdir) == False:
             return False, None
         build_status = dict()
-        ret, ticks = self.build_target_only(appdir, make_options, target, show_output, logfile)
+        ret, ticks = self.build_target_only(appdir, make_options, target, show_output, logfile, parallel)
         cmdsts = True
         if ret == COMMAND_INTERRUPTED:
             print("%s: Exit program due to CTRL - C pressed" % (sys._getframe().f_code.co_name))
@@ -112,8 +130,8 @@ class nsdk_builder(object):
     def clean_app(self, appdir, make_options="", show_output=True, logfile=None):
         return self.build_target(appdir, make_options, "clean", show_output, logfile)
 
-    def compile_app(self, appdir, make_options="", show_output=True, logfile=None):
-        return self.build_target(appdir, make_options, "all", show_output, logfile)
+    def compile_app(self, appdir, make_options="", show_output=True, logfile=None, parallel=""):
+        return self.build_target(appdir, make_options, "all", show_output, logfile, parallel)
 
     def upload_app(self, appdir, make_options="", show_output=True, logfile=None):
         if logfile is None:
@@ -380,6 +398,7 @@ class nsdk_runner(nsdk_builder):
     def build_app_with_config(self, appdir, appconfig:dict, show_output=True, logfile=None):
         build_config = appconfig.get("build_config", None)
         target = appconfig.get("build_target", "all")
+        parallel = appconfig.get("parallel", "")
         make_options = ""
         if isinstance(build_config, dict):
             for key, value in build_config.items():
@@ -390,7 +409,7 @@ class nsdk_runner(nsdk_builder):
                     make_options += " %s=\"%s\""%(key, value)
                 else:
                     make_options += " %s=%s"%(key, value)
-        appcmdsts, appsts = self.build_target(appdir, make_options, target, show_output, logfile)
+        appcmdsts, appsts = self.build_target(appdir, make_options, target, show_output, logfile, parallel)
         buildtime = appsts["time"]["build"]
         print("Build application %s, time cost %s seconds, passed: %s" %(appdir, buildtime, appcmdsts))
 

tools/scripts/nsdk_cli/nsdk_execute.py

@@ -24,6 +24,7 @@ class nsdk_executor(nsdk_runner):
             return False, None
         global_build_config = config.get("build_config", dict())
         global_target = config.get("build_target", "all")
+        global_parallel = config.get("parallel", "")
         rootdirs = config.get("appdirs", [])
         ignored_rootdirs = config.get("appdirs_ignore", [])
         if (isinstance(rootdirs, list) and isinstance(ignored_rootdirs, list)) == False:
@@ -65,6 +66,7 @@ class nsdk_executor(nsdk_runner):
                     applogfile = get_logfile(appdir, rootdir, logdir, "build.log")
                 app_buildcfg = copy.deepcopy(global_build_config)
                 app_buildtarget = copy.deepcopy(global_target)
+                app_parallel = copy.deepcopy(global_parallel)
                 found_cfg = find_local_appconfig(appdir, appconfigs)
                 if found_cfg:
                     appcfg = appconfigs[found_cfg]
@@ -76,12 +78,15 @@ class nsdk_executor(nsdk_runner):
                         app_buildcfg.update(appcfg.get("build_config", dict()))
                         if "build_target" in appcfg:
                             app_buildtarget = appcfg["build_target"]
+                        if "parallel" in appcfg:
+                            app_parallel = appcfg["parallel"]
                     else: # if merge_global is false, then use app config only
                         app_buildcfg = appcfg.get("build_config", dict())
                         app_buildtarget = appcfg.get("build_target", "all")
+                        app_parallel = appcfg.get("parallel", "")
 
                 appconfig = {"build_config": app_buildcfg, "build_target": app_buildtarget, \
-                            "logs": {"build": applogfile}}
+                            "parallel": app_parallel, "logs": {"build": applogfile}}
                 apps_config[appdir] = copy.deepcopy(appconfig)
 
         if len(apps_config) == 0:
@@ -97,6 +102,7 @@ class nsdk_executor(nsdk_runner):
             return False, None
         global_build_config = config.get("build_config", dict())
         global_target = config.get("build_target", "clean all")
+        global_parallel = config.get("parallel", "")
         global_run_config = config.get("run_config", dict())
         global_checks = config.get("checks", None)
         rootdirs = config.get("appdirs", [])
@@ -142,6 +148,7 @@ class nsdk_executor(nsdk_runner):
                     app_runlogfile = get_logfile(appdir, rootdir, logdir, "run.log")
                 app_buildcfg = copy.deepcopy(global_build_config)
                 app_buildtarget = copy.deepcopy(global_target)
+                app_parallel = copy.deepcopy(global_parallel)
                 app_runcfg = copy.deepcopy(global_run_config)
                 app_checks = copy.deepcopy(global_checks)
                 found_cfg = find_local_appconfig(appdir, appconfigs)
@@ -156,14 +163,17 @@ class nsdk_executor(nsdk_runner):
                         app_checks.update(appcfg.get("checks", dict()))
                         if "build_target" in appcfg:
                             app_buildtarget = appcfg["build_target"]
+                        if "parallel" in appcfg:
+                            app_parallel = appcfg["parallel"]
                     else: # if merge_global is false, then use app config only
                         app_buildcfg = appcfg.get("build_config", dict())
                         app_buildtarget = appcfg.get("build_target", "clean all")
+                        app_parallel = appcfg.get("parallel", "")
                         app_runcfg = appcfg.get("run_config", dict())
                         app_checks = appcfg.get("checks", dict())
 
                 appconfig = {"build_config": app_buildcfg, "build_target": app_buildtarget, \
-                            "run_config": app_runcfg, "checks": app_checks, \
+                            "parallel": app_parallel, "run_config": app_runcfg, "checks": app_checks, \
                             "logs": {"build": app_buildlogfile, "run": app_runlogfile}}
                 apps_config[appdir] = copy.deepcopy(appconfig)
 
@@ -181,10 +191,11 @@ def merge_config(appcfg, hwcfg):
     if isinstance(appcfg, dict) == False and isinstance(hwcfg, dict) == True:
         return hwcfg
     merged_appcfg = copy.deepcopy(appcfg)
-    merged_appcfg.update(hwcfg)
+    #merged_appcfg.update(hwcfg)
+    dict_merge(merged_appcfg, hwcfg)
     return merged_appcfg
 
-def merge_cmd_config(config, serport, baudrate, make_options):
+def merge_cmd_config(config, serport, baudrate, make_options, parallel=None):
     if isinstance(config, dict) == False:
         return None
     new_config = copy.deepcopy(config)
@@ -206,6 +217,8 @@ def merge_cmd_config(config, serport, baudrate, make_options):
                 new_config["run_config"]["hardware"]["baudrate"] = int(baudrate)
             else:
                 new_config["run_config"]["hardware"] = {"serport": "/dev/ttyUSB1", "baudrate": int(baudrate)}
+    if parallel is not None:
+        new_config["parallel"] = parallel
     if make_options:
         opt_splits=make_options.strip().split()
         passed_buildcfg = dict()
@@ -272,6 +285,7 @@ if __name__ == '__main__':
     parser.add_argument('--serport', help="Serial port for monitor, if not specified, it will use the specified by appcfg and hwcfg")
     parser.add_argument('--baudrate', help="Serial port baudrate for monitor, it will use the specified by appcfg and hwcfg")
     parser.add_argument('--make_options', help="Extra make options passed to overwrite default build configuration passed via appcfg and hwcfg")
+    parser.add_argument('--parallel', help="parallel value, such as -j4 or -j or -j8, default None")
     parser.add_argument('--run', action='store_true', help="If specified, will do run not build process")
     parser.add_argument('--verbose', action='store_true', help="If specified, will show detailed build/run messsage")
     args = parser.parse_args()
@@ -288,7 +302,7 @@ if __name__ == '__main__':
     # Merge appcfg and hwcfg, hwcfg has higher priority
     config = merge_config(appcfg, hwcfg)
     # Merge options passed by serport, baudrate, make_options
-    config = merge_cmd_config(config, args.serport, args.baudrate, args.make_options)
+    config = merge_cmd_config(config, args.serport, args.baudrate, args.make_options, args.parallel)
 
     nsdk_ext = nsdk_executor()
     if args.run:
@@ -302,12 +316,16 @@ if __name__ == '__main__':
     print("Applications build or run as expected: %s" % (ret))
     save_results(appcfg, hwcfg, config, result, args.logdir)
     if result:
-        print("App, build, run, total, text, data, bss")
+        csvfile = os.path.join(args.logdir, "result.csv")
+        csvstrings = "App, build, run, total, text, data, bss\n"
         for app in result:
             size = result[app]["size"]
             app_status = result[app]["status"]
-            print("%s, %s, %s, %d, %d, %d, %d" % (app, app_status["build"], app_status.get("run", False), \
-                size["total"], size["text"], size["data"], size["bss"]))
+            csvstrings +="%s, %s, %s, %d, %d, %d, %d\n" % (app, app_status["build"], app_status.get("run", False), \
+                size["total"], size["text"], size["data"], size["bss"])
+        with open(csvfile, "w") as cf:
+            cf.write(csvstrings)
+        print(csvstrings)
     # Exit with ret value
     if ret:
         sys.exit(0)

tools/scripts/nsdk_cli/nsdk_report.py

@@ -80,16 +80,29 @@ def analyze_report(config, result, runapp=False):
     passed_apps = dict()
     failed_apps = dict()
     build_cfgs = dict()
+
+    glb_buildcfg = config.get("build_config", dict())
+    # TODO currently this feature only cover cases that the application build_configs
+    # just extend the global build_configs
     # get build configs used per cfgname
     if "build_configs" not in config:
         build_cfgs["default"] = config.get("build_config", dict())
     else:
-        glb_buildcfg = config.get("build_config", dict())
         sub_configs = config["build_configs"]
         for cfgname in sub_configs:
             bcfg = copy.deepcopy(glb_buildcfg)
             bcfg.update(sub_configs[cfgname])
             build_cfgs[cfgname] = bcfg
+    if "appconfig" in config:
+        appcfgs = config.get("appconfig", dict())
+        for app in appcfgs:
+            if "build_configs" in appcfgs[app]:
+                appsub_configs = appcfgs[app]["build_configs"]
+                for cfgname in appsub_configs:
+                    bcfg = copy.deepcopy(glb_buildcfg)
+                    bcfg.update(appsub_configs[cfgname])
+                    build_cfgs[cfgname] = bcfg
+
     def check_app_status(status, expected, runapp=False):
         app_sts = {"expected": True, "exp_build": True, "exp_run": True, "build": True, "run": True}
         for cfgname in status:
@@ -239,7 +252,7 @@ def merge_all_config_and_result(logdir):
 
     return all_mergedcfg, all_result
 
-def generate_report_for_logs(logdir):
+def generate_report_for_logs(logdir, run=False):
     if logdir and os.path.isdir(logdir):
         reportfile =  os.path.join(logdir, "report.md")
 
@@ -251,17 +264,18 @@ def generate_report_for_logs(logdir):
         save_json(config_file, all_mergedcfg)
         save_json(result_file, all_result)
         print("Save generated report file to %s" % (reportfile))
-        generate_report(all_mergedcfg, all_result, reportfile, logdir, True)
+        generate_report(all_mergedcfg, all_result, reportfile, logdir, run)
     pass
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Nuclei SDK Bench report Generate Tools")
     parser.add_argument('--logdir', required=True, help="logs directory where saved the report json files")
+    parser.add_argument('--run', action='store_true', help="If specified, it means this is a runner report")
     args = parser.parse_args()
 
     if os.path.isdir(args.logdir) == False:
         print("The log directory doesn't exist, please check!")
         sys.exit(1)
 
-    generate_report_for_logs(args.logdir)
+    generate_report_for_logs(args.logdir, args.run)