CMSIS-DSP: Improvement to the regression script for tests

CMSIS/DSP/Testing/CMakeLists.txt

@@ -72,6 +72,7 @@ option(BENCHMARK "Benchmarking compiled" OFF)
 option(EXTBENCH "Benchmarking with external traces" OFF)
 option(NN "NN Tests included" OFF)
 option(REFLIB "Use already built reference lib" OFF)
+option(FLOAT16TESTS "Float16 tests" OFF)
 
 project(Testing)
 
@@ -221,7 +222,7 @@ set(TESTSRC
   Source/Tests/ExampleCategoryQ7.cpp
   )
 
-if ((NOT ARMAC5) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL)))
+if ((NOT ARMAC5) AND (FLOAT16TESTS) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL)))
 set(TESTSRC16 
   Source/Tests/BasicTestsF16.cpp
   Source/Tests/TransformCF16.cpp

CMSIS/DSP/Testing/TestScripts/Regression/Commands.py

@@ -153,7 +153,7 @@ class BuildConfig:
     
 
     # Launch cmake command.
-    def createCMake(self,flags):
+    def createCMake(self,flags,benchMode,platform):
         with self.buildFolder() as b:
             self.saveEnv()
             msg("Create cmake for %s\n" % self.buildFolderName())
@@ -162,12 +162,18 @@ class BuildConfig:
             cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(),
                              "-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake,
                              "-DARM_CPU=%s" % self.core(),
-                             "-DPLATFORM=FVP"
+                             "-DPLATFORM=%s" % platform
                     ]
             cmd += flags 
-            cmd += ["-DBENCHMARK=OFF",
-                             "-DWRAPPER=OFF",
-                             "-DCONFIGTABLE=OFF",
+            if benchMode:
+              msg("Benchmark mode\n")
+              cmd += ["-DBENCHMARK=ON"]
+              cmd += ["-DWRAPPER=ON"]
+            else: 
+              cmd += ["-DBENCHMARK=OFF"]
+              cmd += ["-DWRAPPER=OFF"]
+
+            cmd += ["-DCONFIGTABLE=OFF",
                              "-DROOT=%s" % self._rootFolder,
                              "-DCMAKE_BUILD_TYPE=Release",
                              "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()]
@@ -310,7 +316,7 @@ class Test:
         else:
            return(TESTFAILED)
 
-    def runAndProcess(self,compiler,fvp):
+    def runAndProcess(self,compiler,fvp,sim):
         # If we can't parse test description we fail all tests
         self.processTest()
         # Otherwise if only building or those tests are failing, we continue
@@ -323,7 +329,7 @@ class Test:
         # For other compilers only build is tests
         # Since full build is no more possible because of huge pattersn,
         # build is done per test suite.
-        if compiler == "AC6":
+        if sim:
            if fvp is not None:
               self.run(fvp)
               return(self.processResult())
@@ -336,9 +342,9 @@ class Test:
 
 
 # Preprocess the test description
-def preprocess():
-    msg("Process test description file\n")
-    completed = subprocess.run(["python", "preprocess.py","-f","desc.txt"],timeout=3600)
+def preprocess(desc):
+    msg("Process test description file %s\n" % desc)
+    completed = subprocess.run(["python", "preprocess.py","-f",desc],timeout=3600)
     check(completed)
 
 # Generate all missing C code by using all classes in the

CMSIS/DSP/Testing/processResult.py

@@ -359,7 +359,7 @@ def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter):
     benchFile = None
     config=""
     if embedded:
-       prefix = ".*S:[ ]"
+       prefix = ".*[S]+:[ ]"
 
     # Parse the result file.
     # NORMAL mode is when we are parsing suite or group.

CMSIS/DSP/Testing/results.nb

@@ -1,74 +0,0 @@
-(* Content-type: application/vnd.wolfram.mathematica *)
-
-(*** Wolfram Notebook File ***)
-(* http://www.wolfram.com/nb *)
-
-(* CreatedBy='Mathematica 12.0' *)
-
-(*CacheID: 234*)
-(* Internal cache information:
-NotebookFileLineBreakTest
-NotebookFileLineBreakTest
-NotebookDataPosition[       158,          7]
-NotebookDataLength[      2158,         66]
-NotebookOptionsPosition[      1770,         52]
-NotebookOutlinePosition[      2114,         67]
-CellTagsIndexPosition[      2071,         64]
-WindowFrame->Normal*)
-
-(* Beginning of Notebook Content *)
-Notebook[{
-Cell[BoxData[
- RowBox[{
-  RowBox[{"m", "=", 
-   RowBox[{"Dataset", "[", 
-    RowBox[{
-     RowBox[{"Get", "[", 
-      RowBox[{"FileNameJoin", "[", 
-       RowBox[{"{", 
-        RowBox[{
-         RowBox[{"NotebookDirectory", "[", "]"}], ",", "\"\<test.txt\>\""}], 
-        "}"}], "]"}], "]"}], "[", "\"\<Root \>\"", "]"}], "]"}]}], 
-  ";"}]], "Input",
- CellChangeTimes->{{3.7717360240194063`*^9, 3.771736045131504*^9}, {
-   3.771736686569442*^9, 3.771736690261133*^9}, {3.7717512765989704`*^9, 
-   3.7717513417302694`*^9}, {3.771751417685506*^9, 3.7717514233643456`*^9}, {
-   3.7719129941602864`*^9, 3.7719130089901433`*^9}, 3.7719133429206567`*^9},
- CellLabel->"In[1]:=",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
-
-Cell[BoxData[
- RowBox[{"Column", "[", 
-  RowBox[{"Table", "[", 
-   RowBox[{
-    RowBox[{"m", "[", 
-     RowBox[{"[", "i", "]"}], "]"}], ",", 
-    RowBox[{"{", 
-     RowBox[{"i", ",", "1", ",", 
-      RowBox[{
-       RowBox[{"Dimensions", "[", "m", "]"}], "[", 
-       RowBox[{"[", "1", "]"}], "]"}]}], "}"}]}], "]"}], "]"}]], "Input",
- CellChangeTimes->{{3.771913348787628*^9, 3.771913473780173*^9}},
- CellLabel->"In[14]:=",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
-},
-WindowSize->{1063, 641},
-WindowMargins->{{Automatic, 162}, {20, Automatic}},
-FrontEndVersion->"12.0 for Microsoft Windows (64-bit) (April 8, 2019)",
-StyleDefinitions->"Default.nb"
-]
-(* End of Notebook Content *)
-
-(* Internal cache information *)
-(*CellTagsOutline
-CellTagsIndex->{}
-*)
-(*CellTagsIndex
-CellTagsIndex->{}
-*)
-(*NotebookFileOutline
-Notebook[{
-Cell[558, 20, 726, 16, 28, "Input",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
-Cell[1287, 38, 479, 12, 28, "Input",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
-}
-]
-*)
-

CMSIS/DSP/Testing/runAllTests.py

@@ -39,7 +39,47 @@ def updateTestStatus(testStatusForThisBuild,newTestStatus):
     if testStatusForThisBuild == CALLFAILURE:
        return(testStatusForThisBuild)
 
-root = Path(os.getcwd()).parent.parent.parent
+# Analyze the configuration flags (like loopunroll etc ...)
+def analyzeFlags(flags):
+  
+  onoffFlags = []
+  for f in flags:
+    if type(f) is dict:
+      for var in f:
+         if type(f[var]) is bool:
+            if f[var]:
+              onoffFlags.append(["-D%s=ON" % (var)])
+            else:
+              onoffFlags.append(["-D%s=OFF" % (var)])
+         else:   
+           onoffFlags.append(["-D%s=%s" % (var,f[var])])
+    else:
+      onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
+  
+  allConfigs=cartesian(*onoffFlags)
+  return(allConfigs)
+
+# Extract the cmake for a specific compiler
+# and the flag configuration to use for this compiler.
+# This flags configuration will override the global one
+def analyzeToolchain(toolchain, globalConfig):
+    config=globalConfig
+    cmake=""
+    sim=True
+    if type(toolchain) is str:
+       cmake=toolchain 
+    else:
+       for t in toolchain:
+         if type(t) is dict:
+            if "FLAGS" in t:
+               hasConfig=True 
+               config = analyzeFlags(t["FLAGS"])
+            if "SIM" in t:
+               sim = t["SIM"]
+         if type(t) is str:
+           cmake=t 
+    return(cmake,config,sim)
+
 
 
 def cartesian(*somelists):
@@ -48,6 +88,9 @@ def cartesian(*somelists):
        r.append(list(element))
    return(r)
 
+root = Path(os.getcwd()).parent.parent.parent
+
+
 testFailed = 0
 
 init()
@@ -55,11 +98,13 @@ init()
 parser = argparse.ArgumentParser(description='Parse test description')
 parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file")
 parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder")
-parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launchign in parallel")
+parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launching in parallel")
+parser.add_argument('-b', action='store_true', help="Benchmark mode")
+parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file")
+parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running")
 
 args = parser.parse_args()
 
-
 with open(args.i,"r") as f:
      config=yaml.safe_load(f)
 
@@ -67,16 +112,14 @@ with open(args.i,"r") as f:
 
 #print(config["IMPLIEDFLAGS"])
 
-flags = config["FLAGS"]
-onoffFlags = []
-for f in flags:
-  onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
 
-allConfigs=cartesian(*onoffFlags)
+
+
+flags = config["FLAGS"]
+allConfigs = analyzeFlags(flags)
 
 if DEBUGMODE:
    allConfigs=[allConfigs[0]]
-   
 
 failedBuild = {}
 # Test all builds
@@ -97,31 +140,34 @@ def logFailedBuild(root,f):
               status.write("%s : Subprocess failure\n" % build)
 
 
-def buildAndTest(compiler):
+def buildAndTest(compiler,theConfig,cmake,sim):
     # Run all tests for AC6
     try:
        for core in config['CORES']:
          configNb = 0
          if compiler in config['CORES'][core]:
-            for flagConfig in allConfigs:
+            msg("Testing Core %s\n" % core)
+            for flagConfig in theConfig:
                folderCreated = False
                configNb = configNb + 1
                buildStr = "build_%s_%s_%d" % (compiler,core,configNb)
                toUnset = None
                toSet = None
 
-               if compiler in config['UNSET']:
-                  if core in config['UNSET'][compiler]:
-                     toUnset = config['UNSET'][compiler][core]
+               if 'UNSET' in config:
+                  if compiler in config['UNSET']:
+                     if core in config['UNSET'][compiler]:
+                        toUnset = config['UNSET'][compiler][core]
 
-               if compiler in config['SET']:
-                  if core in config['SET'][compiler]:
-                     toSet = config['SET'][compiler][core]
+               if 'SET' in config:
+                  if compiler in config['SET']:
+                     if core in config['SET'][compiler]:
+                        toSet = config['SET'][compiler][core]
 
                build = BuildConfig(toUnset,toSet,args.r,
                   buildStr,
                   config['COMPILERS'][core][compiler],
-                  config['TOOLCHAINS'][compiler],
+                  cmake,
                   config['CORES'][core][compiler],
                   config["CMAKE"]
                   )
@@ -140,16 +186,21 @@ def buildAndTest(compiler):
                try:
                   # This is saving the flag configuration
                   build.createArchive(flags)
-   
-                  build.createCMake(flags)
+                  msg("Config " + str(flagConfig) + "\n")
+
+                  build.createCMake(flags,args.b,args.p)
                   for test in config["TESTS"]:
                       msg(test["testName"]+"\n")
                       testClass=test["testClass"]
                       test = build.getTest(testClass)
                       fvp = None 
-                      if core in config['FVP']:
-                        fvp = config['FVP'][core] 
-                      newTestStatus = test.runAndProcess(compiler,fvp)
+                      if 'FVP' in config:
+                        if core in config['FVP']:
+                           fvp = config['FVP'][core] 
+                      if 'SIM' in config:
+                        if core in config['SIM']:
+                           fvp = config['SIM'][core] 
+                      newTestStatus = test.runAndProcess(compiler,fvp,sim)
                       testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
                       if testStatusForThisBuild != NOTESTFAILED:
                          failedBuild[buildStr] = testStatusForThisBuild
@@ -175,12 +226,15 @@ def buildAndTest(compiler):
 ############## Builds for all toolchains
 
 if not DEBUGMODE:
-   preprocess()
+   preprocess(args.f)
    generateAllCCode()
 
 for t in config["TOOLCHAINS"]:
-    msg("Testing toolchain %s\n" % t)
-    buildAndTest(t)
+    cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)
+    msg("Testing toolchain %s\n" % cmake)
+    buildAndTest(t,localConfig,cmake,sim)
+
+exit(1)
 
 logFailedBuild(args.r,failedBuild)
 sys.exit(testFailed)