Github action for CMSIS-DSP testing (#183)

Add github action for running CMSIS-DSP tests on each commit or PR to the main branch

.github/workflows/runtest.yaml

@@ -0,0 +1,86 @@
+name: Compile and Run
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions: 
+  actions: read
+  security-events: write
+
+jobs:
+   CI_test_run: 
+    runs-on: ubuntu-latest    
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'    
+
+      - name: Install system packages
+        run: |
+          sudo add-apt-repository ppa:deadsnakes/ppa
+          sudo apt-get install libpython3.9 libtinfo5
+
+      - name: Activate vcpkg
+        uses: ARM-software/cmsis-actions/vcpkg@v1
+
+      - name: Activate Arm tool license
+        uses: ARM-software/cmsis-actions/armlm@v1
+
+      - name: Prepare framework
+        run: |
+          cd Testing
+          echo "Create missing folders"
+          mkdir FullBenchmark
+          mkdir Output
+          mkdir GeneratedInclude
+          mkdir GeneratedSource
+          mkdir GeneratedIncludeBench
+          mkdir GeneratedSourceBench
+
+          echo "Install missing python packages"
+          pip install -r requirements.txt
+
+          echo "Preprocess test description"
+          python preprocess.py -f desc.txt -o Output.pickle
+          python preprocess.py  -f desc_f16.txt -o Output_f16.pickle
+
+          echo "Generate missing CPP headers"
+          python processTests.py -gen . -p Patterns -d Parameters -f Output.pickle -e
+          python processTests.py -gen . -p Patterns -d Parameters -f Output_f16.pickle -e
+
+          cd cmsis_build
+
+          echo "Load missing pack"
+          csolution list packs -s  test_ac6.csolution.yml -m > required_packs.txt
+          
+          cat required_packs.txt
+          cpackget add -f required_packs.txt
+
+      - name: Execute
+        run: |
+          cd Testing/cmsis_build
+          echo "Running tests"
+
+          python runall.py -avh $AVH_FVP_PLUGINS/../bin
+
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report
+          path: Testing/cmsis_build/summary.html
+
+
+      - name: Check error
+        run: |
+          cd Testing/cmsis_build
+          
+          echo "Checking output..."
+          test "$(grep "Error" summary.html | wc -l)" -eq 0

Testing/cmsis_build/runall.py

@@ -12,9 +12,11 @@
 parser.add_argument('-n', action='store_true', help="No force rebuild")
 parser.add_argument('-r', action='store_true', help="Raw results only")
 parser.add_argument('-c', action='store_true', help="Display cycles (so passing test are displayed)")
+parser.add_argument('-l', action='store_true', help="Local run (not github action)")
 
 args = parser.parse_args()
 
+
 DEBUG=False 
 if args.d:
     DEBUG=True
@@ -112,9 +114,19 @@ def run(*args,mustPrint=False,dumpStdErr=True):
 }
 
 # Windows executable
-# (At some point this script will also support
-# unix)
-avhExe={
+avhUnixExe={
+    "CS310":"FVP_Corstone_SSE-310_Ethos-U65",
+    "CS300":"FVP_Corstone_SSE-300_Ethos-U55",
+    "M55":"FVP_MPS2_Cortex-M55",
+    "M33_DSP_FP":"FVP_MPS2_Cortex-M33",
+    "M7DP":"FVP_MPS2_Cortex-M7",
+    "M4FP":"FVP_MPS2_Cortex-M4",
+    "M3":"FVP_MPS2_Cortex-M3",
+    "M23":"FVP_MPS2_Cortex-M23",
+    "M0plus":"FVP_MPS2_Cortex-M0plus",
+}
+
+avhWindowsExe={
     "CS310":"VHT_Corstone_SSE-310.exe",
     "CS300":"VHT_Corstone_SSE-300_Ethos-U55.exe",
     "M55":"VHT_MPS2_Cortex-M55.exe",
@@ -138,7 +150,20 @@ def runAVH(build,core):
     if os.path.exists(elf):
         app = elf
     config = os.path.join("configs",configFiles[core])
-    avh = os.path.join(AVHROOT,avhExe[core])
+
+    if AVHROOT:
+       avhAttempt = os.path.join(AVHROOT,avhWindowsExe[core])
+       if os.path.exists(avhAttempt):
+          avh = avhAttempt
+
+       avhAttempt = os.path.join(AVHROOT,avhUnixExe[core])
+       if os.path.exists(avhAttempt):
+          avh = avhAttempt
+    else:
+       avh = avhUnixExe[core]
+
+
+
     res=run(avh,"-f",config,app)
     return(res)
 
@@ -160,50 +185,14 @@ def runAVH(build,core):
 # Test suite and output pickle needed to decode the result
 #print(allSuites)
 
-#allSuites=[
-#("MFCCQ15","../Output.pickle"),
-#("MFCCQ31","../Output.pickle"),
-#("SupportTestsF16","../Output_f16.pickle"),
-#]
 
-#allSuites=[("ComplexTestsF32","../Output.pickle"),
-#("DistanceTestsF32","../Output.pickle"),
-#("UnaryTestsF32","../Output.pickle"),
-#("QuaternionTestsF32","../Output.pickle"),
-#("StatsTestsF32","../Output.pickle")
-#]
 
 #allSuites=[
-#("TransformCF64","../Output.pickle"),
-#("TransformCF32","../Output.pickle"),
-#("TransformCQ31","../Output.pickle"),
-#("TransformCQ15","../Output.pickle"),
-#
-#("TransformRF64","../Output.pickle"),
-#("TransformRF32","../Output.pickle"),
-#("TransformRQ31","../Output.pickle"),
-#("TransformRQ15","../Output.pickle"),
-#
-#("MFCCF32","../Output.pickle"),
-#("MFCCQ31","../Output.pickle"),
-#("MFCCQ15","../Output.pickle"),
-#
-#("MFCCF16","../Output_f16.pickle"),
-#("TransformCF16","../Output_f16.pickle"),
-#("TransformRF16","../Output_f16.pickle")
+#("DECIMF64","../Output.pickle"),
+#("UnaryTestsF32","../Output.pickle"),
+#("UnaryTestsF16","../Output_f16.pickle"),
 #]
 
-#allSuites=[("StatsTestsQ7","../Output.pickle")]
-
-allSuites=[
-("UnaryTestsF64","../Output.pickle"),
-("UnaryTestsF32","../Output.pickle"),
-("UnaryTestsF16","../Output_f16.pickle"),
-#("MISCQ15","../Output.pickle"),
-#("MISCQ7","../Output.pickle"),
-#("FIRF16","../Output_f16.pickle")
-]
-
 # Solution and build file for all
 # the tests
 # It is a pair : csolution target type and AVH identification
@@ -213,7 +202,7 @@ def runAVH(build,core):
     'test_ac6.csolution.yml':[
     #  ("VHT-Corstone-310","CS310"),
       ("VHT-Corstone-300","CS300"),
-      # ("VHT-Corstone-300-NOMVE","CS300"),
+      ("VHT-Corstone-300-NOMVE","CS300"),
       ("VHT_M33","M33_DSP_FP"),
       ("VHT_M7","M7DP"),
       ("VHT_M7_UNROLLED","M7DP"),
@@ -222,24 +211,31 @@ def runAVH(build,core):
     #  #("VHT_M23","M23"),
       ("VHT_M0P","M0plus")
     ],
-    'test_gcc.csolution.yml':[
-      #("VHT-Corstone-310","CS310"),
+    #'test_gcc.csolution.yml':[
+    #  ("VHT-Corstone-300","CS300"),
       #("VHT_M55","M55"),
       ##("VHT_M33","M33_DSP_FP"),
-      ("VHT_M7","M7DP"),
-      ("VHT_M7_UNROLLED","M7DP"),
-      ("VHT_M4","M4FP"),
+    #  ("VHT_M7","M7DP"),
+    #  ("VHT_M7_UNROLLED","M7DP"),
+    #  ("VHT_M4","M4FP"),
       ##("VHT_M3","M3"),
       ##("VHT_M23","M23"),
-      ("VHT_M0P","M0plus")
-    ]
+    #  ("VHT_M0P","M0plus")
+    #]
 }
 
 # Override previous solutions for more restricted testing.
 #solutions={
 #    'test_ac6.csolution.yml':[
-#      #("VHT-Corstone-300","CS300"),
-#      ("VHT_M7_UNROLLED","M7DP"),
+#      ("VHT-Corstone-300","CS300"),
+#      #("VHT_M7_UNROLLED","M7DP"),
+#    ]
+#}
+#
+#solutions={
+#    'test_gcc.csolution.yml':[
+#      ("VHT-Corstone-300","CS300"),
+#      #("VHT_M7_UNROLLED","M7DP"),
 #    ]
 #}
 
@@ -336,7 +332,8 @@ def runAVH(build,core):
                            print(res.msg,file=f)
     print(HTMLFOOTER,file=f)
 
-if ERROR_OCCURED:
-    sys.exit("Error occurred")
-else:
-    sys.exit(0)
+if args.l:
+   if ERROR_OCCURED:
+       sys.exit("Error occurred")
+   else:
+       sys.exit(0)