Refactor parsing tests CI (#2574)

Now parsing tests are performed on all build types.
chipsalliance · Sep 19, 2024 · 078b706 · 078b706
2 parents 803e066 + ca9d464
commit 078b706
Show file tree

Hide file tree

Showing 6 changed files with 165 additions and 166 deletions.
diff --git a/.ci.yml b/.ci.yml
@@ -104,42 +104,45 @@ build_package:
     when: always
     paths:
       - build/
+  script:
+    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type $BUILD_TYPE read_uhdm
+    - ./tests/scripts/run_parsing.sh --type $BUILD_TYPE gather_results
 
 plugin_read_uhdm:
   <<: *parsing_test
   dependencies: [build_plugin]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type plugin read_uhdm
+  variables:
+    BUILD_TYPE: plugin
 
 plugin_read_systemverilog:
   <<: *parsing_test
   dependencies: [build_plugin]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type plugin read_systemverilog
+  variables:
+    BUILD_TYPE: plugin
 
 asan_read_uhdm:
   <<: *parsing_test
   dependencies: [build_asan]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type asan read_uhdm
+  variables:
+    BUILD_TYPE: asan
 
 asan_read_systemverilog:
   <<: *parsing_test
   dependencies: [build_asan]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type asan read_systemverilog
+  variables:
+    BUILD_TYPE: asan
 
 release_read_uhdm:
   <<: *parsing_test
   dependencies: [build_release]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type release read_uhdm
+  variables:
+    BUILD_TYPE: release
 
 release_read_systemverilog:
   <<: *parsing_test
   dependencies: [build_release]
-  script:
-    - ./tests/scripts/run_parsing.sh install_dependencies load_submodules --type release read_systemverilog
+  variables:
+    BUILD_TYPE: release
 
 .job_template: &formal_test
   stage: "Run parsing and formal verification tests"

diff --git a/.github/scripts/generate_summary.sh b/.github/scripts/generate_summary.sh
@@ -1,13 +1,16 @@
 #!/bin/bash
-set -e -u -o pipefail
+set -e -o pipefail
 shopt -s nullglob
 shopt -s extglob
 
 declare -r SELF_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 declare -r ROOT_DIR="$(realpath $SELF_DIR/../..)"
 
-declare -r UHDM_RESULTS_FILE=$ROOT_DIR/test-results-uhdm.log
-declare -r SV_RESULTS_FILE=$ROOT_DIR/test-results-systemverilog.log
+declare -r UHDM_RESULTS_FILE=$ROOT_DIR/$2/$1-test-results-uhdm.log
+declare -r SV_RESULTS_FILE=$ROOT_DIR/$2/$1-test-results-systemverilog.log
+
+NO_UHDM_LOGS=0
+NO_SV_LOGS=0
 
 #--------------------------------------------------------------------------------
 
@@ -18,6 +21,12 @@ function parse_stats() {
     local -n _asan_results="$4"
     local file="$5"
 
+    if [ ! -f $file ]; then
+        if [ "$2" == "uhdm_stats" ]; then NO_UHDM_LOGS=1; fi
+        if [ "$2" == "sv_stats" ]; then NO_SV_LOGS=1; fi
+        return
+    fi
+
     while read test_result asan_result test_name; do
         [[ -z "$test_name" ]] && continue;
 
@@ -60,15 +69,34 @@ total_test_fail=$(( ${uhdm_stats[test_fail]} + ${sv_stats[test_fail]} ))
 total_test_pass=$(( ${uhdm_stats[test_pass]} + ${sv_stats[test_pass]} ))
 total_asan_fail=$(( ${uhdm_stats[asan_fail]} + ${sv_stats[asan_fail]} ))
 
-{
-    printf '| Tool/Parser          | Failed | Passed | ASAN issues |\n'
-    printf '|:---------------------|-------:|-------:|------------:|\n'
-    printf '| `read-uhdm`          | %d     | %d     | %d          |\n' "${uhdm_stats[test_fail]}" "${uhdm_stats[test_pass]}" "${uhdm_stats[asan_fail]}"
-    printf '| `read-systemverilog` | %d     | %d     | %d          |\n' "${sv_stats[test_fail]}" "${sv_stats[test_pass]}" "${sv_stats[asan_fail]}"
-    printf '| **Total**            | **%d** | **%d** | **%d**      |\n' "$total_test_fail" "$total_test_pass" "$total_asan_fail"
-    printf '\n'
-} >> $GITHUB_STEP_SUMMARY
-
+if [ -n "$GITHUB_ACTIONS" ]; then
+    printf "# Parsing tests for build: %s\n" $1
+else
+    printf "#/ Tests summary \#\n"
+fi
+
+if [ "$NO_UHDM_LOGS" == "1" ]; then printf "No read_uhdm logs detected!\n"; fi
+if [ "$NO_SV_LOGS" == "1" ]; then printf "No read_systemverilog logs detected!\n"; fi
+
+if [ "$1" == "asan" ]; then
+    {
+        printf '| Tool/Parser          | Failed | Passed | ASAN issues |\n'
+        printf '|:---------------------|-------:|-------:|------------:|\n'
+        printf '| `read-uhdm`          | %6d | %6d | %11d |\n' "${uhdm_stats[test_fail]}" "${uhdm_stats[test_pass]}" "${uhdm_stats[asan_fail]}"
+        printf '| `read-systemverilog` | %6d | %6d | %11d |\n' "${sv_stats[test_fail]}" "${sv_stats[test_pass]}" "${sv_stats[asan_fail]}"
+        printf '| **Total**            | %6d | %6d | %11d |\n' "$total_test_fail" "$total_test_pass" "$total_asan_fail"
+        printf '\n'
+    }
+else
+    {
+        printf '| Tool/Parser          | Failed | Passed |\n'
+        printf '|:---------------------|-------:|-------:|\n'
+        printf '| `read-uhdm`          | %6d | %6d |\n' "${uhdm_stats[test_fail]}" "${uhdm_stats[test_pass]}"
+        printf '| `read-systemverilog` | %6d | %6d |\n' "${sv_stats[test_fail]}" "${sv_stats[test_pass]}"
+        printf '| **Total**            | %6d | %6d |\n' "$total_test_fail" "$total_test_pass"
+        printf '\n'
+    }
+fi
 failed_test_lines=()
 passed_test_lines=()
 asan_fail_lines=()
@@ -87,6 +115,9 @@ for test_name in "${!test_names[@]}"; do
     (( sv_ok ))        && sv_result_tag="$pass_tag"        || sv_result_tag="$fail_tag"
     (( sv_asan_ok ))   && sv_asan_result_tag="$pass_tag"   || sv_asan_result_tag="$fail_tag"
 
+    if [ "$NO_UHDM_LOGS" == "1" ]; then uhdm_ok=1; uhdm_asan_ok=1; fi
+    if [ "$NO_SV_LOGS" == "1" ]; then sv_ok=1; sv_asan_ok=1; fi
+
     # Puting digits for sorting purposes before "###". They will be removed after sorting.
     if (( !uhdm_ok || !sv_ok )); then
         failed_test_lines+=("${uhdm_ok}${sv_ok} ${test_name} ###| ${uhdm_result_tag} | ${sv_result_tag} | ${test_name} |")
@@ -102,35 +133,47 @@ print_ln() { (( $# )) && printf '%s\n' "$@" || : ; }
 
 {
     if (( ${#failed_test_lines[@]} > 0 )); then
-        print_ln '<details open>'
-        print_ln '<summary><strong>List of failed tests</strong></summary><p>'
+        if [ -n "$GITHUB_ACTIONS" ]; then
+            print_ln '<details open>'
+            print_ln '<summary><strong>List of failed tests</strong></summary><p>'
+        fi
         print_ln ''
         print_ln '| `read_uhdm` | `read_systemverilog` | Test |'
         print_ln '|:-----------:|:--------------------:|:-----|'
         print_ln "${failed_test_lines[@]}" | sort | sed -e 's/^.*###//'
-        print_ln ''
-        print_ln '</p></details>'
+        if [ -n "$GITHUB_ACTIONS" ]; then
+            print_ln ''
+            print_ln '</p></details>'
+        fi
     fi
 
-    if (( ${#asan_fail_lines[@]} > 0 )); then
-        print_ln '<details open>'
-        print_ln '<summary><strong>List of tests with ASAN issues</strong></summary><p>'
-        print_ln ''
-        print_ln '| `read_uhdm` | `read_systemverilog` | Test |'
-        print_ln '|:-----------:|:--------------------:|:-----|'
-        print_ln "${asan_fail_lines[@]}" | sort | sed -e 's/^.*###//'
-        print_ln ''
-        print_ln '</p></details>'
+    if [ "$1" == "asan" ]; then
+        if (( ${#asan_fail_lines[@]} > 0 )); then
+            if [ -n "$GITHUB_ACTIONS" ]; then
+                print_ln '<details open>'
+                print_ln '<summary><strong>List of tests with ASAN issues</strong></summary><p>'
+            fi
+            print_ln ''
+            print_ln '| `read_uhdm` | `read_systemverilog` | Test |'
+            print_ln '|:-----------:|:--------------------:|:-----|'
+            print_ln "${asan_fail_lines[@]}" | sort | sed -e 's/^.*###//'
+            if [ -n "$GITHUB_ACTIONS" ]; then
+                print_ln ''
+                print_ln '</p></details>'
+            fi
+        fi
     fi
-
-    if (( ${#passed_test_lines[@]} > 0 )); then
-        print_ln '<details>'
-        print_ln '<summary><strong>List of passed tests</strong></summary><p>'
-        print_ln ''
-        print_ln '| `read_uhdm` | `read_systemverilog` | Test |'
-        print_ln '|:-----------:|:--------------------:|:-----|'
-        print_ln "${passed_test_lines[@]}" | sort | sed -e 's/^.*###//'
-        print_ln ''
-        print_ln '</p></details>'
+    if [ -n "$GITHUB_ACTIONS" ]; then
+        if (( ${#passed_test_lines[@]} > 0 )); then
+            print_ln '<details>'
+            print_ln '<summary><strong>List of passed tests</strong></summary><p>'
+            print_ln ''
+            print_ln '| `read_uhdm` | `read_systemverilog` | Test |'
+            print_ln '|:-----------:|:--------------------:|:-----|'
+            print_ln "${passed_test_lines[@]}" | sort | sed -e 's/^.*###//'
+            print_ln ''
+            print_ln '</p></details>'
+        fi
     fi
-} >> $GITHUB_STEP_SUMMARY
+    print_ln ''
+}
diff --git a/.github/scripts/run_group_test.sh b/.github/scripts/run_group_test.sh
@@ -13,6 +13,7 @@ declare -r RESULTS_FILE="$(realpath -m "$2")"
 # Input passed via env:
 # - PARSER
 # - TARGET
+# - BUILD_TYPE
 
 # Configure ASAN & LSAN
 
@@ -63,7 +64,7 @@ for test_case in "${test_cases[@]}"; do
         # Used only for current test
         ASAN_OPTIONS+=":log_path=${test_out_dir}/asan"
 
-        if [ "$BUILD_TYPE" == "'plugin'" ]; then
+        if [ "$BUILD_TYPE" == "plugin" ]; then
             BINARY="yosys -Q"
         else
             BINARY="synlig -Q"