Save test results in junit format and add test reporter action (#49)

* Break tests into different tasks and upload tests results Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Use PR branches for testing Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Fix linux path and windows packages string Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Test ubuntu action Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Add test results path as action output Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Add test-reporter as external action Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Document new test-reporter external action Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Reorder actions in readme Signed-off-by: Raul Sanchez-Mateos <[email protected]> * Update actions branch to main Signed-off-by: Raul Sanchez-Mateos <[email protected]> --------- Signed-off-by: Raul Sanchez-Mateos <[email protected]>
eProsima · Jan 9, 2024 · 27c98f2 · 27c98f2
1 parent 65e4129
commit 27c98f2
Show file tree

Hide file tree

Showing 9 changed files with 315 additions and 19 deletions.
diff --git a/README.md b/README.md
@@ -257,5 +257,6 @@ Find below the external actions listed:
 | [mirror-branch-action](external/mirror-branch-action/action.yml) | Mirror branch within the same repository | Apache-2.0 license |
 | [setup-ccache-action](external/setup-ccache-action/action.yml) | Setup CCache in a workflow | MIT license |
 | [setup-python](external/setup-python/action.yml) | Installs a version of Python or PyPy and (by default) adds it to the PATH | MIT license |
+| [test-reporter](external/test-reporter/action.yml) | Display test results directly in GitHub | MIT license |
 | [upload-artifact](external/upload-artifact/action.yml) | Upload build artifact | MIT license |
 | [wait-on-check-action](external/wait-on-check-action/action.yml) | Wait on certain check | MIT license |
diff --git a/external/test-reporter/action.yml b/external/test-reporter/action.yml
@@ -0,0 +1,157 @@
+name: 'test-reporter'
+description: 'https://github.com/dorny/test-reporter'
+
+inputs:
+  artifact:
+    description: >
+      Name or regex of artifact containing test results.
+      Regular expression must be enclosed in '/'.
+      Values from captured groups will replace occurrences of $N in report name.
+      Example:
+        artifact: /test-results-(.*)/
+        name: 'Test report $1'
+        -> Artifact 'test-result-ubuntu' would create report 'Test report ubuntu'
+    required: false
+    default: ''
+
+  name:
+    description: >
+      Name of the Check Run which will be created
+    required: false
+    default: ''
+
+  path:
+    description: >
+      Comma-separated list of paths to test results
+      Supports wildcards via [fast-glob](https://github.com/mrmlnc/fast-glob)
+      All matched result files must be of the same format
+    required: false
+    default: ''
+
+  path-replace-backslashes:
+    description: >
+      The fast-glob library that is internally used interprets backslashes as escape characters.
+      If enabled, all backslashes in provided path will be replaced by forward slashes and act as directory separators.
+      It might be useful when path input variable is composed dynamically from existing directory paths on Windows.
+    required: false
+    default: 'false'
+
+  reporter:
+    description: >
+      Format of test results. Supported options:
+        dart-json
+        dotnet-trx
+        flutter-json
+        java-junit (default)
+        jest-junit
+        mocha-json
+    required: false
+    default: 'java-junit'
+
+  only-summary:
+    description: >
+      Allows you to generate only the summary.
+      If enabled, the report will contain a table listing each test results file and the number of passed, failed, and skipped tests.
+      Detailed listing of test suites and test cases will be skipped.
+    required: false
+    default: 'false'
+
+  list-suites:
+    description: >
+      Limits which test suites are listed:
+        all
+        failed
+    required: false
+    default: 'all'
+
+  list-tests:
+    description: >
+      Limits which test cases are listed:
+        all
+        failed
+        none
+    required: false
+    default: 'all'
+
+  max-annotations:
+    description: >
+      Limits number of created annotations with error message and stack trace captured during test execution.
+      Must be less or equal to 50.
+    required: false
+    default: '10'
+
+  fail-on-error:
+    description: >
+      Set action as failed if test report contains any failed test
+    required: false
+    default: 'true'
+
+  fail-on-empty:
+    description: >
+      Set this action as failed if no test results were found
+    required: false
+    default: 'true'
+
+  working-directory:
+    description: >
+      Relative path under $GITHUB_WORKSPACE where the repository was checked out.
+    required: false
+    default: ''
+
+  token:
+    description: >
+      Personal access token used to interact with Github API
+      Default: github.token
+    required: false
+    default: ${{ github.token }}
+
+outputs:
+  conclusion:
+    description: "Success or failure"
+    value: ${{ steps.generate_test_report.outputs.conclusion }}
+
+  passed:
+    description: "Count of passed tests"
+    value: ${{ steps.generate_test_report.outputs.passed }}
+
+  failed:
+    description: "Count of failed tests"
+    value: ${{ steps.generate_test_report.outputs.failed }}
+
+  skipped:
+    description: "Count of skipped tests"
+    value: ${{ steps.generate_test_report.outputs.skipped }}
+
+  time:
+    description: "Test execution time [ms]"
+    value: ${{ steps.generate_test_report.outputs.time }}
+
+  url:
+    description: "Check run URL"
+    value: ${{ steps.generate_test_report.outputs.url }}
+
+  url_html:
+    description: "Check run URL HTML"
+    value: ${{ steps.generate_test_report.outputs.url_html }}
+
+runs:
+  using: composite
+  steps:
+
+    - name: Generate Test Report
+      id: generate_test_report
+      uses: dorny/test-reporter@v1
+      with:
+        artifact: ${{ inputs.artifact }}
+        name: ${{ inputs.name }}
+        path: ${{ inputs.path }}
+        path-replace-backslashes: ${{ inputs.path-replace-backslashes }}
+        reporter: ${{ inputs.reporter }}
+        only-summary: ${{ inputs.only-summary }}
+        list-suites: ${{ inputs.list-suites }}
+        list-tests: ${{ inputs.list-tests }}
+        max-annotations: ${{ inputs.max-annotations }}
+        fail-on-error: ${{ inputs.fail-on-error }}
+        fail-on-empty: ${{ inputs.fail-on-empty }}
+        working-directory: ${{ inputs.working-directory }}
+        token: ${{ inputs.token }}
diff --git a/multiplatform/asan_build_test/action.yml b/multiplatform/asan_build_test/action.yml
@@ -32,11 +32,17 @@ inputs:
     required: false
     default: ''
 
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.build_and_test.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
 
     - name: Build and test
+      id: build_and_test
       uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main
       with:
         packages_names: ${{ inputs.packages_names }}

diff --git a/multiplatform/colcon_build_test/action.yml b/multiplatform/colcon_build_test/action.yml
@@ -37,6 +37,11 @@ inputs:
     required: false
     default: ''
 
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.test.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
@@ -52,9 +57,11 @@ runs:
         cmake_build_type: ${{ inputs.cmake_build_type }}
 
     - name: Test
+      id: test
       uses: eProsima/eProsima-CI/multiplatform/colcon_test@main
       with:
         workspace: ${{ inputs.workspace }}
-        colcon_test_args: '--packages-select ${{ inputs.packages_names }} ${{ inputs.colcon_test_args }}'
+        packages_names: ${{ inputs.packages_names }}
+        colcon_test_args: ${{ inputs.colcon_test_args }}
         ctest_args: ' ${{ inputs.ctest_args }}'
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
diff --git a/multiplatform/colcon_build_test_flaky/action.yml b/multiplatform/colcon_build_test_flaky/action.yml
@@ -32,6 +32,11 @@ inputs:
     required: false
     default: ''
 
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.test.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
@@ -46,8 +51,11 @@ runs:
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
 
     - name: Test
+      id: test
       uses: eProsima/eProsima-CI/multiplatform/colcon_test@main
       with:
         workspace: ${{ inputs.workspace }}
-        colcon_test_args: '--packages-select ${{ inputs.packages_names }} ${{ inputs.colcon_test_args }}'
-        ctest_args: ' --label-regex "xfail" ${{ inputs.ctest_args }}'
+        packages_names: ${{ inputs.packages_names }}
+        colcon_test_args: ${{ inputs.colcon_test_args }}
+        ctest_args: ' --label-regex "xfail" ${{ inputs.ctest_args }} '
+        workspace_dependencies: ${{ inputs.workspace_dependencies }}
diff --git a/multiplatform/colcon_test/action.yml b/multiplatform/colcon_test/action.yml
@@ -23,6 +23,10 @@ inputs:
     required: false
     default: '--timeout 60'
 
+  packages_names:
+    description: 'Name of the colcon packages to build and test'
+    required: true
+
   workspace:
     description: 'Workspace where built has been done'
     required: false
@@ -33,28 +37,37 @@ inputs:
     required: false
     default: ''
 
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.test_ubuntu.outputs.ctest_results_path || steps.test_windows.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
 
     - name: Run in ubuntu
+      id: test_ubuntu
       uses: eProsima/eProsima-CI/ubuntu/colcon_test@main
       if: runner.os == 'Linux'
       with:
         colcon_test_args: ${{ inputs.colcon_test_args }}
         colcon_test_args_default: ${{ inputs.colcon_test_args_default }}
         ctest_args: ${{ inputs.ctest_args }}
         ctest_args_default: ${{ inputs.ctest_args_default }}
+        packages_names: ${{ inputs.packages_names }}
         workspace: ${{ inputs.workspace }}
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
 
     - name: Run in windows
+      id: test_windows
       uses: eProsima/eProsima-CI/windows/colcon_test@main
       if: runner.os == 'Windows'
       with:
         colcon_test_args: ${{ inputs.colcon_test_args }}
         colcon_test_args_default: ${{ inputs.colcon_test_args_default }}
         ctest_args: ${{ inputs.ctest_args }}
         ctest_args_default: ${{ inputs.ctest_args_default }}
+        packages_names: ${{ inputs.packages_names }}
         workspace: ${{ inputs.workspace }}
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
diff --git a/multiplatform/tsan_build_test/action.yml b/multiplatform/tsan_build_test/action.yml
@@ -32,11 +32,17 @@ inputs:
     required: false
     default: ''
 
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.build_and_test.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
 
     - name: Build and test
+      id: build_and_test
       uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main
       with:
         packages_names: ${{ inputs.packages_names }}

diff --git a/ubuntu/colcon_test/action.yml b/ubuntu/colcon_test/action.yml
@@ -23,6 +23,10 @@ inputs:
     required: false
     default: '--timeout 60'
 
+  packages_names:
+    description: 'Name of the colcon packages to build and test'
+    required: true
+
   workspace:
     description: 'Workspace where built has been done'
     required: false
@@ -33,28 +37,68 @@ inputs:
     required: false
     default: ''
 
+  test_report_artifact:
+    description: >
+      Name of the artifact to be uploaded containing the test report in JUnit format.
+      Default: test_report_<github.job>_<matrix.*>
+    required: false
+    default: ''
+
+outputs:
+  ctest_results_path:
+    description: "Path to test results"
+    value: ${{ steps.test.outputs.ctest_results_path }}
+
 runs:
   using: composite
   steps:
 
-    - name: Build workspace with colcon
+    - name: Run tests with colcon
+      id: test
       run: |
-
         echo "::group::Test colcon ${{ inputs.workspace }}"
 
         if [[ ! -z "${{ inputs.workspace_dependencies }}" ]]; then
           source ${{ inputs.workspace_dependencies }}/setup.bash
         fi
 
         cd ${{ inputs.workspace }}
-
-        colcon test \
-          ${{ inputs.colcon_test_args_default }} \
-          ${{ inputs.colcon_test_args }} \
-          --ctest-args \
-            ${{ inputs.ctest_args_default }} \
-            ${{ inputs.ctest_args }}
+        mkdir test_results
+        echo "ctest_results_path=${{ inputs.workspace }}/test_results/" >> $GITHUB_OUTPUT
 
         echo "::endgroup::"
 
+        set +e
+        exit_code=0
+
+        for package in ${{ inputs.packages_names }}; do
+          echo "::group::Testing ${package} ..."
+
+            colcon test \
+              ${{ inputs.colcon_test_args_default }} \
+              ${{ inputs.colcon_test_args }} \
+              --packages-select ${package} \
+              --ctest-args \
+                ${{ inputs.ctest_args_default }} \
+                ${{ inputs.ctest_args }} \
+                --output-junit ${{ inputs.workspace }}/test_results/${package}_test_results.xml
+
+          last_exit_code=$?
+
+          if [ $last_exit_code -gt 0 ]; then
+            exit_code=$last_exit_code
+          fi
+
+          echo "::endgroup::"
+        done
+
+        exit $exit_code
+
       shell: bash
+
+    - name: Upload test report in JUnit format
+      uses: eProsima/eProsima-CI/external/upload-artifact@main
+      if: success() || failure()
+      with:
+          name: ${{ inputs.test_report_artifact || format('test_report_{0}_{1}', github.job, join(matrix.*, '_')) }}
+          path: ${{ inputs.workspace }}/test_results