Try turning the test job into a matrix build

This should reduce the latency of CI, since it will do all the tests for all the packages in parallel. It also makes it easier and faster to retry if the tests for one package fail.
haskell · Feb 14, 2023 · aea1f8f · aea1f8f
1 parent efbf253
commit aea1f8f
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 142 deletions.
diff --git a/.github/workflows/packages.json b/.github/workflows/packages.json
@@ -0,0 +1,39 @@
+[
+  { "package": "haskell-language-server", "excluded_ghcs" : [] }
+, { "package": "hie-compat", "excluded_ghcs" : [] }
+, { "package": "shake-bench", "excluded_ghcs" : [] }
+, { "package": "hls-graph", "excluded_ghcs" : [] }
+, { "package": "ghcide", "excluded_ghcs" : [] }
+, { "package": "ghcide-bench", "excluded_ghcs" : [] }
+, { "package": "ghcide-test-utils", "excluded_ghcs" : [] }
+, { "package": "hls-plugin-api", "excluded_ghcs" : [] }
+, { "package": "hls-test-utils", "excluded_ghcs" : [] }
+, { "package": "hls-cabal-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-cabal-fmt-plugin", "excluded_ghcs" : ["8.10.7"] }
+, { "package": "hls-tactics-plugin", "excluded_ghcs" : ["9.2.5", "9.4.4"] }
+, { "package": "hls-stylish-haskell-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-fourmolu-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-class-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-eval-plugin", "excluded_ghcs" : ["9.4.4"] }
+, { "package": "hls-explicit-imports-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-refine-imports-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-hlint-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-rename-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-retrie-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-haddock-comments-plugin", "excluded_ghcs" : ["9.2.5", "9.4.4"] }
+, { "package": "hls-splice-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-floskell-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-pragmas-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-module-name-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-ormolu-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-call-hierarchy-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-alternate-number-format-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-qualify-imported-names-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-code-range-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-change-type-signature-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-stan-plugin", "excluded_ghcs" : [ "9.0.2","9.2.5","9.4.4" ] }
+, { "package": "hls-gadt-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-explicit-fixity-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-explicit-record-fields-plugin", "excluded_ghcs" : [] }
+, { "package": "hls-refactor-plugin", "excluded_ghcs" : [] }
+]
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -20,12 +20,20 @@ jobs:
     outputs:
       should_skip: ${{ steps.skip_check.outputs.should_skip }}
       should_skip_ghcide: ${{ steps.skip_ghcide_check.outputs.should_skip }}
-      ghcs: ${{ steps.ghcs.outputs.ghcs }}
+      ghcs: ${{ steps.matrix-inputs.outputs.ghcs }}
+      packages: ${{ steps.matrix-inputs.outputs.packages }}
+      exclusions: ${{ steps.matrix-inputs.outputs.exclusions }}
+
     steps:
-      # Need the repo checked out in order to read the file
+      # Need the repo checked out in order to read the files
       - uses: actions/checkout@v3
-      - id: ghcs
-        run: echo "ghcs=$(cat ./.github/workflows/supported-ghc-versions.json)" >> $GITHUB_OUTPUT
+
+      - id: matrix-inputs
+        run: |
+          echo "ghcs=$(cat ./.github/workflows/supported-ghc-versions.json)" >> $GITHUB_OUTPUT
+          echo "packages=$(cat ./.github/workflows/packages.json | jq '[.[] | .package]')" >> $GITHUB_OUTPUT
+          echo "exclusions=$(cat ./.github/workflows/packages.json | jq '[.[] | .package as $package | (.excluded_ghcs[] | {package: $package, ghc: .})]')" >> $GITHUB_OUTPUT
+
       - id: skip_check
         uses: fkirc/[email protected]
         with:
@@ -67,6 +75,7 @@ jobs:
       # to update the ghc versions in 'caching.yml'.
       matrix:
         ghc: ${{ fromJSON(needs.pre_job.outputs.ghcs) }}
+        package: ${{ fromJSON(needs.pre_job.outputs.packages) }}
         os:
           - ubuntu-latest
           - macOS-latest
@@ -78,6 +87,7 @@ jobs:
              test: true
            - os: windows-latest
              test: true
+        exclude: ${{ fromJSON(needs.pre_job.outputs.exclusions) }}
 
     steps:
       - uses: actions/checkout@v3
@@ -87,9 +97,6 @@ jobs:
           ghc: ${{ matrix.ghc }}
           os:  ${{ runner.os }}
 
-      - name: Build
-        run: cabal build
-
       - name: Set test options
         # run the tests without parallelism, otherwise tasty will attempt to run
         # all functional test cases simultaneously which causes way too many hls
@@ -103,147 +110,18 @@ jobs:
           cache-name: cache-test-log
         with:
           path: "**/.tasty-rerun-log*"
-          key: v1-${{ runner.os }}-${{ matrix.ghc }}-test-log-${{ github.sha }}
-
-      - if: matrix.test
-        name: Test hls-graph
-        run: cabal test hls-graph --test-options="$TEST_OPTS"
-
-      - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
-        name: Test ghcide
-        # run the tests without parallelism to avoid running out of memory
-        run: cabal test ghcide --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
+          key: v1-${{ runner.os }}-${{ matrix.ghc }}-${{ matrix.package}}-test-log-${{ github.sha }}
 
-      - if: matrix.test
-        name: Test hls-plugin-api
-        run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-plugin-api --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test func-test suite
-        env:
-          HLS_TEST_EXE: hls
-          HLS_WRAPPER_TEST_EXE: hls-wrapper
-        run: cabal test func-test --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
+      - name: Build ${{ matrix.package }}
+        run: cabal build ${{ matrix.package }}
 
       - if: matrix.test
-        name: Test wrapper-test suite
+        name: Test ${{ matrix.package }}
+        # For the func and wrapper tests
         env:
           HLS_TEST_EXE: hls
           HLS_WRAPPER_TEST_EXE: hls-wrapper
-        run: cabal test wrapper-test --test-options="$TEST_OPTS --rerun-log-file .tasty-rerun-log-wrapper"
-
-      - if: matrix.test
-        name: Test hls-refactor-plugin
-        run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test 
-        name: Test hls-floskell-plugin
-        run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-class-plugin
-        run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-class-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-pragmas-plugin
-        run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-pragmas-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test && matrix.ghc != '9.4.4'
-        name: Test hls-eval-plugin
-        run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-eval-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
-        name: Test hls-haddock-comments-plugin
-        run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-splice-plugin
-        run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-splice-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test 
-        name: Test hls-stylish-haskell-plugin
-        run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test 
-        name: Test hls-ormolu-plugin
-        run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-ormolu-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-fourmolu-plugin
-        run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
-        name: Test hls-tactics-plugin test suite
-        run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-tactics-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-refine-imports-plugin test suite
-        run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-explicit-imports-plugin test suite
-        run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-call-hierarchy-plugin test suite
-        run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test && matrix.os != 'windows-latest'
-        name: Test hls-rename-plugin test suite
-        run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-rename-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-hlint-plugin test suite
-        run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-hlint-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
-        name: Test hls-stan-plugin test suite
-        run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stan-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-module-name-plugin test suite
-        run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-module-name-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-alternate-number-format-plugin test suite
-        run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-qualify-imported-names-plugin test suite
-        run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-code-range-plugin test suite
-        run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-code-range-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-change-type-signature test suite
-        run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-gadt-plugin test suit
-        run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-gadt-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-explicit-fixity-plugin test suite
-        run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-explicit-record-fields-plugin test suite
-        run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS"
-
-      ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions
-      - if: matrix.test && matrix.ghc == '8.10.7'
-        name: Test hls-cabal-fmt-plugin test suite
-        run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-cabal-plugin test suite
-        run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-plugin --test-options="$TEST_OPTS"
-
-      - if: matrix.test
-        name: Test hls-retrie-plugin test suite
-        run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-retrie-plugin --test-options="$TEST_OPTS"
+        run: cabal test ${{ matrix.package }} --test-options="$TEST_OPTS"
 
   test_post_job:
     if: always()