Skip to content

Commit aea1f8f

Browse files
michaelpjmichaelpj
committed
Try turning the test job into a matrix build
This should reduce the latency of CI, since it will do all the tests for all the packages in parallel. It also makes it easier and faster to retry if the tests for one package fail.
1 parent efbf253 commit aea1f8f

File tree

2 files changed

+59
-142
lines changed

2 files changed

+59
-142
lines changed

‎.github/workflows/packages.json

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
[
2+
{ "package": "haskell-language-server", "excluded_ghcs" : [] }
3+
, { "package": "hie-compat", "excluded_ghcs" : [] }
4+
, { "package": "shake-bench", "excluded_ghcs" : [] }
5+
, { "package": "hls-graph", "excluded_ghcs" : [] }
6+
, { "package": "ghcide", "excluded_ghcs" : [] }
7+
, { "package": "ghcide-bench", "excluded_ghcs" : [] }
8+
, { "package": "ghcide-test-utils", "excluded_ghcs" : [] }
9+
, { "package": "hls-plugin-api", "excluded_ghcs" : [] }
10+
, { "package": "hls-test-utils", "excluded_ghcs" : [] }
11+
, { "package": "hls-cabal-plugin", "excluded_ghcs" : [] }
12+
, { "package": "hls-cabal-fmt-plugin", "excluded_ghcs" : ["8.10.7"] }
13+
, { "package": "hls-tactics-plugin", "excluded_ghcs" : ["9.2.5", "9.4.4"] }
14+
, { "package": "hls-stylish-haskell-plugin", "excluded_ghcs" : [] }
15+
, { "package": "hls-fourmolu-plugin", "excluded_ghcs" : [] }
16+
, { "package": "hls-class-plugin", "excluded_ghcs" : [] }
17+
, { "package": "hls-eval-plugin", "excluded_ghcs" : ["9.4.4"] }
18+
, { "package": "hls-explicit-imports-plugin", "excluded_ghcs" : [] }
19+
, { "package": "hls-refine-imports-plugin", "excluded_ghcs" : [] }
20+
, { "package": "hls-hlint-plugin", "excluded_ghcs" : [] }
21+
, { "package": "hls-rename-plugin", "excluded_ghcs" : [] }
22+
, { "package": "hls-retrie-plugin", "excluded_ghcs" : [] }
23+
, { "package": "hls-haddock-comments-plugin", "excluded_ghcs" : ["9.2.5", "9.4.4"] }
24+
, { "package": "hls-splice-plugin", "excluded_ghcs" : [] }
25+
, { "package": "hls-floskell-plugin", "excluded_ghcs" : [] }
26+
, { "package": "hls-pragmas-plugin", "excluded_ghcs" : [] }
27+
, { "package": "hls-module-name-plugin", "excluded_ghcs" : [] }
28+
, { "package": "hls-ormolu-plugin", "excluded_ghcs" : [] }
29+
, { "package": "hls-call-hierarchy-plugin", "excluded_ghcs" : [] }
30+
, { "package": "hls-alternate-number-format-plugin", "excluded_ghcs" : [] }
31+
, { "package": "hls-qualify-imported-names-plugin", "excluded_ghcs" : [] }
32+
, { "package": "hls-code-range-plugin", "excluded_ghcs" : [] }
33+
, { "package": "hls-change-type-signature-plugin", "excluded_ghcs" : [] }
34+
, { "package": "hls-stan-plugin", "excluded_ghcs" : [ "9.0.2","9.2.5","9.4.4" ] }
35+
, { "package": "hls-gadt-plugin", "excluded_ghcs" : [] }
36+
, { "package": "hls-explicit-fixity-plugin", "excluded_ghcs" : [] }
37+
, { "package": "hls-explicit-record-fields-plugin", "excluded_ghcs" : [] }
38+
, { "package": "hls-refactor-plugin", "excluded_ghcs" : [] }
39+
]

‎.github/workflows/test.yml

Lines changed: 20 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,20 @@ jobs:
2020
outputs:
2121
should_skip: ${{ steps.skip_check.outputs.should_skip }}
2222
should_skip_ghcide: ${{ steps.skip_ghcide_check.outputs.should_skip }}
23-
ghcs: ${{ steps.ghcs.outputs.ghcs }}
23+
ghcs: ${{ steps.matrix-inputs.outputs.ghcs }}
24+
packages: ${{ steps.matrix-inputs.outputs.packages }}
25+
exclusions: ${{ steps.matrix-inputs.outputs.exclusions }}
26+
2427
steps:
25-
# Need the repo checked out in order to read the file
28+
# Need the repo checked out in order to read the files
2629
- uses: actions/checkout@v3
27-
- id: ghcs
28-
run: echo "ghcs=$(cat ./.github/workflows/supported-ghc-versions.json)" >> $GITHUB_OUTPUT
30+
31+
- id: matrix-inputs
32+
run: |
33+
echo "ghcs=$(cat ./.github/workflows/supported-ghc-versions.json)" >> $GITHUB_OUTPUT
34+
echo "packages=$(cat ./.github/workflows/packages.json | jq '[.[] | .package]')" >> $GITHUB_OUTPUT
35+
echo "exclusions=$(cat ./.github/workflows/packages.json | jq '[.[] | .package as $package | (.excluded_ghcs[] | {package: $package, ghc: .})]')" >> $GITHUB_OUTPUT
36+
2937
- id: skip_check
3038
uses: fkirc/[email protected]
3139
with:
@@ -67,6 +75,7 @@ jobs:
6775
# to update the ghc versions in 'caching.yml'.
6876
matrix:
6977
ghc: ${{ fromJSON(needs.pre_job.outputs.ghcs) }}
78+
package: ${{ fromJSON(needs.pre_job.outputs.packages) }}
7079
os:
7180
- ubuntu-latest
7281
- macOS-latest
@@ -78,6 +87,7 @@ jobs:
7887
test: true
7988
- os: windows-latest
8089
test: true
90+
exclude: ${{ fromJSON(needs.pre_job.outputs.exclusions) }}
8191

8292
steps:
8393
- uses: actions/checkout@v3
@@ -87,9 +97,6 @@ jobs:
8797
ghc: ${{ matrix.ghc }}
8898
os: ${{ runner.os }}
8999

90-
- name: Build
91-
run: cabal build
92-
93100
- name: Set test options
94101
# run the tests without parallelism, otherwise tasty will attempt to run
95102
# all functional test cases simultaneously which causes way too many hls
@@ -103,147 +110,18 @@ jobs:
103110
cache-name: cache-test-log
104111
with:
105112
path: "**/.tasty-rerun-log*"
106-
key: v1-${{ runner.os }}-${{ matrix.ghc }}-test-log-${{ github.sha }}
107-
108-
- if: matrix.test
109-
name: Test hls-graph
110-
run: cabal test hls-graph --test-options="$TEST_OPTS"
111-
112-
- if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
113-
name: Test ghcide
114-
# run the tests without parallelism to avoid running out of memory
115-
run: cabal test ghcide --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
113+
key: v1-${{ runner.os }}-${{ matrix.ghc }}-${{ matrix.package}}-test-log-${{ github.sha }}
116114

117-
- if: matrix.test
118-
name: Test hls-plugin-api
119-
run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-plugin-api --test-options="$TEST_OPTS"
120-
121-
- if: matrix.test
122-
name: Test func-test suite
123-
env:
124-
HLS_TEST_EXE: hls
125-
HLS_WRAPPER_TEST_EXE: hls-wrapper
126-
run: cabal test func-test --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
115+
- name: Build ${{ matrix.package }}
116+
run: cabal build ${{ matrix.package }}
127117

128118
- if: matrix.test
129-
name: Test wrapper-test suite
119+
name: Test ${{ matrix.package }}
120+
# For the func and wrapper tests
130121
env:
131122
HLS_TEST_EXE: hls
132123
HLS_WRAPPER_TEST_EXE: hls-wrapper
133-
run: cabal test wrapper-test --test-options="$TEST_OPTS --rerun-log-file .tasty-rerun-log-wrapper"
134-
135-
- if: matrix.test
136-
name: Test hls-refactor-plugin
137-
run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
138-
139-
- if: matrix.test
140-
name: Test hls-floskell-plugin
141-
run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
142-
143-
- if: matrix.test
144-
name: Test hls-class-plugin
145-
run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-class-plugin --test-options="$TEST_OPTS"
146-
147-
- if: matrix.test
148-
name: Test hls-pragmas-plugin
149-
run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-pragmas-plugin --test-options="$TEST_OPTS"
150-
151-
- if: matrix.test && matrix.ghc != '9.4.4'
152-
name: Test hls-eval-plugin
153-
run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-eval-plugin --test-options="$TEST_OPTS"
154-
155-
- if: matrix.test && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
156-
name: Test hls-haddock-comments-plugin
157-
run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS"
158-
159-
- if: matrix.test
160-
name: Test hls-splice-plugin
161-
run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-splice-plugin --test-options="$TEST_OPTS"
162-
163-
- if: matrix.test
164-
name: Test hls-stylish-haskell-plugin
165-
run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS"
166-
167-
- if: matrix.test
168-
name: Test hls-ormolu-plugin
169-
run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-ormolu-plugin --test-options="$TEST_OPTS"
170-
171-
- if: matrix.test
172-
name: Test hls-fourmolu-plugin
173-
run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS"
174-
175-
- if: matrix.test && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
176-
name: Test hls-tactics-plugin test suite
177-
run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-tactics-plugin --test-options="$TEST_OPTS"
178-
179-
- if: matrix.test
180-
name: Test hls-refine-imports-plugin test suite
181-
run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS"
182-
183-
- if: matrix.test
184-
name: Test hls-explicit-imports-plugin test suite
185-
run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS"
186-
187-
- if: matrix.test
188-
name: Test hls-call-hierarchy-plugin test suite
189-
run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS"
190-
191-
- if: matrix.test && matrix.os != 'windows-latest'
192-
name: Test hls-rename-plugin test suite
193-
run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-rename-plugin --test-options="$TEST_OPTS"
194-
195-
- if: matrix.test
196-
name: Test hls-hlint-plugin test suite
197-
run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-hlint-plugin --test-options="$TEST_OPTS"
198-
199-
- if: matrix.test && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.4'
200-
name: Test hls-stan-plugin test suite
201-
run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stan-plugin --test-options="$TEST_OPTS"
202-
203-
- if: matrix.test
204-
name: Test hls-module-name-plugin test suite
205-
run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-module-name-plugin --test-options="$TEST_OPTS"
206-
207-
- if: matrix.test
208-
name: Test hls-alternate-number-format-plugin test suite
209-
run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS"
210-
211-
- if: matrix.test
212-
name: Test hls-qualify-imported-names-plugin test suite
213-
run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS"
214-
215-
- if: matrix.test
216-
name: Test hls-code-range-plugin test suite
217-
run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-code-range-plugin --test-options="$TEST_OPTS"
218-
219-
- if: matrix.test
220-
name: Test hls-change-type-signature test suite
221-
run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS"
222-
223-
- if: matrix.test
224-
name: Test hls-gadt-plugin test suit
225-
run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-gadt-plugin --test-options="$TEST_OPTS"
226-
227-
- if: matrix.test
228-
name: Test hls-explicit-fixity-plugin test suite
229-
run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS"
230-
231-
- if: matrix.test
232-
name: Test hls-explicit-record-fields-plugin test suite
233-
run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS"
234-
235-
## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions
236-
- if: matrix.test && matrix.ghc == '8.10.7'
237-
name: Test hls-cabal-fmt-plugin test suite
238-
run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS"
239-
240-
- if: matrix.test
241-
name: Test hls-cabal-plugin test suite
242-
run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-plugin --test-options="$TEST_OPTS"
243-
244-
- if: matrix.test
245-
name: Test hls-retrie-plugin test suite
246-
run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-retrie-plugin --test-options="$TEST_OPTS"
124+
run: cabal test ${{ matrix.package }} --test-options="$TEST_OPTS"
247125

248126
test_post_job:
249127
if: always()

0 commit comments

Comments
 (0)