Use retry action, don't retry otherwise

This uses a action for retrying steps, which is a bit neater, and lets us more clearly specify what's going on, as well as controlling the number of retries independently. I think it's better to specifically use this on the test suites that we believe to be flaky, rather than adding a lot of noise by doing this on every test invocation.
haskell · Dec 19, 2022 · 9c506c2 · 9c506c2
1 parent 53f61b4
commit 9c506c2
Showing 1 changed file with 55 additions and 32 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -143,20 +143,35 @@ jobs:
         run: cabal test hls-graph --test-options="$TEST_OPTS"
 
       - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
+        uses: nick-fields/retry@v2
         name: Test ghcide
-        # run the tests without parallelism to avoid running out of memory
-        run: cabal test ghcide --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
+        with:
+          # This test is sometimes flaky
+          max_attempts: 3
+          # We don't really want to gate on a timeout
+          timeout_minutes: 180
+          shell: bash
+          command: cabal test ghcide --test-options="$TEST_OPTS" 
+          new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
 
       - if: matrix.test
         name: Test hls-plugin-api
-        run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-plugin-api --test-options="$TEST_OPTS"
+        run: cabal test hls-plugin-api --test-options="$TEST_OPTS" 
 
       - if: matrix.test
+        uses: nick-fields/retry@v2
         name: Test func-test suite
         env:
           HLS_TEST_EXE: hls
           HLS_WRAPPER_TEST_EXE: hls-wrapper
-        run: cabal test func-test --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
+        with:
+          # this test is sometimes flaky
+          max_attempts: 3
+          # We don't really want to gate on a timeout
+          timeout_minutes: 180
+          shell: bash
+          command: cabal test func-test --test-options="$TEST_OPTS" 
+          new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
 
       - if: matrix.test
         name: Test wrapper-test suite
@@ -167,116 +182,124 @@ jobs:
 
       - if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-brittany-plugin
-        run: cabal test hls-brittany-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-brittany-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-brittany-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
+        uses: nick-fields/retry@v2
         name: Test hls-refactor-plugin
-        run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
+        with:
+          # This test is sometimes flaky
+          max_attempts: 2
+          # We don't really want to gate on a timeout
+          timeout_minutes: 180
+          shell: bash
+          command: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" 
+          new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-floskell-plugin
-        run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
 
       - if: matrix.test
         name: Test hls-class-plugin
-        run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-class-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-class-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-pragmas-plugin
-        run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-pragmas-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-eval-plugin
-        run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-eval-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-haddock-comments-plugin
-        run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-splice-plugin
-        run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-splice-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-stylish-haskell-plugin
-        run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-ormolu-plugin
-        run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-ormolu-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-fourmolu-plugin
-        run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-tactics-plugin test suite
-        run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-tactics-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-refine-imports-plugin test suite
-        run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-explicit-imports-plugin test suite
-        run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-call-hierarchy-plugin test suite
-        run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.os != 'windows-latest' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-rename-plugin test suite
-        run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-rename-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-hlint-plugin test suite
-        run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-hlint-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.0.1' && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-stan-plugin test suite
-        run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stan-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-module-name-plugin test suite
-        run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-module-name-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-alternate-number-format-plugin test suite
-        run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-qualify-imported-names-plugin test suite
-        run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-code-range-plugin test suite
-        run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-code-range-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-change-type-signature test suite
-        run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
         name: Test hls-gadt-plugin test suit
-        run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-gadt-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-explicit-fixity-plugin test suite
-        run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-explicit-record-fields-plugin test suite
-        run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" 
 
       ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions
       - if: matrix.test && matrix.ghc == '8.10.7'
         name: Test hls-cabal-fmt-plugin test suite
-        run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS"
+        run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" 
 
       - if: matrix.test
         name: Test hls-cabal-plugin test suite
-        run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-plugin --test-options="$TEST_OPTS"
+        run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" 
 
   test_post_job:
     if: always()