Add all check to CI

.github/scripts/all_check.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+Makes sure:
+
+ * All jobs are listed in the 'all' job
+ * Only existing tests are listed
+
+"""
+
+# SPDX-FileCopyrightText: 2022 Google LLC
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+import yaml
+
+CI_PATH = ".github/workflows/ci.yml"
+ALL_TEST = "all"
+
+def main():
+  ci_yml_fp = open(CI_PATH, "r")
+  ci_yml_parsed = yaml.load(ci_yml_fp, Loader=yaml.FullLoader)
+
+  all_jobs = set(ci_yml_parsed['jobs'].keys()) - {ALL_TEST}
+  all_needs = set(ci_yml_parsed["jobs"][ALL_TEST]["needs"])
+
+  if all_jobs - all_needs:
+    sys.exit(f"Not all jobs mentioned in {ALL_TEST}.needs: {all_jobs - all_needs}")
+
+  if all_needs - all_jobs:
+    sys.exit(f"Non-existing jobs found in {ALL_TEST}.needs: {all_needs - all_jobs}")
+
+
+if __name__ == '__main__':
+  main()

.github/workflows/ci.yml

@@ -131,3 +131,46 @@ jobs:
       - name: Whitespace
         run: |
           .ci/test_whitespace.sh
+
+  # Mandatory check on GitHub
+  all:
+    name: All jobs finished
+    if: always()
+    needs: [
+        cabal,
+        fourmolu,
+        linting,
+        stack,
+      ]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check dependencies for failures
+        run: |
+          # Test all dependencies for success/failure
+          set -x
+          success="${{ contains(needs.*.result, 'success') }}"
+          fail="${{ contains(needs.*.result, 'failure') }}"
+          set +x
+
+          # Test whether success/fail variables contain sane values
+          if [[ "${success}" != "true" && "${success}" != "false" ]]; then exit 1; fi
+          if [[ "${fail}"    != "true" && "${fail}"    != "false" ]]; then exit 1; fi
+
+          # We want to fail if one or more dependencies fail. For safety, we introduce
+          # a second check: if no dependencies succeeded something weird is going on.
+          if [[ "${fail}" == "true" || "${success}" == "false" ]]; then
+            echo "One or more dependency failed, or no dependency succeeded."
+            exit 1
+          fi
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install python3-yaml
+
+      - name: Check that the 'all' job depends on all other jobs
+        run: |
+          .github/scripts/all_check.py