Merge pull request #5 from yut23/gha-reuse-actions

Move repeated workflow logic into custom actions

.github/actions/answer-test-action/action.yml

@@ -0,0 +1,31 @@
+name: 'Run answer tests'
+description: 'Helper for workflows/answer-tests.yml'
+inputs:
+  compiler:
+    description: 'C++ compiler executable'
+    required: true
+  stdlib:
+    description: 'C++ standard library to use (libstdc++ or libc++)'
+    required: false
+    default: libstdc++
+
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      working-directory: ${{ matrix.TARGET.directory }}
+      env:
+        CXX: ${{ inputs.compiler }}
+        USE_LIBCXX: ${{ inputs.stdlib == 'libc++' && 'TRUE' || 'FALSE' }}
+      run: |
+        echo
+        echo '::group::Compiler version'
+        $CXX --version
+        echo '::endgroup::'
+        echo '::group::Build target'
+        make --silent clean
+        make build/release/${{ matrix.TARGET.name }}
+        echo '::endgroup::'
+        echo
+
+        ./run_answer_tests.sh ${{ matrix.TARGET.name }}

.github/actions/build-action/action.yml

@@ -0,0 +1,26 @@
+name: 'Build target'
+description: 'Helper for workflows/test-build.yml'
+inputs:
+  compiler:
+    description: 'C++ compiler executable'
+    required: true
+  stdlib:
+    description: 'C++ standard library to use (libstdc++ or libc++)'
+    required: false
+    default: libstdc++
+
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      working-directory: ${{ matrix.TARGET.directory }}
+      env:
+        CXX: ${{ inputs.compiler }}
+        USE_LIBCXX: ${{ inputs.stdlib == 'libc++' && 'TRUE' || 'FALSE' }}
+      run: |
+        echo
+        echo '::group::Compiler version'
+        $CXX --version
+        echo '::endgroup::'
+        echo
+        make -j4 DISABLE_SANITIZERS=TRUE build/release/${{ matrix.TARGET.name }} build/debug/${{ matrix.TARGET.name }} build/fast/${{ matrix.TARGET.name }}

.github/actions/unit-test-action/action.yml

@@ -0,0 +1,31 @@
+name: 'Run unit tests'
+description: 'Helper for workflows/unit-tests.yml'
+inputs:
+  compiler:
+    description: 'C++ compiler executable'
+    required: true
+  stdlib:
+    description: 'C++ standard library to use (libstdc++ or libc++)'
+    required: false
+    default: libstdc++
+
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      working-directory: ${{ matrix.TARGET.directory }}
+      env:
+        CXX: ${{ inputs.compiler }}
+        USE_LIBCXX: ${{ inputs.stdlib == 'libc++' && 'TRUE' || 'FALSE' }}
+      run: |
+        echo
+        echo '::group::Compiler version info'
+        $CXX --version
+        echo '::endgroup::'
+        echo '::group::Build target'
+        make --silent clean
+        make build/debug/${{ matrix.TARGET.name }}
+        echo '::endgroup::'
+        echo
+
+        make ${{ matrix.TARGET.name }}

.github/workflows/answer-tests.yml

@@ -25,52 +25,37 @@ jobs:
       matrix:
         TARGET: ${{ fromJson(needs.setup-matrix.outputs.targets) }}
     container: ${{ needs.docker-build.outputs.image }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: ${{ matrix.TARGET.directory }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Mark git checkout as safe
+        shell: bash
+        working-directory: ${{ matrix.TARGET.directory }}
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
       - name: Run answer tests (clang++)
         timeout-minutes: 2
-        run: |
-          export CXX=clang++ USE_LIBCXX=FALSE
-          echo '::group::make output'
-          $CXX --version
-          make --silent clean
-          make build/release/${{ matrix.TARGET.name }}
-          echo '::endgroup::'
-
-          ./run_answer_tests.sh ${{ matrix.TARGET.name }}
+        id: clang
+        uses: ./.github/actions/answer-test-action
+        with:
+          compiler: clang++
 
       - name: Run answer tests (g++)
         timeout-minutes: 2
-        if: always()
-        run: |
-          export CXX=g++ USE_LIBCXX=FALSE
-          echo '::group::make output'
-          $CXX --version
-          make --silent clean
-          make build/release/${{ matrix.TARGET.name }}
-          echo '::endgroup::'
-
-          ./run_answer_tests.sh ${{ matrix.TARGET.name }}
+        id: gcc
+        # ignore failures in the previous step
+        if: success() || steps.clang.conclusion == 'failure'
+        uses: ./.github/actions/answer-test-action
+        with:
+          compiler: g++
 
       - name: Run answer tests (clang++-17, libc++)
         timeout-minutes: 2
-        if: always()
-        run: |
-          export CXX=clang++-17 USE_LIBCXX=TRUE
-          echo '::group::make output'
-          $CXX --version
-          make --silent clean
-          make build/release/${{ matrix.TARGET.name }}
-          echo '::endgroup::'
-
-          ./run_answer_tests.sh ${{ matrix.TARGET.name }}
+        # ignore failures in the previous two steps
+        if: success() || steps.clang.conclusion == 'failure' || steps.gcc.conclusion == 'failure'
+        uses: ./.github/actions/answer-test-action
+        with:
+          compiler: clang++-17
+          stdlib: libc++

.github/workflows/docker-build.yml

@@ -21,11 +21,13 @@ on:
       ubuntu-version:
         required: true
         type: string
+        default: 22.04
       llvm-version:
         required: true
         type: string
+        default: 17
       rebuild:
-        required: false
+        required: true
         type: boolean
         default: true
 

.github/workflows/gen_matrix.py

@@ -9,36 +9,63 @@
 from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Generator
 
 ROOT = Path(__file__).resolve().parents[2]
+WORKFLOWS_DIR = ROOT / ".github/workflows"
 
 
 @functools.cache
-def get_includes(path: Path, *include_dirs: Path) -> frozenset[Path]:
-    assert path.suffix in {".cpp", ".hpp", ".h"}
-    include_pat = re.compile(r'#include\s+"(.*?)"')
+def get_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]:
+    if path.suffix in {".cpp", ".hpp", ".h"}:
+        include_dirs = (path.parent, *include_dirs)
+        include_pat = re.compile(r'\s*#include\s+"(.*?)"')
+
+        def find_include_paths(line: str) -> Generator[Path, None, None]:
+            if (m := include_pat.match(line)) is None:
+                return
+            for parent in include_dirs:
+                curr_path = parent / m[1]
+                if curr_path.exists():
+                    yield parent / m[1]
+                    break
+
+    elif path.parent == WORKFLOWS_DIR and path.suffix in {".yml", ".yaml"}:
+        workflow_pat = re.compile(r"uses: \./(\.github/workflows/.*\.ya?ml)")
+        action_pat = re.compile(r"uses: \./(\.github/actions/[^/]+)\b")
+        python_pat = re.compile(
+            r"\s(?:\.|\$GITHUB_WORKSPACE)/(\.github/(?:workflows|actions/[^/]+)/[^/]+\.py)"
+        )
+
+        def find_include_paths(line: str) -> Generator[Path, None, None]:
+            if (m := workflow_pat.search(line)) is not None:
+                yield ROOT / m[1]
+            if (m := python_pat.search(line)) is not None:
+                yield ROOT / m[1]
+            if (m := action_pat.search(line)) is not None:
+                for name in ("action.yml", "action.yaml"):
+                    curr_path = ROOT / m[1] / name
+                    if curr_path.exists():
+                        yield curr_path
+                        break
+
+    else:
+        return frozenset()
+
     includes: set[Path] = set()
-    include_dirs = (path.parent, *include_dirs)
     with open(path, "r") as f:
         for line in f:
-            if (m := include_pat.search(line)) is None:
-                continue
-            for parent in include_dirs:
-                curr_path = parent / m[1]
-                if not curr_path.exists():
-                    continue
-                includes.add(curr_path)
-                break
+            includes.update(find_include_paths(line.rstrip("\n")))
     return frozenset(includes)
 
 
 @functools.cache
-def get_transitive_includes(path: Path, *include_dirs: Path) -> frozenset[Path]:
-    # simple DFS on get_includes(), using functools.cache for memoization
+def get_transitive_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]:
+    # simple DFS on get_dependencies(), using functools.cache for memoization
     all_includes: set[Path] = set()
-    for p in get_includes(path, *include_dirs):
+    for p in get_dependencies(path, *include_dirs):
         all_includes.add(p)
-        all_includes.update(get_transitive_includes(p, *include_dirs))
+        all_includes.update(get_transitive_dependencies(p, *include_dirs))
     return frozenset(all_includes)
 
 
@@ -70,7 +97,7 @@ def get_deps(self, mode: str) -> frozenset[Path]:
         deps = set()
         src = self.base_dir / "src"
         deps.add(src / f"{self}.cpp")
-        for included_file in get_transitive_includes(src / f"{self}.cpp", src):
+        for included_file in get_transitive_dependencies(src / f"{self}.cpp", src):
             deps.add(included_file)
         deps.add(self.base_dir / "Makefile")
         if mode == "answer":
@@ -121,12 +148,15 @@ def validate_path(path: Path) -> None:
 
 class Matrix:
     def __init__(self, mode: str) -> None:
-        if mode == "unit":
-            target_pat = re.compile("test")
-        elif mode == "build":
+        if mode == "build":
             target_pat = re.compile("day|test")
+            workflow_path = WORKFLOWS_DIR / "test-build.yml"
         elif mode == "answer":
             target_pat = re.compile("day")
+            workflow_path = WORKFLOWS_DIR / "answer-tests.yml"
+        elif mode == "unit":
+            target_pat = re.compile("test")
+            workflow_path = WORKFLOWS_DIR / "unit-tests.yml"
         self.targets: set[Target] = set()
 
         all_targets: list[Target] = []
@@ -143,17 +173,13 @@ def __init__(self, mode: str) -> None:
             for dep in target.get_deps(mode):
                 self.file_lookup[dep].append(target)
 
-        self.file_lookup[ROOT / ".github/workflows/gen_matrix.py"] = all_targets
-        for file in ROOT.glob(".github/workflows/*.yml"):
-            with open(file, "r") as f:
-                contents = f.read()
-                if mode in file.name and (
-                    "gen_matrix.py" in contents or "generate-matrix.yml" in contents
-                ):
-                    self.file_lookup[file] = all_targets
+        self.file_lookup[workflow_path] = all_targets
+        for dep in get_transitive_dependencies(workflow_path):
+            self.file_lookup[dep] = all_targets
 
         for file in self.file_lookup:
             validate_path(file)
+        self.all_targets = all_targets
 
     def process_changed_file(self, file: Path) -> None:
         validate_path(file)

.github/workflows/test-build.yml

@@ -25,34 +25,29 @@ jobs:
       matrix:
         TARGET: ${{ fromJson(needs.setup-matrix.outputs.targets) }}
     container: ${{ needs.docker-build.outputs.image }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: ${{ matrix.TARGET.directory }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Build target (clang++)
-        run: |
-          export CXX=clang++ USE_LIBCXX=FALSE
-          $CXX --version
-          make --silent clean
-          make -j4 DISABLE_SANITIZERS=TRUE build/release/${{ matrix.TARGET.name }} build/debug/${{ matrix.TARGET.name }} build/fast/${{ matrix.TARGET.name }}
+        id: clang
+        uses: ./.github/actions/build-action
+        with:
+          compiler: clang++
 
       - name: Build target (g++)
-        if: always()
-        run: |
-          export CXX=g++ USE_LIBCXX=FALSE
-          $CXX --version
-          make --silent clean
-          make -j4 DISABLE_SANITIZERS=TRUE build/release/${{ matrix.TARGET.name }} build/debug/${{ matrix.TARGET.name }} build/fast/${{ matrix.TARGET.name }}
+        id: gcc
+        # ignore failures in the previous step
+        if: success() || steps.clang.conclusion == 'failure'
+        uses: ./.github/actions/build-action
+        with:
+          compiler: g++
 
       - name: Build target (clang++-17, libc++)
-        if: always()
-        run: |
-          export CXX=clang++-17 USE_LIBCXX=TRUE
-          $CXX --version
-          make --silent clean
-          make -j4 DISABLE_SANITIZERS=TRUE build/release/${{ matrix.TARGET.name }} build/debug/${{ matrix.TARGET.name }} build/fast/${{ matrix.TARGET.name }}
+        # ignore failures in the previous two steps
+        if: success() || steps.clang.conclusion == 'failure' || steps.gcc.conclusion == 'failure'
+        uses: ./.github/actions/build-action
+        with:
+          compiler: clang++-17
+          stdlib: libc++

.github/workflows/test-gen_matrix.yml

@@ -0,0 +1,22 @@
+name: Test gen_matrix.py
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+
+      - name: Run pytest
+        run: pytest -v .github/workflows/test_gen_matrix.py