Teach gen_matrix.py to follow workflow file dependencies

Also add a small test suite and workflow to run it.

.github/workflows/gen_matrix.py

@@ -9,36 +9,63 @@
 from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Generator
 
 ROOT = Path(__file__).resolve().parents[2]
+WORKFLOWS_DIR = ROOT / ".github/workflows"
 
 
 @functools.cache
-def get_includes(path: Path, *include_dirs: Path) -> frozenset[Path]:
-    assert path.suffix in {".cpp", ".hpp", ".h"}
-    include_pat = re.compile(r'#include\s+"(.*?)"')
+def get_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]:
+    if path.suffix in {".cpp", ".hpp", ".h"}:
+        include_dirs = (path.parent, *include_dirs)
+        include_pat = re.compile(r'\s*#include\s+"(.*?)"')
+
+        def find_include_paths(line: str) -> Generator[Path, None, None]:
+            if (m := include_pat.match(line)) is None:
+                return
+            for parent in include_dirs:
+                curr_path = parent / m[1]
+                if curr_path.exists():
+                    yield parent / m[1]
+                    break
+
+    elif path.parent == WORKFLOWS_DIR and path.suffix in {".yml", ".yaml"}:
+        workflow_pat = re.compile(r"uses: \./(\.github/workflows/.*\.ya?ml)")
+        action_pat = re.compile(r"uses: \./(\.github/actions/[^/]+)\b")
+        python_pat = re.compile(
+            r"\s(?:\.|\$GITHUB_WORKSPACE)/(\.github/(?:workflows|actions/[^/]+)/[^/]+\.py)"
+        )
+
+        def find_include_paths(line: str) -> Generator[Path, None, None]:
+            if (m := workflow_pat.search(line)) is not None:
+                yield ROOT / m[1]
+            if (m := python_pat.search(line)) is not None:
+                yield ROOT / m[1]
+            if (m := action_pat.search(line)) is not None:
+                for name in ("action.yml", "action.yaml"):
+                    curr_path = ROOT / m[1] / name
+                    if curr_path.exists():
+                        yield curr_path
+                        break
+
+    else:
+        return frozenset()
+
     includes: set[Path] = set()
-    include_dirs = (path.parent, *include_dirs)
     with open(path, "r") as f:
         for line in f:
-            if (m := include_pat.search(line)) is None:
-                continue
-            for parent in include_dirs:
-                curr_path = parent / m[1]
-                if not curr_path.exists():
-                    continue
-                includes.add(curr_path)
-                break
+            includes.update(find_include_paths(line.rstrip("\n")))
     return frozenset(includes)
 
 
 @functools.cache
-def get_transitive_includes(path: Path, *include_dirs: Path) -> frozenset[Path]:
-    # simple DFS on get_includes(), using functools.cache for memoization
+def get_transitive_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]:
+    # simple DFS on get_dependencies(), using functools.cache for memoization
     all_includes: set[Path] = set()
-    for p in get_includes(path, *include_dirs):
+    for p in get_dependencies(path, *include_dirs):
         all_includes.add(p)
-        all_includes.update(get_transitive_includes(p, *include_dirs))
+        all_includes.update(get_transitive_dependencies(p, *include_dirs))
     return frozenset(all_includes)
 
 
@@ -70,7 +97,7 @@ def get_deps(self, mode: str) -> frozenset[Path]:
         deps = set()
         src = self.base_dir / "src"
         deps.add(src / f"{self}.cpp")
-        for included_file in get_transitive_includes(src / f"{self}.cpp", src):
+        for included_file in get_transitive_dependencies(src / f"{self}.cpp", src):
             deps.add(included_file)
         deps.add(self.base_dir / "Makefile")
         if mode == "answer":
@@ -121,12 +148,15 @@ def validate_path(path: Path) -> None:
 
 class Matrix:
     def __init__(self, mode: str) -> None:
-        if mode == "unit":
-            target_pat = re.compile("test")
-        elif mode == "build":
+        if mode == "build":
             target_pat = re.compile("day|test")
+            workflow_path = WORKFLOWS_DIR / "test-build.yml"
         elif mode == "answer":
             target_pat = re.compile("day")
+            workflow_path = WORKFLOWS_DIR / "answer-tests.yml"
+        elif mode == "unit":
+            target_pat = re.compile("test")
+            workflow_path = WORKFLOWS_DIR / "unit-tests.yml"
         self.targets: set[Target] = set()
 
         all_targets: list[Target] = []
@@ -143,17 +173,13 @@ def __init__(self, mode: str) -> None:
             for dep in target.get_deps(mode):
                 self.file_lookup[dep].append(target)
 
-        self.file_lookup[ROOT / ".github/workflows/gen_matrix.py"] = all_targets
-        for file in ROOT.glob(".github/workflows/*.yml"):
-            with open(file, "r") as f:
-                contents = f.read()
-                if mode in file.name and (
-                    "gen_matrix.py" in contents or "generate-matrix.yml" in contents
-                ):
-                    self.file_lookup[file] = all_targets
+        self.file_lookup[workflow_path] = all_targets
+        for dep in get_transitive_dependencies(workflow_path):
+            self.file_lookup[dep] = all_targets
 
         for file in self.file_lookup:
             validate_path(file)
+        self.all_targets = all_targets
 
     def process_changed_file(self, file: Path) -> None:
         validate_path(file)

.github/workflows/test-gen_matrix.yml

@@ -0,0 +1,22 @@
+name: Test gen_matrix.py
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+
+      - name: Run pytest
+        run: pytest -v .github/workflows/test_gen_matrix.py

.github/workflows/test_gen_matrix.py

@@ -0,0 +1,95 @@
+from gen_matrix import ROOT, WORKFLOWS_DIR, Matrix, get_dependencies
+
+ACTIONS_DIR = ROOT / ".github/actions"
+
+
+def test_get_dependencies_cpp():
+    base_dir = ROOT / "2023"
+    src = base_dir / "src"
+    assert get_dependencies(src / "day01.cpp") == {src / "lib.hpp"}
+    assert get_dependencies(src / "day05.cpp") == {src / "day05.hpp", src / "lib.hpp"}
+    assert get_dependencies(src / "day05.hpp") == {src / "lib.hpp"}
+    assert get_dependencies(src / "test05.cpp") == {
+        src / "day05.hpp",
+        src / "unit_test/unit_test.hpp",
+    }
+
+
+def test_get_dependencies_workflows():
+    assert get_dependencies(WORKFLOWS_DIR / "unit-tests.yml") == {
+        WORKFLOWS_DIR / "generate-matrix.yml",
+        WORKFLOWS_DIR / "docker-build.yml",
+        ACTIONS_DIR / "unit-test-action/action.yml",
+    }
+    assert get_dependencies(WORKFLOWS_DIR / "test-build.yml") == {
+        WORKFLOWS_DIR / "generate-matrix.yml",
+        WORKFLOWS_DIR / "docker-build.yml",
+        ACTIONS_DIR / "build-action/action.yml",
+    }
+    assert get_dependencies(WORKFLOWS_DIR / "generate-matrix.yml") == {
+        WORKFLOWS_DIR / "gen_matrix.py"
+    }
+    assert get_dependencies(WORKFLOWS_DIR / "answer-tests.yml") == {
+        WORKFLOWS_DIR / "generate-matrix.yml",
+        WORKFLOWS_DIR / "docker-build.yml",
+        ACTIONS_DIR / "answer-test-action/action.yml",
+    }
+
+
+def matrix_helper(*changed_files: str) -> dict[str, set[str]]:
+    result = {}
+    for mode in ("build", "answer", "unit"):
+        matrix = Matrix(mode)
+        for f in changed_files:
+            matrix.process_changed_file(ROOT / f)
+        result[mode] = {
+            "{directory}/{name}".format(**target.to_dict()) for target in matrix.targets
+        }
+    return result
+
+
+def test_matrix():
+    # pylint: disable=use-dict-literal
+    everything = matrix_helper("2022/Makefile", "2023/Makefile")
+
+    # C++ source files
+    assert matrix_helper("2023/src/day05.cpp") == dict(
+        build={"2023/day05"}, answer={"2023/day05"}, unit=set()
+    )
+    assert matrix_helper("2023/src/day05.hpp") == dict(
+        build={"2023/day05", "2023/test05"}, answer={"2023/day05"}, unit={"2023/test05"}
+    )
+    assert matrix_helper("2023/src/test05.cpp") == dict(
+        build={"2023/test05"}, answer=set(), unit={"2023/test05"}
+    )
+
+    # answer tests
+    assert matrix_helper("2023/answer_tests/day01/example2.txt") == dict(
+        build=set(), answer={"2023/day01"}, unit=set()
+    )
+
+    # workflow files
+    assert matrix_helper(".github/workflows/gen_matrix.py") == everything
+    assert matrix_helper(".github/workflows/generate-matrix.yml") == everything
+    assert matrix_helper(".github/workflows/docker-build.yml") == everything
+
+    assert matrix_helper(".github/workflows/test-build.yml") == dict(
+        build=everything["build"], answer=set(), unit=set()
+    )
+    assert matrix_helper(".github/workflows/answer-tests.yml") == dict(
+        build=set(), answer=everything["answer"], unit=set()
+    )
+    assert matrix_helper(".github/workflows/unit-tests.yml") == dict(
+        build=set(), answer=set(), unit=everything["unit"]
+    )
+
+    # workflow action helpers
+    assert matrix_helper(".github/actions/build-action/action.yml") == matrix_helper(
+        ".github/workflows/test-build.yml"
+    )
+    assert matrix_helper(
+        ".github/actions/answer-test-action/action.yml"
+    ) == matrix_helper(".github/workflows/answer-tests.yml")
+    assert matrix_helper(
+        ".github/actions/unit-test-action/action.yml"
+    ) == matrix_helper(".github/workflows/unit-tests.yml")