From 853525481fa9adff7f28bedca5510b9961fd83a3 Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 23 Jan 2024 18:50:23 -0500 Subject: [PATCH] Teach gen_matrix.py to follow workflow file dependencies Also add a small test suite and workflow to run it. --- .github/workflows/gen_matrix.py | 82 +++++++++++++++-------- .github/workflows/test-gen_matrix.yml | 22 +++++++ .github/workflows/test_gen_matrix.py | 95 +++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/test-gen_matrix.yml create mode 100644 .github/workflows/test_gen_matrix.py diff --git a/.github/workflows/gen_matrix.py b/.github/workflows/gen_matrix.py index 69a7a5c..1b70def 100755 --- a/.github/workflows/gen_matrix.py +++ b/.github/workflows/gen_matrix.py @@ -9,36 +9,63 @@ from collections import defaultdict from dataclasses import dataclass from pathlib import Path +from typing import Generator ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS_DIR = ROOT / ".github/workflows" @functools.cache -def get_includes(path: Path, *include_dirs: Path) -> frozenset[Path]: - assert path.suffix in {".cpp", ".hpp", ".h"} - include_pat = re.compile(r'#include\s+"(.*?)"') +def get_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]: + if path.suffix in {".cpp", ".hpp", ".h"}: + include_dirs = (path.parent, *include_dirs) + include_pat = re.compile(r'\s*#include\s+"(.*?)"') + + def find_include_paths(line: str) -> Generator[Path, None, None]: + if (m := include_pat.match(line)) is None: + return + for parent in include_dirs: + curr_path = parent / m[1] + if curr_path.exists(): + yield parent / m[1] + break + + elif path.parent == WORKFLOWS_DIR and path.suffix in {".yml", ".yaml"}: + workflow_pat = re.compile(r"uses: \./(\.github/workflows/.*\.ya?ml)") + action_pat = re.compile(r"uses: \./(\.github/actions/[^/]+)\b") + python_pat = re.compile( + r"\s(?:\.|\$GITHUB_WORKSPACE)/(\.github/(?:workflows|actions/[^/]+)/[^/]+\.py)" + ) + + def find_include_paths(line: str) -> Generator[Path, None, None]: + if (m := workflow_pat.search(line)) is not None: + yield ROOT / m[1] + if (m := python_pat.search(line)) is not None: + yield ROOT / m[1] + if (m := action_pat.search(line)) is not None: + for name in ("action.yml", "action.yaml"): + curr_path = ROOT / m[1] / name + if curr_path.exists(): + yield curr_path + break + + else: + return frozenset() + includes: set[Path] = set() - include_dirs = (path.parent, *include_dirs) with open(path, "r") as f: for line in f: - if (m := include_pat.search(line)) is None: - continue - for parent in include_dirs: - curr_path = parent / m[1] - if not curr_path.exists(): - continue - includes.add(curr_path) - break + includes.update(find_include_paths(line.rstrip("\n"))) return frozenset(includes) @functools.cache -def get_transitive_includes(path: Path, *include_dirs: Path) -> frozenset[Path]: - # simple DFS on get_includes(), using functools.cache for memoization +def get_transitive_dependencies(path: Path, *include_dirs: Path) -> frozenset[Path]: + # simple DFS on get_dependencies(), using functools.cache for memoization all_includes: set[Path] = set() - for p in get_includes(path, *include_dirs): + for p in get_dependencies(path, *include_dirs): all_includes.add(p) - all_includes.update(get_transitive_includes(p, *include_dirs)) + all_includes.update(get_transitive_dependencies(p, *include_dirs)) return frozenset(all_includes) @@ -70,7 +97,7 @@ def get_deps(self, mode: str) -> frozenset[Path]: deps = set() src = self.base_dir / "src" deps.add(src / f"{self}.cpp") - for included_file in get_transitive_includes(src / f"{self}.cpp", src): + for included_file in get_transitive_dependencies(src / f"{self}.cpp", src): deps.add(included_file) deps.add(self.base_dir / "Makefile") if mode == "answer": @@ -121,12 +148,15 @@ def validate_path(path: Path) -> None: class Matrix: def __init__(self, mode: str) -> None: - if mode == "unit": - target_pat = re.compile("test") - elif mode == "build": + if mode == "build": target_pat = re.compile("day|test") + workflow_path = WORKFLOWS_DIR / "test-build.yml" elif mode == "answer": target_pat = re.compile("day") + workflow_path = WORKFLOWS_DIR / "answer-tests.yml" + elif mode == "unit": + target_pat = re.compile("test") + workflow_path = WORKFLOWS_DIR / "unit-tests.yml" self.targets: set[Target] = set() all_targets: list[Target] = [] @@ -143,17 +173,13 @@ def __init__(self, mode: str) -> None: for dep in target.get_deps(mode): self.file_lookup[dep].append(target) - self.file_lookup[ROOT / ".github/workflows/gen_matrix.py"] = all_targets - for file in ROOT.glob(".github/workflows/*.yml"): - with open(file, "r") as f: - contents = f.read() - if mode in file.name and ( - "gen_matrix.py" in contents or "generate-matrix.yml" in contents - ): - self.file_lookup[file] = all_targets + self.file_lookup[workflow_path] = all_targets + for dep in get_transitive_dependencies(workflow_path): + self.file_lookup[dep] = all_targets for file in self.file_lookup: validate_path(file) + self.all_targets = all_targets def process_changed_file(self, file: Path) -> None: validate_path(file) diff --git a/.github/workflows/test-gen_matrix.yml b/.github/workflows/test-gen_matrix.yml new file mode 100644 index 0000000..5ebfbca --- /dev/null +++ b/.github/workflows/test-gen_matrix.yml @@ -0,0 +1,22 @@ +name: Test gen_matrix.py + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + + - name: Run pytest + run: pytest -v .github/workflows/test_gen_matrix.py diff --git a/.github/workflows/test_gen_matrix.py b/.github/workflows/test_gen_matrix.py new file mode 100644 index 0000000..8bdbf73 --- /dev/null +++ b/.github/workflows/test_gen_matrix.py @@ -0,0 +1,95 @@ +from gen_matrix import ROOT, WORKFLOWS_DIR, Matrix, get_dependencies + +ACTIONS_DIR = ROOT / ".github/actions" + + +def test_get_dependencies_cpp(): + base_dir = ROOT / "2023" + src = base_dir / "src" + assert get_dependencies(src / "day01.cpp") == {src / "lib.hpp"} + assert get_dependencies(src / "day05.cpp") == {src / "day05.hpp", src / "lib.hpp"} + assert get_dependencies(src / "day05.hpp") == {src / "lib.hpp"} + assert get_dependencies(src / "test05.cpp") == { + src / "day05.hpp", + src / "unit_test/unit_test.hpp", + } + + +def test_get_dependencies_workflows(): + assert get_dependencies(WORKFLOWS_DIR / "unit-tests.yml") == { + WORKFLOWS_DIR / "generate-matrix.yml", + WORKFLOWS_DIR / "docker-build.yml", + ACTIONS_DIR / "unit-test-action/action.yml", + } + assert get_dependencies(WORKFLOWS_DIR / "test-build.yml") == { + WORKFLOWS_DIR / "generate-matrix.yml", + WORKFLOWS_DIR / "docker-build.yml", + ACTIONS_DIR / "build-action/action.yml", + } + assert get_dependencies(WORKFLOWS_DIR / "generate-matrix.yml") == { + WORKFLOWS_DIR / "gen_matrix.py" + } + assert get_dependencies(WORKFLOWS_DIR / "answer-tests.yml") == { + WORKFLOWS_DIR / "generate-matrix.yml", + WORKFLOWS_DIR / "docker-build.yml", + ACTIONS_DIR / "answer-test-action/action.yml", + } + + +def matrix_helper(*changed_files: str) -> dict[str, set[str]]: + result = {} + for mode in ("build", "answer", "unit"): + matrix = Matrix(mode) + for f in changed_files: + matrix.process_changed_file(ROOT / f) + result[mode] = { + "{directory}/{name}".format(**target.to_dict()) for target in matrix.targets + } + return result + + +def test_matrix(): + # pylint: disable=use-dict-literal + everything = matrix_helper("2022/Makefile", "2023/Makefile") + + # C++ source files + assert matrix_helper("2023/src/day05.cpp") == dict( + build={"2023/day05"}, answer={"2023/day05"}, unit=set() + ) + assert matrix_helper("2023/src/day05.hpp") == dict( + build={"2023/day05", "2023/test05"}, answer={"2023/day05"}, unit={"2023/test05"} + ) + assert matrix_helper("2023/src/test05.cpp") == dict( + build={"2023/test05"}, answer=set(), unit={"2023/test05"} + ) + + # answer tests + assert matrix_helper("2023/answer_tests/day01/example2.txt") == dict( + build=set(), answer={"2023/day01"}, unit=set() + ) + + # workflow files + assert matrix_helper(".github/workflows/gen_matrix.py") == everything + assert matrix_helper(".github/workflows/generate-matrix.yml") == everything + assert matrix_helper(".github/workflows/docker-build.yml") == everything + + assert matrix_helper(".github/workflows/test-build.yml") == dict( + build=everything["build"], answer=set(), unit=set() + ) + assert matrix_helper(".github/workflows/answer-tests.yml") == dict( + build=set(), answer=everything["answer"], unit=set() + ) + assert matrix_helper(".github/workflows/unit-tests.yml") == dict( + build=set(), answer=set(), unit=everything["unit"] + ) + + # workflow action helpers + assert matrix_helper(".github/actions/build-action/action.yml") == matrix_helper( + ".github/workflows/test-build.yml" + ) + assert matrix_helper( + ".github/actions/answer-test-action/action.yml" + ) == matrix_helper(".github/workflows/answer-tests.yml") + assert matrix_helper( + ".github/actions/unit-test-action/action.yml" + ) == matrix_helper(".github/workflows/unit-tests.yml")