diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e187ea92..065e0bbd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,18 +25,25 @@ jobs: - 'tests/curators/test_curator.py' - id: set-matrix + shell: bash run: | - if [[ "${{ github.event_name }}" == "push" ]]; then - # On push, run all - echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT - elif [[ "${{ steps.changes.outputs.curator }}" == "true" ]]; then - # If curator paths changed, include curator - echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT + BASE_GROUPS=$(jq -n -c '["unit-core", "unit-storage", "tutorial", "guide", "biology", "faq", "storage", "cli"]') + + if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.curator }}" == "true" ]]; then + # Run everything on push or when curator paths change + MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["curator"])}') else - # Otherwise, run all except curator - echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\"]}" >> $GITHUB_OUTPUT + # Otherwise only run base groups + MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}') fi + # Output as single line for GitHub Actions + echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT + + # Pretty print for debugging + echo "Generated matrix:" + echo "$MATRIX" | jq . + test: needs: pre-filter runs-on: ubuntu-latest @@ -81,6 +88,8 @@ jobs: - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci" + - run: nox -s configure_coverage -- '${{needs.pre-filter.outputs.matrix}}' + - name: install postgres if: ${{ matrix.group == 'faq' }} run: sudo apt-get install libpq-dev @@ -100,7 +109,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - run: nox -s "build(group='${{ matrix.group }}')" + - run: nox -s "test(group='${{ matrix.group }}')" - name: upload coverage uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index ef5d7b027..c7e070e48 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __MACOSX/ # LaminDB +.coveragerc *.db *.lndb *.jpg diff --git a/noxfile.py b/noxfile.py index 714cef2c9..f89a57e91 100644 --- a/noxfile.py +++ b/noxfile.py @@ -157,6 +157,38 @@ def install_ci(session, group): ) +@nox.session +def configure_coverage(session) -> None: + """Write a coverage config file, adding extra patterns to omit.""" + import tomlkit + + groups_str = session.posargs[0] # first positional argument + + print(groups_str) # for debugging + assert isinstance(groups_str, str) # noqa: S101 so that we don't change this away from string + + if "curator" not in groups_str: + extra_omit_patterns = ["**/curators/*"] + else: + extra_omit_patterns = [] + + # Read patterns from pyproject.toml + base_config_path = Path("pyproject.toml") + with open(base_config_path) as f: + config = tomlkit.load(f) + + # Update the omit patterns + base_patterns = config["tool"]["coverage"]["run"]["omit"] + all_patterns = base_patterns + extra_omit_patterns + config["tool"]["coverage"]["run"]["omit"] = all_patterns + + # Write back to pyproject.toml + with open(base_config_path, "w") as f: + tomlkit.dump(config, f) + + print(base_config_path.read_text()) + + @nox.session @nox.parametrize( "group", @@ -172,7 +204,7 @@ def install_ci(session, group): "cli", ], ) -def build(session, group): +def test(session, group): import lamindb as ln login_testuser2(session) @@ -182,7 +214,7 @@ def build(session, group): if group == "unit-core": run( session, - f"pytest {coverage_args} --ignore=tests/core/test_curator.py ./tests/core --durations=50", + f"pytest {coverage_args} ./tests/core --durations=50", ) elif group == "unit-storage": run(session, f"pytest {coverage_args} ./tests/storage --durations=50") @@ -210,7 +242,7 @@ def build(session, group): elif group == "curator": run( session, - f"pytest {coverage_args} tests/curators/test_curator.py --durations=50", + f"pytest {coverage_args} tests/curators --durations=50", ) elif group == "cli": run(session, f"pytest {coverage_args} ./sub/lamin-cli/tests --durations=50") diff --git a/pyproject.toml b/pyproject.toml index f241d3d39..d5f21897d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ omop = [ ] dev = [ # basic test + "tomlkit", "line_profiler", "pre-commit", "nox", @@ -214,6 +215,4 @@ exclude_lines = [ ] [tool.coverage.run] -omit = [ - "**/core/datasets/*", # Datasets don't perform any logic -] +omit = ["**/core/datasets/*", "**/migrations/*"]