👷 Adjust coverage calculation if a job isn't run and prettify build.yml

.github/workflows/build.yml

@@ -25,18 +25,25 @@ jobs:
               - 'tests/curators/test_curator.py'
 
       - id: set-matrix
+        shell: bash
         run: |
-          if [[ "${{ github.event_name }}" == "push" ]]; then
-            # On push, run all
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
-          elif [[ "${{ steps.changes.outputs.curator }}" == "true" ]]; then
-            # If curator paths changed, include curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
+          BASE_GROUPS=$(jq -n -c '["unit-core", "unit-storage", "tutorial", "guide", "biology", "faq", "storage", "cli"]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.curator }}" == "true" ]]; then
+            # Run everything on push or when curator paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["curator"])}')
           else
-            # Otherwise, run all except curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\"]}" >> $GITHUB_OUTPUT
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
           fi
 
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
   test:
     needs: pre-filter
     runs-on: ubuntu-latest
@@ -81,6 +88,8 @@ jobs:
 
       - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
 
+      - run: nox -s configure_coverage -- '${{needs.pre-filter.outputs.matrix}}'
+
       - name: install postgres
         if: ${{ matrix.group == 'faq' }}
         run: sudo apt-get install libpq-dev
@@ -100,7 +109,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-      - run: nox -s "build(group='${{ matrix.group }}')"
+      - run: nox -s "test(group='${{ matrix.group }}')"
 
       - name: upload coverage
         uses: actions/upload-artifact@v4

.gitignore

@@ -1,6 +1,7 @@
 __MACOSX/
 
 # LaminDB
+.coveragerc
 *.db
 *.lndb
 *.jpg

noxfile.py

@@ -157,6 +157,38 @@ def install_ci(session, group):
             )
 
 
+@nox.session
+def configure_coverage(session) -> None:
+    """Write a coverage config file, adding extra patterns to omit."""
+    import tomlkit
+
+    groups_str = session.posargs[0]  # first positional argument
+
+    print(groups_str)  # for debugging
+    assert isinstance(groups_str, str)  # noqa: S101 so that we don't change this away from string
+
+    if "curator" not in groups_str:
+        extra_omit_patterns = ["**/curators/*"]
+    else:
+        extra_omit_patterns = []
+
+    # Read patterns from pyproject.toml
+    base_config_path = Path("pyproject.toml")
+    with open(base_config_path) as f:
+        config = tomlkit.load(f)
+
+    # Update the omit patterns
+    base_patterns = config["tool"]["coverage"]["run"]["omit"]
+    all_patterns = base_patterns + extra_omit_patterns
+    config["tool"]["coverage"]["run"]["omit"] = all_patterns
+
+    # Write back to pyproject.toml
+    with open(base_config_path, "w") as f:
+        tomlkit.dump(config, f)
+
+    print(base_config_path.read_text())
+
+
 @nox.session
 @nox.parametrize(
     "group",
@@ -172,7 +204,7 @@ def install_ci(session, group):
         "cli",
     ],
 )
-def build(session, group):
+def test(session, group):
     import lamindb as ln
 
     login_testuser2(session)
@@ -182,7 +214,7 @@ def build(session, group):
     if group == "unit-core":
         run(
             session,
-            f"pytest {coverage_args} --ignore=tests/core/test_curator.py ./tests/core --durations=50",
+            f"pytest {coverage_args} ./tests/core --durations=50",
         )
     elif group == "unit-storage":
         run(session, f"pytest {coverage_args} ./tests/storage --durations=50")
@@ -210,7 +242,7 @@ def build(session, group):
     elif group == "curator":
         run(
             session,
-            f"pytest {coverage_args} tests/curators/test_curator.py --durations=50",
+            f"pytest {coverage_args} tests/curators --durations=50",
         )
     elif group == "cli":
         run(session, f"pytest {coverage_args} ./sub/lamin-cli/tests --durations=50")

pyproject.toml

@@ -69,6 +69,7 @@ omop = [
 ]
 dev = [
     # basic test
+    "tomlkit",
     "line_profiler",
     "pre-commit",
     "nox",
@@ -214,6 +215,4 @@ exclude_lines = [
 ]
 
 [tool.coverage.run]
-omit = [
-    "**/core/datasets/*",  # Datasets don't perform any logic
-]
+omit = ["**/core/datasets/*", "**/migrations/*"]