[misc] move ubuntu runner to self-hosted for py-unittest workflow (#924)

* move ubuntu runner to self-hosted * remove not very useful test that presumed a specific AWS account * enable caching of pip deps * fix missing code cov step * debug logging * add conditional * continue to fiddle with cache config * debug coverage fail * update action version * ongoing debugging * add coveragerc and set relative path option * include all coverage files * remove debugging code * add missing runner tag
chanzuckerberg · Jan 11, 2024 · 8484a16 · 8484a16
1 parent 03d873e
commit 8484a16
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 38 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+relative_files = True
diff --git a/.github/workflows/full-unittests.yml b/.github/workflows/full-unittests.yml
@@ -57,6 +57,10 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: |
+            api/**/pyproject.toml
+            api/**/requirements*.txt
 
       - name: install python dependencies (including experimental)
         run: |

diff --git a/.github/workflows/py-unittests.yml b/.github/workflows/py-unittests.yml
@@ -11,7 +11,7 @@ jobs:
   unit_tests_python_api:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [single-cell-8c64g-runner, macos-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     runs-on: ${{matrix.os}}
@@ -22,6 +22,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            api/**/pyproject.toml
+            api/**/requirements*.txt
       - name: Install dependencies (including experimental)
         run: |
           python -m pip install -U pip setuptools wheel
@@ -34,17 +38,16 @@ jobs:
       - name: Test with pytest (API, experimental)
         run: |
           PYTHONPATH=. coverage run --parallel-mode -m pytest -v -rP --durations=20 --experimental ./api/python/cellxgene_census/tests/experimental
-      - uses: actions/upload-artifact@v3
-        if: matrix.os == 'ubuntu-latest'
+      - uses: actions/upload-artifact@v4
         with:
-          name: coverage
+          name: coverage-api-${{ matrix.os }}-${{ matrix.python-version }}
           path: ./.coverage*
           retention-days: 3
 
   unit_tests_builder:
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [single-cell-8c64g-runner]
         python-version: ["3.11"]
 
     runs-on: ${{matrix.os}}
@@ -55,6 +58,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            tools/**/pyproject.toml
+            tools/**/requirements*.txt
       - name: Install dependencies
         run: |
           python -m pip install -U pip setuptools wheel
@@ -63,28 +70,30 @@ jobs:
       - name: Test with pytest (builder)
         run: |
           PYTHONPATH=. coverage run --parallel-mode -m pytest -v -rP ./tools/cellxgene_census_builder/tests/
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: coverage
+          name: coverage-builder-${{ matrix.os }}-${{ matrix.python-version }}
           path: ./.coverage*
           retention-days: 3
 
   submit-codecoverage:
     needs:
       - unit_tests_builder
       - unit_tests_python_api
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, Linux, X64]
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: coverage
+          pattern: coverage-*
+          merge-multiple: true
           path: .
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: "3.10"
+          cache: "pip"
       - name: install requirements
         run: |
           pip install coverage

diff --git a/tools/cellxgene_census_builder/tests/conftest.py b/tools/cellxgene_census_builder/tests/conftest.py
@@ -189,22 +189,6 @@ def setup(monkeypatch: MonkeyPatch, census_build_args: CensusBuildArgs) -> None:
     monkeypatch.setitem(CENSUS_X_LAYERS_PLATFORM_CONFIG["raw"]["tiledb"]["create"]["dims"]["soma_dim_1"], "tile", 2)
 
 
-def has_aws_credentials() -> bool:
-    """Return true if we have AWS credentials"""
-    import botocore
-
-    try:
-        session = botocore.session.get_session()
-        client = session.create_client("sts")
-        id = client.get_caller_identity()
-        print(id)
-        return True
-    except botocore.exceptions.BotoCoreError as e:
-        print(e)
-
-    return False
-
-
 @pytest.fixture
 def empty_blocklist(tmp_path: pathlib.Path) -> str:
     blocklist_path = tmp_path / "blocklist.txt"

diff --git a/tools/cellxgene_census_builder/tests/test_release_manifest.py b/tools/cellxgene_census_builder/tests/test_release_manifest.py
@@ -15,8 +15,6 @@
     validate_release_manifest,
 )
 
-from .conftest import has_aws_credentials
-
 # Should not be a real URL
 TEST_CENSUS_BASE_URL = "s3://bucket/path/"
 TEST_CENSUS_BASE_PREFIX = "/path/"
@@ -35,15 +33,6 @@ def test_get_release_manifest() -> None:
     validate_release_manifest(census_base_url, release_manifest, s3_anon=True)
 
 
-@pytest.mark.skipif(not has_aws_credentials(), reason="Unable to run without AWS credentials")
-def test_get_release_manifest_path() -> None:
-    with pytest.raises(OSError):
-        get_release_manifest("s3://no-such-bucket/or/path")
-
-    with pytest.raises(FileNotFoundError):
-        get_release_manifest("s3://cellxgene-data-public/no/such/base/path/")
-
-
 def soma_locator(tag: CensusVersionName) -> CensusLocator:
     return {
         "uri": f"{TEST_CENSUS_BASE_URL}{tag}/soma/",