diff --git a/.github/workflows/build_and_publish_dev.yaml b/.github/workflows/build_and_publish_dev.yaml index c0f4a94d..a309be29 100644 --- a/.github/workflows/build_and_publish_dev.yaml +++ b/.github/workflows/build_and_publish_dev.yaml @@ -25,6 +25,7 @@ jobs: java-version: graalvm@20.0.0 - name: Push Scala Dataflow Docker image run: sbt publish + # us.gcr.io/broad-dsp-gcr-public/hca-transformation-pipeline - name: Get artifact slug id: get-artifact-slug run: 'echo ::set-output name=slug::$(git rev-parse --short "$GITHUB_SHA")' diff --git a/.github/workflows/build_and_publish_main.yaml b/.github/workflows/build_and_publish_main.yaml index 4d3fc3f6..b6dc2265 100644 --- a/.github/workflows/build_and_publish_main.yaml +++ b/.github/workflows/build_and_publish_main.yaml @@ -29,6 +29,7 @@ jobs: run: gcloud auth configure-docker --quiet us.gcr.io,us-east4-docker.pkg.dev - name: Push Scala Dataflow Docker image run: sbt publish + # us.gcr.io/broad-dsp-gcr-public/hca-transformation-pipeline - name: Get artifact slug id: get-artifact-slug run: 'echo ::set-output name=slug::$(git rev-parse --short "$GITHUB_SHA")' diff --git a/.github/workflows/generate-requirements-file.yaml b/.github/workflows/generate-requirements-file.yaml index 02dd48a6..c47a1ff9 100644 --- a/.github/workflows/generate-requirements-file.yaml +++ b/.github/workflows/generate-requirements-file.yaml @@ -22,7 +22,7 @@ jobs: with: python-version: 3.9.16 - name: Install Poetry - uses: snok/install-poetry@v1.2 + uses: snok/install-poetry@v1 with: version: 1.1.9 virtualenvs-create: true diff --git a/.github/workflows/validate_pull_request_main.yaml b/.github/workflows/validate_pull_request_main.yaml index 416bf3e5..c0b23d11 100644 --- a/.github/workflows/validate_pull_request_main.yaml +++ b/.github/workflows/validate_pull_request_main.yaml @@ -23,15 +23,16 @@ jobs: with: python-version: 3.9.16 - name: Install Poetry - uses: snok/install-poetry@v1.2 + uses: snok/install-poetry@v1 with: - version: 1.1.9 + version: 1.8.0 - name: Restore cache dependencies uses: actions/cache@v2 env: cache-name: cache-poetry-v2 with: path: ~/.cache/pypoetry + # key uses pyproject.toml hash, so it's unique to each version of pyproject.toml key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./orchestration/pyproject.toml') }} restore-keys: | ${{ runner.os }}-build-${{ env.cache-name }}- diff --git a/.github/workflows/validate_python.yaml b/.github/workflows/validate_python.yaml index e1e2049f..ca1250a2 100644 --- a/.github/workflows/validate_python.yaml +++ b/.github/workflows/validate_python.yaml @@ -28,7 +28,7 @@ jobs: with: python-version: 3.9.16 - name: Install Poetry - uses: snok/install-poetry@v1.2 + uses: snok/install-poetry@v1 with: version: 1.1.9 - name: Cache dependencies diff --git a/orchestration/Dockerfile b/orchestration/Dockerfile index eb8dbe85..995e356b 100644 --- a/orchestration/Dockerfile +++ b/orchestration/Dockerfile @@ -9,7 +9,7 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_NO_CACHE_DIR=off \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 \ - POETRY_VERSION=1.1.8 \ + POETRY_VERSION=1.1.9 \ SENTRY_DSN=https://922d76ad2dceb6cd328436ae551ffb11@o54426.ingest.sentry.io/4506559533088768 RUN pip install "poetry==$POETRY_VERSION" diff --git a/orchestration/hca_manage/manifest.py b/orchestration/hca_manage/manifest.py index c866eadb..a1041a91 100644 --- a/orchestration/hca_manage/manifest.py +++ b/orchestration/hca_manage/manifest.py @@ -54,6 +54,7 @@ "dev": { "EBI": "gs://broad-dsp-monster-hca-dev-ebi-staging/dev", "UCSC": "gs://broad-dsp-monster-hca-dev-ebi-staging/dev", + "TEST": "gs://broad-dsp-monster-hca-prod-ebi-storage/broad_test_dataset" } } ENV_PIPELINE_ENDINGS = { @@ -101,7 +102,8 @@ def _parse_csv(csv_path: str, env: str, project_id_only: bool = False, continue assert len(row) == 2 - institution = row[0] + row = [x.strip() for x in row] + institution = row[0].upper() project_id = find_project_id_in_str(row[1]) key = None @@ -109,7 +111,6 @@ def _parse_csv(csv_path: str, env: str, project_id_only: bool = False, project_id = row[1] key = project_id else: - # TODO check for all caps - change to all caps if not, then match if institution not in STAGING_AREA_BUCKETS[env]: raise Exception(f"Unknown institution {institution} found") @@ -178,7 +179,6 @@ def _enumerate_manifests(env: str) -> None: def load(args: argparse.Namespace) -> None: - parse_and_load_manifest(args.env, args.csv_path, args.release_tag, "load_hca") parse_and_load_manifest(args.env, args.csv_path, args.release_tag, "per_project_load_hca") parse_and_load_manifest(args.env, args.csv_path, args.release_tag, "validate_ingress") parse_and_load_manifest( diff --git a/orchestration/poetry.lock b/orchestration/poetry.lock index 3901186d..8f4c2ad2 100644 --- a/orchestration/poetry.lock +++ b/orchestration/poetry.lock @@ -597,7 +597,7 @@ beautifulsoup4 = "*" [[package]] name = "google-api-core" -version = "2.19.0" +version = "2.23.0" description = "Google API client core library" category = "main" optional = false @@ -609,10 +609,11 @@ googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""} grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""} proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" [package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"] grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.49.1,<2.0.dev0)"] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] @@ -2057,7 +2058,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "3.9.16" -content-hash = "188d2c1e4fa7c8d434aa39e0854b49e1ff4a606096161d60ac1ecc9b43b04e95" +content-hash = "333564b623c9d14fd825bf3a3da5629544d6b7103d87a580f02357806c184a10" [metadata.files] aiohttp = [] diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml index 335af35a..b49cbe12 100644 --- a/orchestration/pyproject.toml +++ b/orchestration/pyproject.toml @@ -14,7 +14,7 @@ cffi = "1.16.0" # TODO: we'll probably want to use just the dagster version here and not the API versions as well # https://github.com/dagster-io/dagster/blob/master/MIGRATION.md#migrating-to-10 dagster = "0.12.14" -dagster-gcp = "^0.12.14" +dagster-gcp = "0.12.14" dagster-k8s = "0.12.14" dagster-postgres = "0.12.14" dagster-slack = "0.12.14" @@ -58,7 +58,7 @@ soft_delete = "hca_manage.soft_delete:run" job = "hca_manage.job:fetch_job_info" [build-system] -requires = ["poetry-core=^1.1.8"] +requires = ["poetry-core<=1.1.9"] build-backend = "poetry.core.masonry.api" [tool.autopep8] diff --git a/orchestration/requirements.txt b/orchestration/requirements.txt index 544c4e4c..ca718011 100644 --- a/orchestration/requirements.txt +++ b/orchestration/requirements.txt @@ -24,7 +24,7 @@ dagster==0.12.14 data-repo-client==1.542.0 docstring-parser==0.15; python_version >= "3.9" and python_version < "3.10" frozenlist==1.4.0; python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0" -google-api-core==2.19.0; python_version >= "3.9" and python_version < "3.10" and (python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7") +google-api-core==2.23.0; python_version >= "3.9" and python_version < "3.10" and (python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7") google-api-python-client==1.12.11; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" google-auth-httplib2==0.1.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" google-auth==2.23.3; python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10"