From b9d29043265250a745e11d58539fb60bdfd41233 Mon Sep 17 00:00:00 2001 From: Faisal Alquaddoomi Date: Thu, 18 Apr 2024 13:57:27 -0600 Subject: [PATCH 1/5] Adds helper for deploying GCP funcs, workflows for convert-ids, ml funcs --- .../workflows/deploy-func-convert_ids.yaml | 22 ++++ .github/workflows/deploy-func-ml.yaml | 21 ++++ .github/workflows/helper-deploy-func.yaml | 115 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 .github/workflows/deploy-func-convert_ids.yaml create mode 100644 .github/workflows/deploy-func-ml.yaml create mode 100644 .github/workflows/helper-deploy-func.yaml diff --git a/.github/workflows/deploy-func-convert_ids.yaml b/.github/workflows/deploy-func-convert_ids.yaml new file mode 100644 index 0000000..2878347 --- /dev/null +++ b/.github/workflows/deploy-func-convert_ids.yaml @@ -0,0 +1,22 @@ +name: Deploy Cloud Functions +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'functions/convert_ids/convert_ids_deploy/**' + +jobs: + deploy-convert-ids-func: + uses: ./.github/workflows/helper-deploy-func.yaml + with: + func-name: "gpz-convert-ids" + func-src-dir: "functions/convert_ids/convert_ids_deploy" + func-entrypoint: "convert_ids" + func-memory-mb: 1024 + func-data-gcs-url: "gs://geneplexus-func-data/convert-ids/convert-ids_data.tar.gz" + permissions: + contents: 'read' + id-token: 'write' + secrets: inherit diff --git a/.github/workflows/deploy-func-ml.yaml b/.github/workflows/deploy-func-ml.yaml new file mode 100644 index 0000000..c6481d6 --- /dev/null +++ b/.github/workflows/deploy-func-ml.yaml @@ -0,0 +1,21 @@ +name: Deploy Cloud Functions +on: + push: + branches: + - main + paths: + - 'functions/ml/ml_deploy/**' + +jobs: + deploy-ml-func: + uses: ./.github/workflows/helper-deploy-func.yaml + with: + func-name: "gpz-ml" + func-src-dir: "functions/ml/ml_deploy" + func-entrypoint: "run_pipeline" + func-memory-mb: 8192 + func-data-gcs-url: "gs://geneplexus-func-data/ml/ml_data.tar.gz" + permissions: + contents: 'read' + id-token: 'write' + secrets: inherit diff --git a/.github/workflows/helper-deploy-func.yaml b/.github/workflows/helper-deploy-func.yaml new file mode 100644 index 0000000..b054dc5 --- /dev/null +++ b/.github/workflows/helper-deploy-func.yaml @@ -0,0 +1,115 @@ +name: Deploy Cloud Function + +on: + workflow_call: + inputs: + func-name: + required: true + type: string + func-src-dir: + required: true + type: string + description: Path in the repo containing this function's code + func-entrypoint: + required: true + type: string + description: Name of the function entrypoint in main.py + func-data-gcs-url: + required: true + type: string + description: GCS URL to the archive containing the data for this function + func-memory-mb: + required: true + type: number + + func-runtime: + default: python311 + type: string + func-svc-acct: + default: logging-monitoring@gap-som-dbmi-geneplx-app-p0n.iam.gserviceaccount.com + type: string + project-id: + default: gap-som-dbmi-geneplx-app-p0n + type: string + region: + default: us-central1 + type: string + func-data-local-path: + default: 'data' + type: string + description: 'Path under func-src-dir where the GCS archive is extracted, default ./data' + secrets: + JSON_GCLOUD_SERVICE_ACCOUNT_JSON: + required: true +jobs: + deploy-cloud-func-helper: + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + id-token: 'write' + + env: + DATA_ARCHIVE_HASH: 'none' + + steps: + - uses: 'actions/checkout@v4' + + - id: 'auth' + uses: 'google-github-actions/auth@v2' + with: + # workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' + # service_account: 'cloud-function-deployer@gap-som-dbmi-geneplx-app-p0n.iam.gserviceaccount.com' + credentials_json: ${{ secrets.JSON_GCLOUD_SERVICE_ACCOUNT_JSON }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + - name: Get hash of the data archive for this function + id: get-data-hash + run: | + gsutil ls -L ${{ inputs.func-data-gcs-url }} | \ + grep "Hash (crc32c)" | \ + awk '{printf "DATA_ARCHIVE_HASH=%s",$3}' >> "$GITHUB_ENV" + + # if the cached files aren't present, this will defer caching the files + # until the end of a successful run of this workflow. the next time the + # workflow runs, it'll retrieve the cached files from the previous run + # and skip the download step. + # the hash of the current datafile is included as part of the key, so + # that fresh data will be fetched if the datafile has changed in GCS. + - name: Cache existing data folder + id: cache-existing-data + uses: actions/cache@v4 + with: + path: ${{ inputs.func-src-dir }}/${{ inputs.func-data-local-path }} + key: ${{ inputs.func-name }}-data-${{ env.DATA_ARCHIVE_HASH }} + + - name: Download function data from GCS + id: download-data + if: steps.cache-existing-data.outputs.cache-hit != 'true' + run: | + gsutil cp ${{ inputs.func-data-gcs-url }} /tmp/data.tar.gz + mkdir -p ${{ inputs.func-src-dir }}/${{ inputs.func-data-local-path }} + tar -xvf /tmp/data.tar.gz -C ${{ inputs.func-src-dir }} + rm /tmp/data.tar.gz + + - name: Check filesystem status + id: check-fs + run: | + find ${{ inputs.func-src-dir }} + + - name: Deploy function '${{ inputs.func-name }}' to GCP + id: deploy + uses: 'google-github-actions/deploy-cloud-functions@v2' + with: + name: ${{ inputs.func-name }} + runtime: ${{ inputs.func-runtime }} + entry_point: ${{ inputs.func-entrypoint }} + memory_mb: ${{ inputs.func-memory-mb }} + source_dir: ${{ inputs.func-src-dir }} + ingress_settings: "ALLOW_ALL" + project_id: ${{ inputs.project-id }} + region: ${{ inputs.region }} + service_account_email: ${{ inputs.func-svc-acct }} From b4ac711e2779b5ef1713ab67125ae55b93119b8c Mon Sep 17 00:00:00 2001 From: Faisal Alquaddoomi Date: Tue, 23 Apr 2024 14:04:15 -0600 Subject: [PATCH 2/5] Addresses review comments re: permissions and changed entrypoint --- .github/workflows/deploy-func-convert_ids.yaml | 3 --- .github/workflows/deploy-func-ml.yaml | 5 +---- .github/workflows/helper-deploy-func.yaml | 4 ---- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/deploy-func-convert_ids.yaml b/.github/workflows/deploy-func-convert_ids.yaml index 2878347..57b48c6 100644 --- a/.github/workflows/deploy-func-convert_ids.yaml +++ b/.github/workflows/deploy-func-convert_ids.yaml @@ -16,7 +16,4 @@ jobs: func-entrypoint: "convert_ids" func-memory-mb: 1024 func-data-gcs-url: "gs://geneplexus-func-data/convert-ids/convert-ids_data.tar.gz" - permissions: - contents: 'read' - id-token: 'write' secrets: inherit diff --git a/.github/workflows/deploy-func-ml.yaml b/.github/workflows/deploy-func-ml.yaml index c6481d6..e6f5915 100644 --- a/.github/workflows/deploy-func-ml.yaml +++ b/.github/workflows/deploy-func-ml.yaml @@ -12,10 +12,7 @@ jobs: with: func-name: "gpz-ml" func-src-dir: "functions/ml/ml_deploy" - func-entrypoint: "run_pipeline" + func-entrypoint: "ml" func-memory-mb: 8192 func-data-gcs-url: "gs://geneplexus-func-data/ml/ml_data.tar.gz" - permissions: - contents: 'read' - id-token: 'write' secrets: inherit diff --git a/.github/workflows/helper-deploy-func.yaml b/.github/workflows/helper-deploy-func.yaml index b054dc5..68e289e 100644 --- a/.github/workflows/helper-deploy-func.yaml +++ b/.github/workflows/helper-deploy-func.yaml @@ -44,10 +44,6 @@ on: jobs: deploy-cloud-func-helper: runs-on: 'ubuntu-latest' - permissions: - contents: 'read' - id-token: 'write' - env: DATA_ARCHIVE_HASH: 'none' From e8512fa0eeb45541c6c2a84d8f7616299a535e18 Mon Sep 17 00:00:00 2001 From: Faisal Alquaddoomi Date: Tue, 23 Apr 2024 14:04:55 -0600 Subject: [PATCH 3/5] Switches from deploy-cloud-functions action to just running gcloud functions deploy, since the action doesn't support gen2 functions --- .github/workflows/helper-deploy-func.yaml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/helper-deploy-func.yaml b/.github/workflows/helper-deploy-func.yaml index 68e289e..a81bf4c 100644 --- a/.github/workflows/helper-deploy-func.yaml +++ b/.github/workflows/helper-deploy-func.yaml @@ -98,14 +98,15 @@ jobs: - name: Deploy function '${{ inputs.func-name }}' to GCP id: deploy - uses: 'google-github-actions/deploy-cloud-functions@v2' - with: - name: ${{ inputs.func-name }} - runtime: ${{ inputs.func-runtime }} - entry_point: ${{ inputs.func-entrypoint }} - memory_mb: ${{ inputs.func-memory-mb }} - source_dir: ${{ inputs.func-src-dir }} - ingress_settings: "ALLOW_ALL" - project_id: ${{ inputs.project-id }} - region: ${{ inputs.region }} - service_account_email: ${{ inputs.func-svc-acct }} + run: | + gcloud functions deploy ${{ inputs.func-name }} \ + --gen2 \ + --runtime=${{ inputs.func-runtime }} \ + --project=${{ inputs.project-id }} \ + --region=${{ inputs.region }} \ + --source=${{ inputs.func-src-dir }} \ + --entry-point=${{ inputs.func-entrypoint }} \ + --trigger-http \ + --allow-unauthenticated \ + --memory=${{ inputs.func-memory-mb }}MB \ + --service-account=${{ inputs.func-svc-acct }} From 06041bf6a5f928df473ddffde0a0a24937042006 Mon Sep 17 00:00:00 2001 From: Faisal Alquaddoomi Date: Tue, 23 Apr 2024 14:17:46 -0600 Subject: [PATCH 4/5] Adds tmate action before deploy to aid deployment debugging (thanks, Vince) --- .github/workflows/helper-deploy-func.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/helper-deploy-func.yaml b/.github/workflows/helper-deploy-func.yaml index a81bf4c..aa52acc 100644 --- a/.github/workflows/helper-deploy-func.yaml +++ b/.github/workflows/helper-deploy-func.yaml @@ -96,6 +96,9 @@ jobs: run: | find ${{ inputs.func-src-dir }} + - if: runner.debug == '1' + uses: mxschmitt/action-tmate@v3 + - name: Deploy function '${{ inputs.func-name }}' to GCP id: deploy run: | From 4c3137dd29cc0e88b383b4b0659b1e289fd06993 Mon Sep 17 00:00:00 2001 From: Faisal Alquaddoomi Date: Wed, 24 Apr 2024 11:05:43 -0600 Subject: [PATCH 5/5] Removed unnecessary ids, added missing step names --- .github/workflows/helper-deploy-func.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/helper-deploy-func.yaml b/.github/workflows/helper-deploy-func.yaml index aa52acc..1506cb6 100644 --- a/.github/workflows/helper-deploy-func.yaml +++ b/.github/workflows/helper-deploy-func.yaml @@ -50,20 +50,19 @@ jobs: steps: - uses: 'actions/checkout@v4' - - id: 'auth' + - name: Authenticate to GCP uses: 'google-github-actions/auth@v2' with: # workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' # service_account: 'cloud-function-deployer@gap-som-dbmi-geneplx-app-p0n.iam.gserviceaccount.com' credentials_json: ${{ secrets.JSON_GCLOUD_SERVICE_ACCOUNT_JSON }} - - name: 'Set up Cloud SDK' + - name: Set up Cloud SDK uses: 'google-github-actions/setup-gcloud@v2' with: version: '>= 363.0.0' - name: Get hash of the data archive for this function - id: get-data-hash run: | gsutil ls -L ${{ inputs.func-data-gcs-url }} | \ grep "Hash (crc32c)" | \ @@ -83,7 +82,6 @@ jobs: key: ${{ inputs.func-name }}-data-${{ env.DATA_ARCHIVE_HASH }} - name: Download function data from GCS - id: download-data if: steps.cache-existing-data.outputs.cache-hit != 'true' run: | gsutil cp ${{ inputs.func-data-gcs-url }} /tmp/data.tar.gz @@ -92,15 +90,13 @@ jobs: rm /tmp/data.tar.gz - name: Check filesystem status - id: check-fs run: | find ${{ inputs.func-src-dir }} - if: runner.debug == '1' uses: mxschmitt/action-tmate@v3 - + - name: Deploy function '${{ inputs.func-name }}' to GCP - id: deploy run: | gcloud functions deploy ${{ inputs.func-name }} \ --gen2 \