Merge pull request #5849 from chanzuckerberg/staging

chore: Deploy to Prod

.github/workflows/build-images-and-create-deployment.yml

@@ -8,8 +8,6 @@ on:
       - prod
 
 env:
-  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-  DEPLOYMENT_STAGE: test
   # Force using BuildKit instead of normal Docker, required so that metadata
   # is written/read to allow us to use layers of previous builds as cache.
   DOCKER_BUILDKIT: 1

.github/workflows/deploy-happy-stack.yml

@@ -211,30 +211,8 @@ jobs:
       matrix:
         # Only run Chrome for now, since GHA only has 250 workers and will cancel jobs if it runs out
         project: [chromium]
-        shardCurrent:
-          [
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12,
-            13,
-            14,
-            15,
-            16,
-            17,
-            18,
-            19,
-            20,
-          ]
-        shardTotal: [20]
+        shardCurrent: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        shardTotal: [10]
 
     defaults:
       run:
@@ -291,15 +269,6 @@ jobs:
           path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/frontend/blob-report
           retention-days: 1
 
-      - uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,commit,author,eventName,workflow,job,mention
-          mention: "here"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
-        if: failure() && github.ref == 'refs/heads/main'
-
   wait-for-shards:
     needs: run-e2e-tests # run after shards
     runs-on: ubuntu-22.04
@@ -344,6 +313,7 @@ jobs:
 
   functional-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     needs:
       - upgrade
     steps:
@@ -361,37 +331,25 @@ jobs:
           aws-region: us-west-2
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           role-duration-seconds: 2700
-      # Login to ECR needed solely to be able to retrieve cached layers
-      - name: Login to Prod ECR
-        uses: docker/login-action@v1
-        if: github.event.deployment.environment == 'prod'
-        with:
-          registry: ${{ secrets.ECR_REPO_PROD }}
-      - name: Login to ECR
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ secrets.ECR_REPO }}
       - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.deployment.sha }}
           fetch-depth: 1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
       - name: Run functional test
         env:
           TFE_TOKEN: ${{ secrets.TFE_TOKEN }}
           DEPLOYMENT_STAGE: ${{ github.event.deployment.environment }}
         if: github.event.deployment.environment != 'prod'
         run: |
-          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
-          mkdir -p .local/bin
-          curl -Ls https://github.com/segmentio/chamber/releases/download/v2.9.1/chamber-v2.9.1-linux-amd64 > .local/bin/chamber &&
-          chmod +x .local/bin/chamber
-          PATH="$PATH:$(pwd)/.local/bin"
           if [ "${DEPLOYMENT_STAGE}" == "stage" ]; then
             export DEPLOYMENT_STAGE=staging
           fi
           echo DEPLOYMENT_STAGE ${DEPLOYMENT_STAGE}
-          docker-compose up --no-deps -d backend
-          BOTO_ENDPOINT_URL= make local-functional-test
+          pip3 install -r tests/functional/requirements.txt
+          make functional-test
       ### Need to write success failure way because Github API doesn't allow doing
       ### "if: always(), state: ${{ success() }}:
       - name: Set deployment status to success if no errors
@@ -410,6 +368,7 @@ jobs:
 
   performance-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     needs:
       - upgrade
     steps:

.github/workflows/lint-pr.yml

@@ -1,4 +1,4 @@
-name: "Lint PR commit"
+name: "Lint PR Title"
 
 on:
   pull_request_target:
@@ -7,12 +7,13 @@ on:
       - edited
       - synchronize
 
+permissions:
+  pull-requests: read
+
 jobs:
   main:
     runs-on: ubuntu-latest
     steps:
       - uses: amannn/[email protected]
-        with:
-          validateSingleCommit: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push-rdev.yml

@@ -24,14 +24,24 @@ permissions:
 
 jobs:
   build-push-images:
+    strategy:
+      matrix:
+        image:
+          - frontend # pushed both the frontend and backend images
+          - upload_failures
+          - upload_success
+          - dataset_submissions
+          - processing
+          - wmg_processing
+          - cellguide_pipeline
     runs-on: ubuntu-22.04
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-region: us-west-2
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 900
+          role-duration-seconds: 1800
       - name: Login to ECR
         uses: docker/login-action@v1
         with:
@@ -42,13 +52,25 @@ jobs:
       - name: Install happy
         uses: chanzuckerberg/github-actions/.github/actions/[email protected]
         with:
-          happy_version: "0.59.0"
-      - name: Build component
-        shell: bash
+          happy_version: "0.92.0"
+      - name: Push images
         run: |
-          happy push --aws-profile "" --tag sha-${GITHUB_SHA:0:7}
+          echo "HAPPY_COMMIT=$(git rev-parse --verify HEAD)" >> envfile
+          echo "HAPPY_BRANCH=$(git branch --show-current)" >> envfile
+          export IMAGE_TAG=sha-${GITHUB_SHA:0:8}
+          happy push devstack --env dev --slice ${{ matrix.image }} \
+          --docker-compose-env-file envfile --aws-profile "" \
+          --tags ${IMAGE_TAG},${STACK_NAME},branch-$(echo ${GITHUB_REF_NAME} | sed 's/[\+\/]/-/g')
+
+  summarize:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
       - name: Create Summary With Happy Commands
         run: |
+          export 
           echo "### Happy Commands :rocket: :upside_down_face:" >> $GITHUB_STEP_SUMMARY
-          echo "* \`happy create <_stack-name_> --tag ${GITHUB_SHA:0:7} --create-tag=false --skip-check-tag\`" >> $GITHUB_STEP_SUMMARY
-          echo "* \`happy update <_stack-name_> --tag ${GITHUB_SHA:0:7} --create-tag=false --skip-check-tag\`" >> $GITHUB_STEP_SUMMARY
+          echo "* \`happy create <_stack-name_> --tag ${GITHUB_SHA:0:8} --create-tag=false --skip-check-tag\`" >> $GITHUB_STEP_SUMMARY
+          echo "* \`happy update <_stack-name_> --tag ${GITHUB_SHA:0:8} --create-tag=false --skip-check-tag\`" >> $GITHUB_STEP_SUMMARY

.github/workflows/push-tests.yml

@@ -63,30 +63,8 @@ jobs:
       matrix:
         # Only run Chrome for now, since GHA only has 250 workers and will cancel jobs if it runs out
         project: [chromium]
-        shardCurrent:
-          [
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12,
-            13,
-            14,
-            15,
-            16,
-            17,
-            18,
-            19,
-            20,
-          ]
-        shardTotal: [20]
+        shardCurrent: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        shardTotal: [10]
 
     defaults:
       run:
@@ -233,6 +211,7 @@ jobs:
 
   backend-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -286,6 +265,7 @@ jobs:
 
   backend-wmg-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -339,6 +319,7 @@ jobs:
 
   processing-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -397,6 +378,7 @@ jobs:
 
   wmg-processing-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -451,6 +433,7 @@ jobs:
         if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
   cellguide-pipeline-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -506,6 +489,7 @@ jobs:
 
   cxg-admin-unit-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -559,6 +543,7 @@ jobs:
 
   backend-integration-test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2

.github/workflows/rdev-delete-for-pr.yml

@@ -0,0 +1,41 @@
+name: Delete rdev for PR
+
+on:
+  pull_request:
+    branches-ignore:
+      - prod
+    types:
+      - closed
+
+env:
+  # Force using BuildKit instead of normal Docker, required so that metadata
+  # is written/read to allow us to use layers of previous builds as cache.
+  DOCKER_BUILDKIT: 1
+  COMPOSE_DOCKER_CLI_BUILD: 1
+  DOCKER_REPO: ${{ secrets.ECR_REPO }}/
+  STACK_NAME: pr-${{ github.event.number }}
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  delete-rdev:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 900
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Delete deployment
+        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
+        with:
+          tfe-token: ${{ secrets.TFE_TOKEN }}
+          stack-name: ${{ env.STACK_NAME }}
+          operation: delete
+          happy_version: "0.92.0"