Merging staging branch into prod branch

.github/workflows/build-images-and-create-deployment.yml

@@ -24,7 +24,6 @@ jobs:
   build_images:
     uses: ./.github/workflows/build-images.yml
     secrets: inherit
-    needs: get_previous_image_digests
 
   create_deployment:
     needs:

.github/workflows/build-images.yml

@@ -55,7 +55,7 @@ jobs:
         run: |
           echo "HAPPY_COMMIT=$(git rev-parse --verify HEAD)" >> envfile
           echo "HAPPY_BRANCH=$(git branch --show-current)" >> envfile
-          export IMAGE_TAG=sha-${GITHUB_SHA:0:7}
+          export IMAGE_TAG=sha-${GITHUB_SHA:0:8}
           export BRANCH_TAG=branch-$(echo ${GITHUB_REF_NAME} | sed 's/[\+\/]/-/g')
           echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT
           happy push devstack --env dev --slice ${{ matrix.image }} \

.github/workflows/push-rdev.yml

@@ -26,7 +26,6 @@ jobs:
   build_images:
     uses: ./.github/workflows/build-images.yml
     secrets: inherit
-    needs: get_previous_image_digests
 
   summarize:
     runs-on: ubuntu-22.04

.github/workflows/rebuild-processing-image.yml

@@ -16,9 +16,13 @@ permissions:
 on:
   repository_dispatch:
     types: [rebuild-processing]
+  workflow_call:
 
 jobs:
-  rebuild-and-push-processing-image-dev:
+  rebuild-and-push-processing-image:
+    strategy:
+      matrix:
+        branch: [main, staging, prod]
     runs-on: ubuntu-22.04
     steps:
       - name: Configure AWS Credentials
@@ -31,66 +35,10 @@ jobs:
         uses: docker/login-action@v3
         with:
           registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Dev (main)
+      - name: Checkout ${{ matrix.branch }}
         uses: actions/checkout@v2
         with:
-          ref: main
-          fetch-depth: 1
-      - name: Install happy
-        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
-        with:
-          happy_version: "0.110.1"
-      - name: Docker build, push, and tag
-        shell: bash
-        run: |
-          export BRANCH_SHA=$(git rev-parse --short=8 HEAD)
-          happy push "" --aws-profile "" --tag sha-${BRANCH_SHA} --slice processing
-
-  rebuild-and-push-processing-image-staging:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: us-west-2
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 1800
-      - name: Login to ECR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Staging
-        uses: actions/checkout@v2
-        with:
-          ref: staging
-          fetch-depth: 1
-      - name: Install happy
-        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
-        with:
-          happy_version: "0.110.1"
-      - name: Docker build, push, and tag
-        shell: bash
-        run: |
-          export BRANCH_SHA=$(git rev-parse --short=8 HEAD)
-          happy push "" --aws-profile "" --tag sha-${BRANCH_SHA} --slice processing
-
-  rebuild-and-push-processing-image-prod:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: us-west-2
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 1800
-      - name: Login to ECR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Prod
-        uses: actions/checkout@v2
-        with:
-          ref: prod
+          ref: ${{ matrix.branch }}
           fetch-depth: 1
       - name: Install happy
         uses: chanzuckerberg/github-actions/.github/actions/[email protected]
@@ -108,4 +56,4 @@ jobs:
           fields: repo,eventName,workflow,job
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
-        if: always() # Pick up events even if the job fails or is canceled.
+        if: matrix.branch == 'prod' && always() # Pick up events even if the job fails or is canceled.

.github/workflows/schema-migration.yml

@@ -18,101 +18,16 @@ on:
     types: [schema-migration]
 
 jobs:
-  rebuild-and-push-processing-image-dev:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: us-west-2
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 1800
-      - name: Login to ECR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Dev (main)
-        uses: actions/checkout@v2
-        with:
-          ref: main
-          fetch-depth: 1
-      - name: Install happy
-        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
-        with:
-          happy_version: "0.110.1"
-      - name: Docker build, push, and tag
-        shell: bash
-        run: |
-          export BRANCH_SHA=$(git rev-parse --short=8 HEAD)
-          happy push "" --aws-profile "" --tag sha-${BRANCH_SHA} --slice processing
-
-  rebuild-and-push-processing-image-staging:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: us-west-2
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 1800
-      - name: Login to ECR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Staging
-        uses: actions/checkout@v2
-        with:
-          ref: staging
-          fetch-depth: 1
-      - name: Install happy
-        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
-        with:
-          happy_version: "0.110.1"
-      - name: Docker build, push, and tag
-        shell: bash
-        run: |
-          export BRANCH_SHA=$(git rev-parse --short=8 HEAD)
-          happy push "" --aws-profile "" --tag sha-${BRANCH_SHA} --slice processing
-
-  rebuild-and-push-processing-image-prod:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: us-west-2
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 1800
-      - name: Login to ECR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.ECR_REPO }}
-      - name: Checkout Prod
-        uses: actions/checkout@v2
-        with:
-          ref: prod
-          fetch-depth: 1
-      - name: Install happy
-        uses: chanzuckerberg/github-actions/.github/actions/[email protected]
-        with:
-          happy_version: "0.110.1"
-      - name: Docker build, push, and tag
-        shell: bash
-        run: |
-          export BRANCH_SHA=$(git rev-parse --short=8 HEAD)
-          happy push "" --aws-profile "" --tag sha-${BRANCH_SHA} --slice processing
-      - name: Alert in Slack
-        uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,eventName,workflow,job
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
-        if: always() # Pick up events even if the job fails or is canceled.
+  rebuild-processing-images:
+    uses: ./.github/actions/rebuild-processing-images.yml
+    secrets:
+      ECR_REPO: ${{ secrets.ECR_REPO }}
+      AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
 
   trigger-schema-migration:
     runs-on: ubuntu-22.04
-    needs: rebuild-and-push-processing-image-prod
+    needs: rebuild-processing-images
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4

.happy/terraform/modules/ecs-stack/main.tf

@@ -112,6 +112,9 @@ module frontend_service {
   task_role_arn              = local.ecs_role_arn
   service_port               = 9000
   memory                     = var.frontend_memory
+
+  # 30gb of disk storage allocated for the task running the frontend container
+  task_storage_size_gb       = 30 
   cpu                        = 2048
   deployment_stage           = local.deployment_stage
   step_function_arn          = module.upload_sfn.step_function_arn
@@ -141,6 +144,9 @@ module backend_service {
   security_groups            = local.security_groups
   task_role_arn              = local.ecs_role_arn
   service_port               = 5000
+
+  # 100gb of disk storage allocated for the task running backend container 
+  task_storage_size_gb       = 100
   memory                     = var.backend_memory
   cpu                        = var.backend_cpus * 1024
   cmd                        = local.backend_cmd
@@ -154,6 +160,10 @@ module backend_service {
   dataset_submissions_bucket = local.dataset_submissions_bucket
   datasets_bucket            = local.datasets_bucket
   execution_role             = local.ecs_execution_role
+
+  # Bump health_check_interval from 15 seconds to 30 seconds so that WMG snapshot download,
+  # which at the time of this writing is around 27GB, has time to complete.
+  health_check_interval      = 30 
   dd_key_secret_arn          = var.dd_key_secret_arn
 
   wait_for_steady_state = local.wait_for_steady_state

.happy/terraform/modules/ecs-stack/variables.tf

@@ -35,6 +35,12 @@ variable happy_config_secret {
   description = "Happy Path configuration secret name"
 }
 
+variable "task_storage_size_gb" {
+  type        = number
+  description = "ephemeral disk storage in GB available for the task"
+  default     = 30 
+}
+
 variable deployment_stage {
   type        = string
   description = "Deployment stage for the app"

.happy/terraform/modules/service/main.tf

@@ -39,6 +39,9 @@ resource aws_ecs_task_definition task_definition {
   task_role_arn            = var.task_role_arn
   execution_role_arn       = var.execution_role
   requires_compatibilities = ["FARGATE"]
+  ephemeral_storage {
+    size_in_gib = var.task_storage_size_gb
+  }
   container_definitions = <<EOF
 [
   {

.happy/terraform/modules/service/variables.tf

@@ -138,6 +138,12 @@ variable "execution_role" {
   description = "Execution role to use for fargate tasks - required for fargate services!"
 }
 
+variable "task_storage_size_gb" {
+  type        = number
+  description = "ephemeral disk storage in GB available for the task"
+  default     = 30 
+}
+
 variable "health_check_interval" {
   type        = number
   description = "Interval for the health check pings"

Dockerfile.backend

@@ -17,6 +17,8 @@ COPY /python_dependencies/backend/ .
 # see ticket: https://github.com/chanzuckerberg/single-cell-data-portal/issues/5821
 RUN python3 -m pip install cmake
 RUN python3 -m pip install -r requirements.txt
+# Install awscli to download wmg snapshot to the local disk
+RUN python3 -m pip install awscli
 EXPOSE 5000
 
 # Install utilities to /single-cell-data-portal so we can run db migrations.

backend/wmg/api/v2.py

@@ -1,3 +1,4 @@
+import os
 from collections import defaultdict
 from typing import Any, Dict, Iterable, List
 
@@ -13,6 +14,8 @@
     READER_WMG_CUBE_QUERY_VALID_ATTRIBUTES,
     READER_WMG_CUBE_QUERY_VALID_DIMENSIONS,
     WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+    WMG_API_READ_FS_CACHED_SNAPSHOT,
+    WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH,
     WMG_API_SNAPSHOT_SCHEMA_VERSION,
 )
 from backend.wmg.data.ontology_labels import gene_term_label, ontology_term_label
@@ -26,7 +29,16 @@
 )
 from backend.wmg.data.schemas.cube_schema import expression_summary_non_indexed_dims
 from backend.wmg.data.snapshot import WmgSnapshot, load_snapshot
-from backend.wmg.data.utils import depluralize, find_all_dim_option_values, find_dim_option_values
+from backend.wmg.data.utils import (
+    depluralize,
+    find_all_dim_option_values,
+    find_dim_option_values,
+)
+
+DEPLOYMENT_STAGE = os.environ.get("DEPLOYMENT_STAGE", "")
+SNAPSHOT_FS_ROOT_PATH = (
+    WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH if (WMG_API_READ_FS_CACHED_SNAPSHOT and DEPLOYMENT_STAGE != "test") else None
+)
 
 
 # TODO: add cache directives: no-cache (i.e. revalidate); impl etag
@@ -42,6 +54,7 @@ def primary_filter_dimensions():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     return jsonify(snapshot.primary_filter_dimensions)
@@ -64,6 +77,7 @@ def query():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     with ServerTiming.time("query tiledb"):
@@ -149,6 +163,7 @@ def filters():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     with ServerTiming.time("calculate filters and build response"):
@@ -173,6 +188,7 @@ def markers():
     snapshot: WmgSnapshot = load_snapshot(
         snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
         explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+        snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
     )
 
     criteria = MarkerGeneQueryCriteria(

backend/wmg/api/wmg_api_config.py

@@ -1,3 +1,12 @@
+# When this config flag is set, the API will load the snapshot
+# from the local disk. When the flag is False, the API
+# will load the snapshot from S3
+WMG_API_READ_FS_CACHED_SNAPSHOT = True
+
+# Local filesystem root path where WMG snapshot
+# is cached
+WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH = "/single-cell-data-portal/wmg_snapshot_cache"
+
 # When the API is set to read a particular
 # snapshot schema version, it will load the
 # latest snapshot for the schema version by