Merging main branch into staging branch

.happy/terraform/modules/ecs-stack/main.tf

@@ -112,6 +112,9 @@ module frontend_service {
   task_role_arn              = local.ecs_role_arn
   service_port               = 9000
   memory                     = var.frontend_memory
+
+  # 30gb of disk storage allocated for the task running the frontend container
+  task_storage_size_gb       = 30 
   cpu                        = 2048
   deployment_stage           = local.deployment_stage
   step_function_arn          = module.upload_sfn.step_function_arn
@@ -141,6 +144,9 @@ module backend_service {
   security_groups            = local.security_groups
   task_role_arn              = local.ecs_role_arn
   service_port               = 5000
+
+  # 100gb of disk storage allocated for the task running backend container 
+  task_storage_size_gb       = 100
   memory                     = var.backend_memory
   cpu                        = var.backend_cpus * 1024
   cmd                        = local.backend_cmd
@@ -154,6 +160,10 @@ module backend_service {
   dataset_submissions_bucket = local.dataset_submissions_bucket
   datasets_bucket            = local.datasets_bucket
   execution_role             = local.ecs_execution_role
+
+  # Bump health_check_interval from 15 seconds to 30 seconds so that WMG snapshot download,
+  # which at the time of this writing is around 27GB, has time to complete.
+  health_check_interval      = 30 
   dd_key_secret_arn          = var.dd_key_secret_arn
 
   wait_for_steady_state = local.wait_for_steady_state

.happy/terraform/modules/ecs-stack/variables.tf

@@ -35,6 +35,12 @@ variable happy_config_secret {
   description = "Happy Path configuration secret name"
 }
 
+variable "task_storage_size_gb" {
+  type        = number
+  description = "ephemeral disk storage in GB available for the task"
+  default     = 30 
+}
+
 variable deployment_stage {
   type        = string
   description = "Deployment stage for the app"

.happy/terraform/modules/service/main.tf

@@ -39,6 +39,9 @@ resource aws_ecs_task_definition task_definition {
   task_role_arn            = var.task_role_arn
   execution_role_arn       = var.execution_role
   requires_compatibilities = ["FARGATE"]
+  ephemeral_storage {
+    size_in_gib = var.task_storage_size_gb
+  }
   container_definitions = <<EOF
 [
   {

.happy/terraform/modules/service/variables.tf

@@ -138,6 +138,12 @@ variable "execution_role" {
   description = "Execution role to use for fargate tasks - required for fargate services!"
 }
 
+variable "task_storage_size_gb" {
+  type        = number
+  description = "ephemeral disk storage in GB available for the task"
+  default     = 30 
+}
+
 variable "health_check_interval" {
   type        = number
   description = "Interval for the health check pings"

Dockerfile.backend

@@ -17,6 +17,8 @@ COPY /python_dependencies/backend/ .
 # see ticket: https://github.com/chanzuckerberg/single-cell-data-portal/issues/5821
 RUN python3 -m pip install cmake
 RUN python3 -m pip install -r requirements.txt
+# Install awscli to download wmg snapshot to the local disk
+RUN python3 -m pip install awscli
 EXPOSE 5000
 
 # Install utilities to /single-cell-data-portal so we can run db migrations.

backend/wmg/api/v2.py

@@ -1,3 +1,4 @@
+import os
 from collections import defaultdict
 from typing import Any, Dict, Iterable, List
 
@@ -13,6 +14,8 @@
     READER_WMG_CUBE_QUERY_VALID_ATTRIBUTES,
     READER_WMG_CUBE_QUERY_VALID_DIMENSIONS,
     WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+    WMG_API_READ_FS_CACHED_SNAPSHOT,
+    WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH,
     WMG_API_SNAPSHOT_SCHEMA_VERSION,
 )
 from backend.wmg.data.ontology_labels import gene_term_label, ontology_term_label
@@ -26,7 +29,16 @@
 )
 from backend.wmg.data.schemas.cube_schema import expression_summary_non_indexed_dims
 from backend.wmg.data.snapshot import WmgSnapshot, load_snapshot
-from backend.wmg.data.utils import depluralize, find_all_dim_option_values, find_dim_option_values
+from backend.wmg.data.utils import (
+    depluralize,
+    find_all_dim_option_values,
+    find_dim_option_values,
+)
+
+DEPLOYMENT_STAGE = os.environ.get("DEPLOYMENT_STAGE", "")
+SNAPSHOT_FS_ROOT_PATH = (
+    WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH if (WMG_API_READ_FS_CACHED_SNAPSHOT and DEPLOYMENT_STAGE != "test") else None
+)
 
 
 # TODO: add cache directives: no-cache (i.e. revalidate); impl etag
@@ -42,6 +54,7 @@ def primary_filter_dimensions():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     return jsonify(snapshot.primary_filter_dimensions)
@@ -64,6 +77,7 @@ def query():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     with ServerTiming.time("query tiledb"):
@@ -149,6 +163,7 @@ def filters():
         snapshot: WmgSnapshot = load_snapshot(
             snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
             explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+            snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
         )
 
     with ServerTiming.time("calculate filters and build response"):
@@ -173,6 +188,7 @@ def markers():
     snapshot: WmgSnapshot = load_snapshot(
         snapshot_schema_version=WMG_API_SNAPSHOT_SCHEMA_VERSION,
         explicit_snapshot_id_to_load=WMG_API_FORCE_LOAD_SNAPSHOT_ID,
+        snapshot_fs_root_path=SNAPSHOT_FS_ROOT_PATH,
     )
 
     criteria = MarkerGeneQueryCriteria(

backend/wmg/api/wmg_api_config.py

@@ -1,3 +1,12 @@
+# When this config flag is set, the API will load the snapshot
+# from the local disk. When the flag is False, the API
+# will load the snapshot from S3
+WMG_API_READ_FS_CACHED_SNAPSHOT = True
+
+# Local filesystem root path where WMG snapshot
+# is cached
+WMG_API_SNAPSHOT_FS_CACHE_ROOT_PATH = "/single-cell-data-portal/wmg_snapshot_cache"
+
 # When the API is set to read a particular
 # snapshot schema version, it will load the
 # latest snapshot for the schema version by