Dataset download zip email

gateway/.envs/example/django.prod-example.env

@@ -7,6 +7,8 @@ DJANGO_SETTINGS_MODULE=config.settings.production
 DJANGO_SECRET_KEY=
 DJANGO_ADMIN_URL=
 DJANGO_ALLOWED_HOSTS=localhost,sds.crc.nd.edu
+SITE_DOMAIN=sds.crc.nd.edu
+USE_HTTPS=True
 
 # SECURITY
 # ------------------------------------------------------------------------------
@@ -20,12 +22,10 @@ AUTH0_DOMAIN=https://dev-XXXXXXXXXX.us.auth0.com
 CLIENT_ID=
 CLIENT_SECRET=
 
-# EMAIL
+# ND EMAIL
 # ------------------------------------------------------------------------------
-DJANGO_SERVER_EMAIL=
-
-MAILGUN_API_KEY=
-MAILGUN_DOMAIN=
+DJANGO_EMAIL_HOST= # institution whitelisted server email host
+DJANGO_EMAIL_PORT=25
 
 
 # DJANGO-ALLAUTH

gateway/compose.local.yaml

@@ -6,7 +6,8 @@
 
 volumes:
     # for safety, all local volumes start with "sds-gateway-local-"
-    # sds-gateway-prod-app-media: {}    # used in prod only
+    sds-gateway-local-app-media: {}  # for Django media files
+    sds-gateway-local-temp-zips: {}  # for temporary zip files
     sds-gateway-local-minio-files: {}
     sds-gateway-local-opensearch-data: {}
     sds-gateway-local-postgres-data-backups: {}
@@ -42,6 +43,8 @@ services:
                 condition: service_started
         volumes:
             - .:/app:z  # bind mount used in local for hot reloading; prod uses built assets
+            - sds-gateway-local-app-media:/app/sds_gateway/media  # persistent media storage
+            - sds-gateway-local-temp-zips:/app/sds_gateway/media/temp_zips  # temporary zip files
             # - ./staticfiles/:/app/staticfiles/:z  # used in prod only
         env_file:
             - ./.envs/local/django.env
@@ -143,6 +146,69 @@ services:
         container_name: sds-gateway-local-redis
         volumes:
             - sds-gateway-local-redis-data:/data
+        networks:
+            - sds-network-local
+
+    # ==========================
+    # Celery services for background tasks
+    celery-worker:
+        build:
+            context: .
+            dockerfile: ./compose/local/django/Dockerfile
+        image: sds-gateway-local-app
+        container_name: sds-gateway-local-celery-worker
+        depends_on:
+            redis:
+                condition: service_started
+            postgres:
+                condition: service_started
+            minio:
+                condition: service_started
+        volumes:
+            - .:/app:z
+            - sds-gateway-local-app-media:/app/sds_gateway/media  # persistent media storage
+            - sds-gateway-local-temp-zips:/app/sds_gateway/media/temp_zips  # temporary zip files
+        env_file:
+            - ./.envs/local/django.env
+            - ./.envs/local/minio.env
+            - ./.envs/local/postgres.env
+            - ./.envs/local/opensearch.env
+        command: celery -A config.celery_app worker -l INFO
+        networks:
+            - sds-gateway-local-minio-net
+            - sds-gateway-local-opensearch-net
+            - sds-network-local
+
+    celery-flower:
+        # Celery monitoring and administration tool
+        build:
+            context: .
+            dockerfile: ./compose/local/django/Dockerfile
+        image: sds-gateway-local-app
+        container_name: sds-gateway-local-celery-flower
+        depends_on:
+            redis:
+                condition: service_started
+            postgres:
+                condition: service_started
+            minio:
+                condition: service_started
+        volumes:
+            - .:/app:z
+            - sds-gateway-local-app-media:/app/sds_gateway/media  # persistent media storage
+            - sds-gateway-local-temp-zips:/app/sds_gateway/media/temp_zips  # temporary zip files
+        env_file:
+            - ./.envs/local/django.env
+            - ./.envs/local/minio.env
+            - ./.envs/local/postgres.env
+            - ./.envs/local/opensearch.env
+        command: /start
+        ports:
+            - "5555:5555"  # Flower web interface
+        networks:
+            - sds-gateway-local-minio-net
+            - sds-gateway-local-opensearch-net
+            - sds-network-local
 
     # ==========================
     # local development services
@@ -162,3 +228,13 @@ services:
         command: npm run dev
         ports:
             - "3000:3000"
+
+    mailhog:
+        # email testing service for local development
+        image: mailhog/mailhog:latest
+        container_name: sds-gateway-local-mailhog
+        ports:
+            - "1025:1025"  # SMTP server
+            - "8025:8025"  # Web UI
+        networks:
+            - sds-network-local

gateway/compose.production.yaml

@@ -7,6 +7,7 @@
 volumes:
     # for safety, all gateway production volumes start with "sds-gateway-prod-"
     sds-gateway-prod-app-media: {}
+    sds-gateway-prod-temp-zips: {}  # for temporary zip files
     sds-gateway-prod-minio-files: {}
     sds-gateway-prod-opensearch-data: {}
     sds-gateway-prod-postgres-data-backups: {}
@@ -47,6 +48,12 @@ services:
               read_only: false
               bind:
                 selinux: z
+            - source: sds-gateway-prod-temp-zips
+              target: /app/sds_gateway/media/temp_zips
+              type: volume
+              read_only: false
+              bind:
+                selinux: z
             - source: sds-gateway-prod-static
               target: /app/staticfiles/
               type: volume
@@ -158,6 +165,95 @@ services:
         container_name: sds-gateway-prod-redis
         volumes:
             - sds-gateway-prod-redis-data:/data
+        networks:
+            - sds-network-prod
+
+    # ===================
+    # Celery services for background tasks
+    celery-worker:
+        build:
+            context: .
+            dockerfile: ./compose/production/django/Dockerfile
+        image: sds-gateway-prod-app
+        container_name: sds-gateway-prod-celery-worker
+        depends_on:
+            redis:
+                condition: service_started
+            postgres:
+                condition: service_started
+            minio:
+                condition: service_started
+        volumes:
+            - source: sds-gateway-prod-app-media
+              target: /app/sds_gateway/media
+              type: volume
+              read_only: false
+              bind:
+                selinux: z
+            - source: sds-gateway-prod-temp-zips
+              target: /app/sds_gateway/media/temp_zips
+              type: volume
+              read_only: false
+              bind:
+                selinux: z
+            - source: ./opensearch/data/certs-django
+              target: /app/opensearch/data/certs
+              type: bind
+              read_only: true
+        env_file:
+            - ./.envs/production/django.env
+            - ./.envs/production/minio.env
+            - ./.envs/production/postgres.env
+            - ./.envs/production/opensearch.env
+        command: celery -A config.celery_app worker -l INFO
+        networks:
+            - sds-gateway-prod-opensearch-net
+            - sds-gateway-prod-minio-net
+            - sds-network-prod
+
+    celery-flower:
+        # Celery monitoring and administration tool
+        build:
+            context: .
+            dockerfile: ./compose/production/django/Dockerfile
+        image: sds-gateway-prod-app
+        container_name: sds-gateway-prod-celery-flower
+        depends_on:
+            redis:
+                condition: service_started
+            postgres:
+                condition: service_started
+            minio:
+                condition: service_started
+        volumes:
+            - source: sds-gateway-prod-app-media
+              target: /app/sds_gateway/media
+              type: volume
+              read_only: false
+              bind:
+                selinux: z
+            - source: sds-gateway-prod-temp-zips
+              target: /app/sds_gateway/media/temp_zips
+              type: volume
+              read_only: false
+              bind:
+                selinux: z
+            - source: ./opensearch/data/certs-django
+              target: /app/opensearch/data/certs
+              type: bind
+              read_only: true
+        env_file:
+            - ./.envs/production/django.env
+            - ./.envs/production/minio.env
+            - ./.envs/production/postgres.env
+            - ./.envs/production/opensearch.env
+        command: /start
+        ports:
+            - "15555:5555"  # Flower web interface
+        networks:
+            - sds-gateway-prod-opensearch-net
+            - sds-gateway-prod-minio-net
+            - sds-network-prod
 
     # ===================
     # production services

gateway/config/settings/base.py

@@ -22,6 +22,9 @@ def __get_random_token(length: int) -> str:
 
 env.read_env()
 
+SITE_DOMAIN: str = env.str("SITE_DOMAIN", default="localhost:8000")
+USE_HTTPS: bool = env.bool("USE_HTTPS", default=False)
+
 BASE_DIR: Path = Path(__file__).resolve(strict=True).parent.parent.parent
 
 API_VERSION: str = env.str("API_VERSION", default="v1")
@@ -285,6 +288,13 @@ def __get_random_token(length: int) -> str:
     "DJANGO_EMAIL_BACKEND",
     default="django.core.mail.backends.smtp.EmailBackend",
 )
+EMAIL_HOST: str = env("DJANGO_EMAIL_HOST", default="mailhog")
+EMAIL_PORT: int = env("DJANGO_EMAIL_PORT", default=1025)
+EMAIL_HOST_USER: str = env("DJANGO_EMAIL_HOST_USER", default="")
+EMAIL_HOST_PASSWORD: str = env("DJANGO_EMAIL_HOST_PASSWORD", default="")
+EMAIL_USE_TLS: bool = env("DJANGO_EMAIL_USE_TLS", default=False)
+EMAIL_USE_SSL: bool = env("DJANGO_EMAIL_USE_SSL", default=False)
+
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-timeout
 EMAIL_TIMEOUT: int = 5
 
@@ -330,37 +340,35 @@ def __get_random_token(length: int) -> str:
 
 # CELERY
 # ------------------------------------------------------------------------------
-# if USE_TZ:
-#     # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone
-#     CELERY_TIMEZONE: str = TIME_ZONE
-#
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url
-# CELERY_BROKER_URL: str = env("CELERY_BROKER_URL")
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend
-# CELERY_RESULT_BACKEND: str = CELERY_BROKER_URL
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-extended
-# CELERY_RESULT_EXTENDED: bool = True
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-always-retry
-# # https://github.com/celery/celery/pull/6122
-# CELERY_RESULT_BACKEND_ALWAYS_RETRY: bool = True
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-max-retries
-# CELERY_RESULT_BACKEND_MAX_RETRIES: int = 10
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-accept_content
-# CELERY_ACCEPT_CONTENT: list[str] = ["json"]
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-task_serializer
-# CELERY_TASK_SERIALIZER: str = "json"
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_serializer
-# CELERY_RESULT_SERIALIZER: str = "json"
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-time-limit
-# CELERY_TASK_TIME_LIMIT: int = 5 * 60
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-soft-time-limit
-# CELERY_TASK_SOFT_TIME_LIMIT: int = 60
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-scheduler
-# CELERY_BEAT_SCHEDULER: str = "django_celery_beat.schedulers:DatabaseScheduler"
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#worker-send-task-events
-# CELERY_WORKER_SEND_TASK_EVENTS: bool = True
-# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_send_sent_event
-# CELERY_TASK_SEND_SENT_EVENT: bool = True
+if USE_TZ:
+    # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone
+    CELERY_TIMEZONE: str = TIME_ZONE
+
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url
+CELERY_BROKER_URL: str = env("CELERY_BROKER_URL", default="redis://redis:6379/0")
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend
+CELERY_RESULT_BACKEND: str = CELERY_BROKER_URL
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-extended
+CELERY_RESULT_EXTENDED: bool = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-always-retry
+# https://github.com/celery/celery/pull/6122
+CELERY_RESULT_BACKEND_ALWAYS_RETRY: bool = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-max-retries
+CELERY_RESULT_BACKEND_MAX_RETRIES: int = 10
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-accept_content
+CELERY_ACCEPT_CONTENT: list[str] = ["json"]
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-task_serializer
+CELERY_TASK_SERIALIZER: str = "json"
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_serializer
+CELERY_RESULT_SERIALIZER: str = "json"
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-time-limit
+CELERY_TASK_TIME_LIMIT: int = 5 * 60
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-soft-time-limit
+CELERY_TASK_SOFT_TIME_LIMIT: int = 60
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#worker-send-task-events
+CELERY_WORKER_SEND_TASK_EVENTS: bool = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_send_sent_event
+CELERY_TASK_SEND_SENT_EVENT: bool = True
 
 # django-allauth
 # ------------------------------------------------------------------------------

gateway/config/settings/local.py

@@ -43,11 +43,9 @@
 
 # EMAIL
 # ------------------------------------------------------------------------------
-# https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
-EMAIL_BACKEND: str = env(
-    "DJANGO_EMAIL_BACKEND",
-    default="django.core.mail.backends.console.EmailBackend",
-)
+
+# MailHog SMTP settings for local development
+DEFAULT_FROM_EMAIL: str = "[email protected]"
 
 # WHITENOISE
 # ------------------------------------------------------------------------------

gateway/config/settings/production.py

@@ -5,7 +5,6 @@
 
 from .base import *  # noqa: F403 pylint: disable=wildcard-import,unused-wildcard-import
 from .base import DATABASES
-from .base import INSTALLED_APPS
 from .base import SPECTACULAR_SETTINGS
 from .base import env
 
@@ -104,16 +103,16 @@
 # Anymail
 # ------------------------------------------------------------------------------
 # https://anymail.readthedocs.io/en/stable/installation/#installing-anymail
-INSTALLED_APPS.extend(["anymail"])
+# INSTALLED_APPS.extend(["anymail"])
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 # https://anymail.readthedocs.io/en/stable/installation/#anymail-settings-reference
 # https://anymail.readthedocs.io/en/stable/esps/mailgun/
-EMAIL_BACKEND: str = "anymail.backends.mailgun.EmailBackend"
-ANYMAIL: dict[str, str] = {
-    "MAILGUN_API_KEY": env("MAILGUN_API_KEY"),
-    "MAILGUN_SENDER_DOMAIN": env("MAILGUN_DOMAIN"),
-    "MAILGUN_API_URL": env("MAILGUN_API_URL", default="https://api.mailgun.net/v3"),
-}
+# EMAIL_BACKEND: str = "anymail.backends.mailgun.EmailBackend"
+# ANYMAIL: dict[str, str] = {
+#     "MAILGUN_API_KEY": env("MAILGUN_API_KEY"),
+#     "MAILGUN_SENDER_DOMAIN": env("MAILGUN_DOMAIN"),
+#     "MAILGUN_API_URL": env("MAILGUN_API_URL", default="https://api.mailgun.net/v3"),
+# }
 
 # ⚠️ Setting overrides for PRODUCTION
 

gateway/sds_gateway/api_methods/admin.py

@@ -24,3 +24,10 @@ class DatasetAdmin(admin.ModelAdmin):  # pyright: ignore[reportMissingTypeArgume
     list_display = ("name", "doi")
     search_fields = ("name", "doi")
     ordering = ("-updated_at",)
+
+
+@admin.register(models.TemporaryZipFile)
+class TemporaryZipFileAdmin(admin.ModelAdmin):  # pyright: ignore[reportMissingTypeArgument]
+    list_display = ("uuid", "owner", "created_at", "expires_at")
+    search_fields = ("uuid", "owner")
+    ordering = ("-created_at",)