Merge pull request #74 from maykinmedia/feature/get-test-env-working

Get test env working

Dockerfile

@@ -47,6 +47,7 @@ COPY ./frontend/package-lock.json ./frontend/package.json ./
 RUN npm ci
 
 COPY ./frontend .
+COPY ./frontend/.env.production.template ./.env.production
 
 RUN npm run build
 
@@ -65,13 +66,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
+
 COPY ./backend/bin/docker_start.sh /start.sh
+COPY ./backend/bin/celery_worker.sh /celery_worker.sh
+COPY ./backend/bin/celery_beat.sh /celery_beat.sh
+COPY ./backend/bin/celery_flower.sh /celery_flower.sh
+COPY ./backend/bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py
+COPY ./frontend/scripts/replace-envvars.sh /replace-envvars.sh
 
 RUN mkdir -p /app/log /app/media /app/src/openarchiefbeheer/static/
 
 # copy backend build deps
 COPY --from=backend-build /usr/local/lib/python3.12 /usr/local/lib/python3.12
 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
+COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
 
 COPY ./backend/src /app/src
 

backend/Dockerfile

@@ -66,11 +66,13 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
+
 COPY ./bin/docker_start.sh /start.sh
-# Uncomment if you use celery
-# COPY ./bin/celery_worker.sh /celery_worker.sh
-# COPY ./bin/celery_beat.sh /celery_beat.sh
-# COPY ./bin/celery_flower.sh /celery_flower.sh
+COPY ./bin/celery_worker.sh /celery_worker.sh
+COPY ./bin/celery_beat.sh /celery_beat.sh
+COPY ./bin/celery_flower.sh /celery_flower.sh
+COPY ./bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py
+
 RUN mkdir /app/bin /app/log /app/media
 
 VOLUME ["/app/log", "/app/media"]

backend/bin/celery_beat.sh

@@ -7,8 +7,6 @@ LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
 mkdir -p celerybeat
 
 echo "Starting celery beat"
-exec celery beat \
-    --app openarchiefbeheer \
+exec celery --workdir src --app openarchiefbeheer.celery worker \
     -l $LOGLEVEL \
-    --workdir src \
     -s ../celerybeat/beat

backend/bin/check_celery_worker_liveness.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Check the health of a Celery worker.
+#
+# The worker process writes and periodically touches a number of files that indicate it
+# is available and still healthy. If the worker becomes unhealthy for any reason, the
+# timestamp of when the heartbeat file was last touched will not update and the delta
+# becomes too big, allowing (container) orchestration to terminate and restart the
+# worker process.
+#
+# Example usage with Kubernetes, as a liveness probe:
+#
+# .. code-block:: yaml
+#
+#       livenessProbe:
+#         exec:
+#           command:
+#           - python
+#           - /app/bin/check_celery_worker_liveness.py
+#         initialDelaySeconds: 10
+#         periodSeconds: 30  # must be smaller than `MAX_WORKER_LIVENESS_DELTA`
+#
+# Reference: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command
+#
+# Supported environment variables:
+#
+# * ``MAX_WORKER_LIVENESS_DELTA``: maximum delta between heartbeats before reporting
+#   failure, in seconds. Defaults to 60 (one minute).
+
+
+import os
+import sys
+import time
+from pathlib import Path
+
+HEARTBEAT_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_heartbeat"
+READINESS_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_ready"
+MAX_WORKER_LIVENESS_DELTA = int(os.getenv("MAX_WORKER_LIVENESS_DELTA", 60))  # seconds
+
+
+# check if worker is ready
+if not READINESS_FILE.is_file():
+    print("Celery worker not ready.")
+    sys.exit(1)
+
+# check if worker is live
+if not HEARTBEAT_FILE.is_file():
+    print("Celery worker heartbeat not found.")
+    sys.exit(1)
+
+# check if worker heartbeat satisfies constraint
+stats = HEARTBEAT_FILE.stat()
+worker_timestamp = stats.st_mtime
+current_timestamp = time.time()
+time_diff = current_timestamp - worker_timestamp
+
+if time_diff > MAX_WORKER_LIVENESS_DELTA:
+    print("Celery worker heartbeat: interval exceeds constraint (60s).")
+    sys.exit(1)
+
+print("Celery worker heartbeat found: OK.")
+sys.exit(0)

backend/bin/docker_start.sh

@@ -26,6 +26,10 @@ done
 >&2 echo "Apply database migrations"
 python src/manage.py migrate
 
+# Populate the environment variables for the frontend
+>&2 echo "Replace frontend env vars"
+/replace-envvars.sh
+
 # Start server
 >&2 echo "Starting server"
 exec uwsgi \

docker-compose.yml

@@ -24,7 +24,7 @@ services:
 
   web:
     image: maykinmedia/open-archiefbeheer:latest
-    environment:
+    environment: &web_env
       - ALLOWED_HOSTS=localhost
       - DJANGO_SETTINGS_MODULE=openarchiefbeheer.conf.docker
       - SECRET_KEY=${SECRET_KEY:-django-insecure-!bkx+tx18&lvp(@_9)9ut(y(keqho*zhz1&^sqqgq9*i=__w(}
@@ -41,6 +41,11 @@ services:
       - SESSION_COOKIE_SECURE=False
       - TWO_FACTOR_FORCE_OTP_ADMIN=False
       - TWO_FACTOR_PATCH_ADMIN=False
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - CELERY_LOGLEVEL=DEBUG
+      - REACT_APP_API_URL=http://localhost:8080
+      - REACT_APP_API_PATH=/api/v1
     ports:
       - 8080:8080
     depends_on:
@@ -49,6 +54,32 @@ services:
     networks:
       - open-archiefbeheer-dev
 
+  celery:
+    image: maykinmedia/open-archiefbeheer:latest
+    command: /celery_worker.sh
+    environment: *web_env
+    healthcheck:
+      test: [ "CMD", "python", "/app/bin/check_celery_worker_liveness.py" ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    depends_on:
+      - db
+      - redis
+    networks:
+      - open-archiefbeheer-dev
+
+  celery-beat:
+    image: maykinmedia/open-archiefbeheer:latest
+    command: /celery_beat.sh
+    environment: *web_env
+    depends_on:
+      - db
+      - redis
+    networks:
+      - open-archiefbeheer-dev
+
   nginx:
     image: nginx
     volumes:

frontend/.env.example

@@ -0,0 +1,2 @@
+REACT_APP_API_URL=http://localhost:8080
+REACT_APP_API_PATH=/api/v1

frontend/.env.production.template

@@ -0,0 +1,2 @@
+REACT_APP_API_URL=REACT_APP_API_URL_PLACEHOLDER
+REACT_APP_API_PATH=REACT_APP_API_PATH_PLACEHOLDER

frontend/scripts/replace-envvars.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+for file in /app/static/js/*.js;
+do
+  sed -i 's|REACT_APP_API_URL_PLACEHOLDER|'${REACT_APP_API_URL}'|g' $file
+  sed -i 's|REACT_APP_API_PATH_PLACEHOLDER|'${REACT_APP_API_PATH}'|g' $file
+done

frontend/src/lib/api/request.ts

@@ -1,19 +1,13 @@
 import { getCookie } from "../cookie/cookie";
 
-/** Scheme for all API requests.. */
-export const API_SCHEME = "http";
-
-/** The host for the API server. */
-export const API_HOST = "localhost";
-
-/** The port for the API server. */
-export const API_PORT = 8080;
+/** Scheme for all API requests. */
+export const API_URL = process.env.REACT_APP_API_URL;
 
 /** The base path for all API requests. */
-export const API_PATH = "/api/v1";
+export const API_PATH = process.env.REACT_APP_API_PATH;
 
 /** The base url for all API requests. */
-export const API_BASE_URL = `${API_SCHEME}://${API_HOST}:${API_PORT}${API_PATH}`;
+export const API_BASE_URL = `${API_URL}${API_PATH}`;
 
 /**
  * Makes an actual fetch request to the API, should be used by all other API implementations.