diff --git a/Dockerfile b/Dockerfile index 7515069fe..81e63510a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,7 @@ COPY ./frontend/package-lock.json ./frontend/package.json ./ RUN npm ci COPY ./frontend . +COPY ./frontend/.env.production.template ./.env.production RUN npm run build @@ -65,13 +66,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* WORKDIR /app + COPY ./backend/bin/docker_start.sh /start.sh +COPY ./backend/bin/celery_worker.sh /celery_worker.sh +COPY ./backend/bin/celery_beat.sh /celery_beat.sh +COPY ./backend/bin/celery_flower.sh /celery_flower.sh +COPY ./backend/bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py +COPY ./frontend/scripts/replace-envvars.sh /replace-envvars.sh RUN mkdir -p /app/log /app/media /app/src/openarchiefbeheer/static/ # copy backend build deps COPY --from=backend-build /usr/local/lib/python3.12 /usr/local/lib/python3.12 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi +COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery COPY ./backend/src /app/src diff --git a/backend/Dockerfile b/backend/Dockerfile index ee4827799..435ffa39e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -66,11 +66,13 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco && rm -rf /var/lib/apt/lists/* WORKDIR /app + COPY ./bin/docker_start.sh /start.sh -# Uncomment if you use celery -# COPY ./bin/celery_worker.sh /celery_worker.sh -# COPY ./bin/celery_beat.sh /celery_beat.sh -# COPY ./bin/celery_flower.sh /celery_flower.sh +COPY ./bin/celery_worker.sh /celery_worker.sh +COPY ./bin/celery_beat.sh /celery_beat.sh +COPY ./bin/celery_flower.sh /celery_flower.sh +COPY ./bin/check_celery_worker_liveness.py /check_celery_worker_liveness.py + RUN mkdir /app/bin /app/log /app/media VOLUME ["/app/log", "/app/media"] diff --git a/backend/bin/celery_beat.sh b/backend/bin/celery_beat.sh index 24e1f8089..2a35830da 100755 --- a/backend/bin/celery_beat.sh +++ b/backend/bin/celery_beat.sh @@ -7,8 +7,6 @@ LOGLEVEL=${CELERY_LOGLEVEL:-INFO} mkdir -p celerybeat echo "Starting celery beat" -exec celery beat \ - --app openarchiefbeheer \ +exec celery --workdir src --app openarchiefbeheer.celery worker \ -l $LOGLEVEL \ - --workdir src \ -s ../celerybeat/beat diff --git a/backend/bin/check_celery_worker_liveness.py b/backend/bin/check_celery_worker_liveness.py new file mode 100644 index 000000000..ee23be437 --- /dev/null +++ b/backend/bin/check_celery_worker_liveness.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Check the health of a Celery worker. +# +# The worker process writes and periodically touches a number of files that indicate it +# is available and still healthy. If the worker becomes unhealthy for any reason, the +# timestamp of when the heartbeat file was last touched will not update and the delta +# becomes too big, allowing (container) orchestration to terminate and restart the +# worker process. +# +# Example usage with Kubernetes, as a liveness probe: +# +# .. code-block:: yaml +# +# livenessProbe: +# exec: +# command: +# - python +# - /app/bin/check_celery_worker_liveness.py +# initialDelaySeconds: 10 +# periodSeconds: 30 # must be smaller than `MAX_WORKER_LIVENESS_DELTA` +# +# Reference: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command +# +# Supported environment variables: +# +# * ``MAX_WORKER_LIVENESS_DELTA``: maximum delta between heartbeats before reporting +# failure, in seconds. Defaults to 60 (one minute). + + +import os +import sys +import time +from pathlib import Path + +HEARTBEAT_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_heartbeat" +READINESS_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_ready" +MAX_WORKER_LIVENESS_DELTA = int(os.getenv("MAX_WORKER_LIVENESS_DELTA", 60)) # seconds + + +# check if worker is ready +if not READINESS_FILE.is_file(): + print("Celery worker not ready.") + sys.exit(1) + +# check if worker is live +if not HEARTBEAT_FILE.is_file(): + print("Celery worker heartbeat not found.") + sys.exit(1) + +# check if worker heartbeat satisfies constraint +stats = HEARTBEAT_FILE.stat() +worker_timestamp = stats.st_mtime +current_timestamp = time.time() +time_diff = current_timestamp - worker_timestamp + +if time_diff > MAX_WORKER_LIVENESS_DELTA: + print("Celery worker heartbeat: interval exceeds constraint (60s).") + sys.exit(1) + +print("Celery worker heartbeat found: OK.") +sys.exit(0) diff --git a/backend/bin/docker_start.sh b/backend/bin/docker_start.sh index bfbe2853a..85a1a067b 100755 --- a/backend/bin/docker_start.sh +++ b/backend/bin/docker_start.sh @@ -26,6 +26,10 @@ done >&2 echo "Apply database migrations" python src/manage.py migrate +# Populate the environment variables for the frontend +>&2 echo "Replace frontend env vars" +/replace-envvars.sh + # Start server >&2 echo "Starting server" exec uwsgi \ diff --git a/docker-compose.yml b/docker-compose.yml index 4255182ae..7c97ae5a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: web: image: maykinmedia/open-archiefbeheer:latest - environment: + environment: &web_env - ALLOWED_HOSTS=localhost - DJANGO_SETTINGS_MODULE=openarchiefbeheer.conf.docker - SECRET_KEY=${SECRET_KEY:-django-insecure-!bkx+tx18&lvp(@_9)9ut(y(keqho*zhz1&^sqqgq9*i=__w(} @@ -41,6 +41,11 @@ services: - SESSION_COOKIE_SECURE=False - TWO_FACTOR_FORCE_OTP_ADMIN=False - TWO_FACTOR_PATCH_ADMIN=False + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - CELERY_LOGLEVEL=DEBUG + - REACT_APP_API_URL=http://localhost:8080 + - REACT_APP_API_PATH=/api/v1 ports: - 8080:8080 depends_on: @@ -49,6 +54,32 @@ services: networks: - open-archiefbeheer-dev + celery: + image: maykinmedia/open-archiefbeheer:latest + command: /celery_worker.sh + environment: *web_env + healthcheck: + test: [ "CMD", "python", "/app/bin/check_celery_worker_liveness.py" ] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + depends_on: + - db + - redis + networks: + - open-archiefbeheer-dev + + celery-beat: + image: maykinmedia/open-archiefbeheer:latest + command: /celery_beat.sh + environment: *web_env + depends_on: + - db + - redis + networks: + - open-archiefbeheer-dev + nginx: image: nginx volumes: diff --git a/frontend/.env.example b/frontend/.env.example new file mode 100644 index 000000000..a1382b298 --- /dev/null +++ b/frontend/.env.example @@ -0,0 +1,2 @@ +REACT_APP_API_URL=http://localhost:8080 +REACT_APP_API_PATH=/api/v1 diff --git a/frontend/.env.production.template b/frontend/.env.production.template new file mode 100644 index 000000000..74c16dccd --- /dev/null +++ b/frontend/.env.production.template @@ -0,0 +1,2 @@ +REACT_APP_API_URL=REACT_APP_API_URL_PLACEHOLDER +REACT_APP_API_PATH=REACT_APP_API_PATH_PLACEHOLDER diff --git a/frontend/scripts/replace-envvars.sh b/frontend/scripts/replace-envvars.sh new file mode 100755 index 000000000..aca09e80f --- /dev/null +++ b/frontend/scripts/replace-envvars.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eux -o pipefail + +for file in /app/static/js/*.js; +do + sed -i 's|REACT_APP_API_URL_PLACEHOLDER|'${REACT_APP_API_URL}'|g' $file + sed -i 's|REACT_APP_API_PATH_PLACEHOLDER|'${REACT_APP_API_PATH}'|g' $file +done diff --git a/frontend/src/lib/api/request.ts b/frontend/src/lib/api/request.ts index 361ce2b99..bf08cafb5 100644 --- a/frontend/src/lib/api/request.ts +++ b/frontend/src/lib/api/request.ts @@ -1,19 +1,13 @@ import { getCookie } from "../cookie/cookie"; -/** Scheme for all API requests.. */ -export const API_SCHEME = "http"; - -/** The host for the API server. */ -export const API_HOST = "localhost"; - -/** The port for the API server. */ -export const API_PORT = 8080; +/** Scheme for all API requests. */ +export const API_URL = process.env.REACT_APP_API_URL; /** The base path for all API requests. */ -export const API_PATH = "/api/v1"; +export const API_PATH = process.env.REACT_APP_API_PATH; /** The base url for all API requests. */ -export const API_BASE_URL = `${API_SCHEME}://${API_HOST}:${API_PORT}${API_PATH}`; +export const API_BASE_URL = `${API_URL}${API_PATH}`; /** * Makes an actual fetch request to the API, should be used by all other API implementations.