Introduce multi-stage Docker build (#1636)

To ensure that the 'dev' and 'prod' container images are derived from the same base image and reduce the amount of divergence between them.

.dockerignore

@@ -15,12 +15,12 @@
 /Vagrantfile.local
 /npm-debug.log
 /tmp/
+/.venv/
 /venv/
 __pycache__
 coverage
 node_modules/
 pytest.ini
-requirements-gae.txt
 
 # Distribution / packaging
 *.egg
@@ -41,10 +41,9 @@ sdist/
 var/
 wheels/
 
-.gcloudignore
 .git
+.github/
 .gitignore
 /*initdb.d/
 Makefile
-app.yml
-docker-compose*.yml
+docker-compose*.yml

.github/workflows/config.yml

@@ -18,10 +18,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: make test
-      - uses: actions/upload-artifact@v3
-        with:
-          name: django-coverage-report
-          path: htmlcov
 
   selenium-tests:
     runs-on: ubuntu-latest
@@ -30,8 +26,3 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: make selenium-test
-      - uses: actions/upload-artifact@v3
-        with:
-          name: django-coverage-report
-          path: htmlcov
-

Dockerfile

@@ -1,56 +1,58 @@
-# Use an official Python runtime based on Debian 10 "buster" as a parent image.
-FROM python:3.8.1-slim-buster
-
-# Add user that will be used in the container.
-RUN useradd wagtail
-
-# Port used by this container to serve HTTP.
-EXPOSE 8000
-
-# Set environment variables.
-# 1. Force Python stdout and stderr streams to be unbuffered.
-# 2. Set PORT variable that is used by Gunicorn. This should match "EXPOSE"
-#    command.
-ENV PYTHONUNBUFFERED=1 \
-    PORT=8000
-
-COPY requirements.txt /
-
-# Install system packages required by Wagtail and Django.
+FROM python:3.8.17-slim-bookworm AS builder
+ENV PIP_NO_CACHE_DIR=1 \
+    PYTHONUNBUFFERED=1
 RUN apt-get update --yes --quiet \
-  && apt-get install --yes --quiet --no-install-recommends \
+ && apt-get install --yes --quiet --no-install-recommends \
     build-essential \
     gettext \
     git \
-    libpq5 \
+    libpango1.0-0 \
     libpq-dev \
+    libpq5 \
+ && pip install --upgrade pip \
+ && pip install pip-tools \
+ && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt
+RUN python -m venv venv
+ENV PATH="/opt/venv/bin:$PATH"
+RUN pip install --upgrade pip
+ARG requirements=requirements.txt
+COPY ${requirements} .
+RUN pip install -r $requirements
+
+FROM python:3.8.17-slim-bookworm AS base
+EXPOSE 8000
+ENV PATH="/opt/venv/bin:$PATH" \
+    PIP_NO_CACHE_DIR=1 \
+    PYTHONUNBUFFERED=1
+RUN useradd wagtail
+RUN apt-get update --yes --quiet \
+ && apt-get install --yes --quiet --no-install-recommends \
+    gettext \
     libpango1.0-0 \
- && pip install --no-cache-dir --upgrade pip \
- && pip install --no-cache-dir "gunicorn==20.0.4" \
- && pip install --no-cache-dir -r /requirements.txt \
- && apt remove --yes --quiet git build-essential libpq-dev \
- && apt autoremove --yes --quiet \
+    libpq5 \
  && rm -rf /var/lib/apt/lists/*
-
-# Use /app folder as a directory where the source code is stored.
 WORKDIR /app
-
-# Set this directory to be owned by the "wagtail" user. This Wagtail project
-# uses SQLite, the folder needs to be owned by the user that
-# will be writing to the database file.
 RUN chown wagtail:wagtail /app
 
-# Copy the source code of the project into the container.
+FROM base AS dev
+RUN apt-get update --yes --quiet \
+ && apt-get install --yes --quiet --no-install-recommends \
+    tini \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=builder --chown=wagtail:wagtail /opt/venv /opt/venv
 COPY --chown=wagtail:wagtail . .
-
-# Use user "wagtail" to run the build commands below and the server itself.
 USER wagtail
+ENTRYPOINT ["tini", "--"]
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
 
-# Collect static files.
+FROM base AS prod
+# PORT variable is used by Gunicorn. Should match "EXPOSE" command.
+ENV PORT=8000
+USER wagtail
+COPY --from=builder --chown=wagtail:wagtail /opt/venv /opt/venv
+RUN pip install "gunicorn==20.0.4"
+COPY --chown=wagtail:wagtail . .
 RUN python manage.py collectstatic --noinput --clear
-
-# Compile files for localization
 RUN python manage.py compilemessages
-
-# Start the application server.
-CMD gunicorn iogt.wsgi:application
+CMD ["gunicorn", "iogt.wsgi:application"]

Makefile

@@ -12,20 +12,16 @@ migrate:
 setup:
 	make migrate
 	make update_elasticsearch_index
-# SSH into the django container
-ssh:
-	docker-compose run django /bin/bash
 up:
 	docker-compose up
 update_elasticsearch_index:
 	docker-compose run django python manage.py update_index
 compile-requirements:
-	docker-compose run --rm django /app/scripts/compile-requirements.sh
+	docker-compose -f docker-compose.builder.yml run --rm builder
 test:
-	docker-compose -f docker-compose.test.yml up --build -d django
+	docker-compose -f docker-compose.test.yml up --build -d
 	docker-compose -f docker-compose.test.yml exec -T django python manage.py collectstatic --noinput
-	docker-compose -f docker-compose.test.yml exec -T django coverage run --source='.' manage.py test --noinput --parallel $(IOGT_TEST_PARALLEL)
-	docker-compose -f docker-compose.test.yml exec -T django coverage html
+	docker-compose -f docker-compose.test.yml exec -T django python manage.py test --noinput --parallel $(IOGT_TEST_PARALLEL)
 	docker-compose -f docker-compose.test.yml down --remove-orphans
 selenium-test: selenium-up selenium-local selenium-down
 selenium-up:

docker-compose.builder.yml

@@ -0,0 +1,11 @@
+version: "3"
+
+services:
+  builder:
+    build:
+      context: .
+      target: builder
+    command:
+      - /app/scripts/compile-requirements.sh
+    volumes:
+      - ./:/app/

docker-compose.selenium.yml

@@ -1,15 +1,15 @@
-version: '3.5'
+version: "3"
 
 services:
   django:
     build:
       context: ./
-      dockerfile: Dockerfile.dev
+      target: dev
+      args:
+        requirements: requirements.dev.txt
     environment:
       DJANGO_SETTINGS_MODULE: iogt.settings.test
     command: ["tini", "--", "sleep", "infinity"]
-    volumes:
-      - ./:/app/
     depends_on:
       - database
       - selenium-hub

docker-compose.test.yml

@@ -1,24 +1,23 @@
-version: '3.5'
+version: "3"
 
 services:
   django:
     build:
       context: ./
-      dockerfile: Dockerfile.dev
+      target: dev
+      args:
+        requirements: requirements.dev.txt
     environment:
       DB_HOST: db
       DB_NAME: iogt
       DB_PASSWORD: iogt
       DB_PORT: '5432'
       DB_USER: iogt
       DJANGO_SETTINGS_MODULE: iogt.settings.test
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - ./:/app/
     depends_on:
       - db
   db:
-    image: postgres:alpine
+    image: postgres:14-alpine
     environment:
       POSTGRES_USER: iogt
       POSTGRES_PASSWORD: iogt

docker-compose.yml

@@ -4,10 +4,11 @@ services:
   django:
     build:
       context: ./
-      dockerfile: Dockerfile.dev
+      target: dev
+      args:
+        requirements: requirements.dev.txt
     environment:
       DJANGO_SETTINGS_MODULE: iogt.settings.dev
-    command: python manage.py runserver 0.0.0.0:8000
     volumes:
       - ./:/app/
     ports:

docs/dependency-management.md

@@ -1,37 +1,50 @@
 # Introduce pip-tools
 
 ## What is it?
-Pip-tools is a command-line toolset for managing dependencies in Python projects. 
-It streamlines the process of handling project dependencies by providing essential functionalities.
+Pip-tools is a command-line toolset for managing dependencies in Python projects. It streamlines the process of handling project dependencies by providing essential functionalities.
 
 ## Why do we use it?
 
 ### Dependency Management
-Pip-tools helps manage a project's dependencies by resolving and pinning specific versions. 
-It ensures that everyone working on the project uses the same versions of dependencies, promoting consistency and reproducibility.
+
+Pip-tools helps manage a project's dependencies by resolving and pinning specific versions. It ensures that everyone working on the project uses the same versions of dependencies, promoting consistency and reproducibility.
 
 ### Simplified Workflow
-Pip-tools simplifies the workflow of managing dependencies by keeping the requirements.in file separate from the requirements.txt file. 
+
 The requirements.in file contains high-level dependencies, while the requirements.txt file contains the pinned versions. This separation enables easy updating and maintenance of dependencies.
 
 ## How do we use it?
 
-### For Virtual Environment:
+### For virtual environment:
 
-#### Install or Upgrade pip:
-```pip install --upgrade pip```
+Install or upgrade pip.
 
-#### Install pip-tools:
-```pip install pip-tools```
+```sh
+pip install --upgrade pip
+```
 
-#### Generate Pinned Versions:
-Add or update the package and its version specifier according to your needs in <requirements_file>.in file.
+Install pip-tools.
 
-```pip-compile --generate-hashes --resolver=backtracking -o <requirements_file>.txt <requirements_file>.in```
+```sh
+pip install pip-tools
+```
+
+To generate pinned versions, add or update a package and its version specifier according to your needs in the _requirements.in_ or _requirements.dev.in_ file.
+
+```sh
+pip-compile --generate-hashes --resolver=backtracking -o <requirements_file>.txt <requirements_file>.in
+```
 
 ### For Docker
-Add or update the package and its version specifier according to your needs in <requirements_file>.in file.
+
+Add or update the package and its version specifier according to your needs in a _\*.in_ file. In the project root directory.
 
 ```
 make compile-requirements
-```
+```
+
+Alternatively, you could run `pip-compile` directly in the _builder_ container.
+
+```
+docker compose -f docker-compose.builder.yml run --rm builder pip-compile ...
+```

requirements.dev.in

@@ -1,5 +1,4 @@
 -r requirements.txt
-beautifulsoup4
 coverage
 django-test-plus>=1.4.0
 factory-boy==3.2.*

requirements.dev.txt

@@ -58,7 +58,6 @@ beautifulsoup4==4.9.3 \
     --hash=sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25 \
     --hash=sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666
     # via
-    #   -r /app/requirements.dev.in
     #   -r /app/requirements.txt
     #   django-google-analytics-app
     #   wagtail

requirements.in

@@ -1,4 +1,5 @@
 Markdown==3.3.7
+beautifulsoup4~=4.9.3
 cairosvg==2.7.0
 django-allauth~=0.44
 django-comments-xtd==2.9.*

requirements.txt

@@ -41,6 +41,7 @@ beautifulsoup4==4.9.3 \
     --hash=sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25 \
     --hash=sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666
     # via
+    #   -r /app/requirements.in
     #   django-google-analytics-app
     #   wagtail
 billiard==4.1.0 \

scripts/compile-requirements.sh

@@ -6,5 +6,10 @@ REQ_FILES=(
 )
 
 for f in "${REQ_FILES[@]}"; do
-  pip-compile --generate-hashes --resolver=backtracking -o ${f}.txt ${f}.in || exit 1;
+  pip-compile --cache-dir=/tmp/pip-tools \
+              --generate-hashes \
+              --resolver=backtracking \
+              --output-file ${f}.txt \
+              ${f}.in \
+      || exit 1;
 done