[compose] Use Docker secrets

.github/workflows/django.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, 3.10]
+        python-version: ['3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -43,8 +43,17 @@ jobs:
 
     - name: Run Tests
       run: |
-        SECRET_KEY=9 poetry run python manage.py createsecrets --exclude superuser --exclude sendgrid --exclude postgres --output ./.env
-        export $(grep -v '^#' .env | xargs)
+        poetry run python manage.py createsecrets
+
+        export SECRET_KEY="$(cat ./compose/secrets/development/secret_key.txt)"
+        export DJANGO_SUPERUSER_USERNAME="$(cat ./compose/secrets/development/django_superuser_username.txt)"
+        export DJANGO_SUPERUSER_PASSWORD="$(cat ./compose/secrets/development/django_superuser_password.txt)"
+        export DJANGO_SUPERUSER_EMAIL="$(cat ./compose/secrets/development/django_superuser_email.txt)"
+        export SENDGRID_API_KEY="$(cat ./compose/secrets/development/sendgrid_api_key.txt)"
+        export POSTGRES_PASSWORD="$(cat ./compose/secrets/development/postgres_password.txt)"
+        export POSTGRES_USER="$(cat ./compose/secrets/development/postgres_user.txt)"
+        export POSTGRES_DB="$(cat ./compose/secrets/development/postgres_db.txt)"
+
         poetry run python manage.py migrate
         poetry run python -Wa manage.py check --deploy
         poetry run python -Wa manage.py test

.gitignore

@@ -30,7 +30,6 @@ parts/
 sdist/
 var/
 wheels/
-pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -60,6 +59,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+cover/
 
 # Translations
 *.mo
@@ -84,6 +84,7 @@ instance/
 docs/_build/
 
 # PyBuilder
+.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -94,7 +95,9 @@ profile_default/
 ipython_config.py
 
 # pyenv
-.python-version
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -140,6 +143,11 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# Ignore redis binary dump (dump.rdb) files
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
 
+# Ignore redis binary dump (dump.rdb) files
 *.rdb

compose/django/entrypoint.sh

@@ -1,6 +1,17 @@
 #!/bin/sh
 set -e
 
+# these are all paths to files under /run/secrets
+export \
+  SECRET_KEY="$(cat $SECRET_KEY)" \
+  DJANGO_SUPERUSER_USERNAME="$(cat $DJANGO_SUPERUSER_USERNAME)" \
+  DJANGO_SUPERUSER_PASSWORD="$(cat $DJANGO_SUPERUSER_PASSWORD)" \
+  DJANGO_SUPERUSER_EMAIL="$(cat $DJANGO_SUPERUSER_EMAIL)" \
+  POSTGRES_USER="$(cat $POSTGRES_USER)" \
+  POSTGRES_PASSWORD="$(cat $POSTGRES_PASSWORD)" \
+  POSTGRES_DB="$(cat $POSTGRES_DB)" \
+  SENDGRID_API_KEY="$(cat $SENDGRID_API_KEY)"
+
 urlencode () {
     python3 - "$@" << END
 from urllib.parse import quote_plus
@@ -12,6 +23,7 @@ print(quote_plus(" ".join(argv[1:])))
 END
 }
 
+DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT:-5432}/${POSTGRES_DB}"
 ENCODED_DATABASE_URL=$(urlencode "$DATABASE_URL")
 
 until pg_isready --dbname="$ENCODED_DATABASE_URL" --host="postgres" --username="$POSTGRES_USER"; do

compose/secrets/development/django_superuser_email.txt

@@ -0,0 +1 @@
+[email protected]

compose/secrets/development/django_superuser_password.txt

@@ -0,0 +1 @@
+admin-secret

compose/secrets/development/django_superuser_username.txt

@@ -0,0 +1 @@
+live_admin

compose/secrets/development/postgres_db.txt

@@ -0,0 +1 @@
+hacktj_live

compose/secrets/development/postgres_password.txt

@@ -0,0 +1 @@
+817m5da7fyleau^108yko2ib!&+*!0ba38gh%g8ps()56)=gsv

compose/secrets/development/postgres_user.txt

@@ -0,0 +1 @@
+live_postgres

compose/secrets/development/secret_key.txt

@@ -0,0 +1 @@
+django-secret

compose/secrets/development/sendgrid_api_key.txt

@@ -0,0 +1 @@
+SG.KEY

docker-compose.dev.yml

@@ -1,4 +1,4 @@
-version: '3.6'
+version: '3.9'
 
 services:
   django:
@@ -9,18 +9,24 @@ services:
       - .:/app
     ports:
       - "8000:8000"
-    env_file:
-      - .env.local
     environment:
       - DEBUG=true
       - NODE_ENV=development
 
-  postgres:
-    env_file:
-      - .env.local
-
-  redis:
-    build: ./compose/redis
-    restart: always
-    sysctls:
-      net.core.somaxconn: '511'
+secrets:
+  secret_key:
+    file: ./compose/secrets/development/secret_key.txt
+  django_superuser_username:
+    file: ./compose/secrets/development/django_superuser_username.txt
+  django_superuser_password:
+    file: ./compose/secrets/development/django_superuser_password.txt
+  django_superuser_email:
+    file: ./compose/secrets/development/django_superuser_email.txt
+  postgres_user:
+    file: ./compose/secrets/development/postgres_user.txt
+  postgres_password:
+    file: ./compose/secrets/development/postgres_password.txt
+  postgres_db:
+    file: ./compose/secrets/development/postgres_db.txt
+  sendgrid_api_key:
+    file: ./compose/secrets/development/sendgrid_api_key.txt

docker-compose.prod.yml

@@ -1,34 +1,27 @@
-version: '3.6'
+version: '3.9'
 
 services:
   django:
     build:
       args:
         - DEBUG=false
-    ports:
-      # - '80:8000'
-      - '8000:8000'
-    # expose:
-    #   - 8000
     volumes:
       - ./static:/app/static
       - ./backups:/app/backup
       - ./logs:/app/logs
+    ports:
+      - "8000:8000"
     deploy:
       resources:  # for Gunicorn
         # https://github.com/benoitc/gunicorn/issues/1440#issuecomment-411785926
         limits:
           cpus: "0.1"
           memory: "4G"
-    env_file:
-      - .env
     environment:
       - DEBUG=false
       - NODE_ENV=production
 
   postgres:
-    env_file:
-      - .env
     volumes:
       - ./data:/var/lib/postgresql/data
 
@@ -45,3 +38,21 @@ services:
   #     LISTEN_IPV6: 'true'
   #     ERROR_LOG: stdout
   #     ACCESS_LOG: stderr
+
+secrets:
+  secret_key:
+    file: ./compose/secrets/production/secret_key.txt
+  django_superuser_username:
+    file: ./compose/secrets/production/django_superuser_username.txt
+  django_superuser_password:
+    file: ./compose/secrets/production/django_superuser_password.txt
+  django_superuser_email:
+    file: ./compose/secrets/production/django_superuser_email.txt
+  postgres_user:
+    file: ./compose/secrets/production/postgres_user.txt
+  postgres_password:
+    file: ./compose/secrets/production/postgres_password.txt
+  postgres_db:
+    file: ./compose/secrets/production/postgres_db.txt
+  sendgrid_api_key:
+    file: ./compose/secrets/production/sendgrid_api_key.txt

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.6'
+version: '3.9'
 
 services:
   django:
@@ -13,28 +13,57 @@ services:
       net.core.somaxconn: 2048
     command: ./compose/django/start.sh
     environment:
-      - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT:-5432}/${POSTGRES_DB}
+      - SECRET_KEY=/run/secrets/secret_key
+      - DJANGO_SUPERUSER_USERNAME=/run/secrets/django_superuser_username
+      - DJANGO_SUPERUSER_PASSWORD=/run/secrets/django_superuser_password
+      - DJANGO_SUPERUSER_EMAIL=/run/secrets/django_superuser_email
+      - POSTGRES_USER=/run/secrets/postgres_user
+      - POSTGRES_PASSWORD=/run/secrets/postgres_password
+      - POSTGRES_PORT=5432
+      - POSTGRES_DB=/run/secrets/postgres_db
+      - SENDGRID_API_KEY=/run/secrets/sendgrid_api_key
       - DOCKER=true
+      # - POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 --auth-local=scram-sha-256 --data-checksums
+      # - POSTGRES_HOST_AUTH_METHOD=scram-sha-256
     depends_on:
       - postgres
       - redis
       - memcached
     volumes:
-      - ./judge/fixtures:/app/judge/fixtures
       - ./utils/fixtures:/app/utils/fixtures
+      - ./judge/fixtures:/app/judge/fixtures
+    secrets:
+      - secret_key
+      - django_superuser_username
+      - django_superuser_password
+      - django_superuser_email
+      - postgres_user
+      - postgres_password
+      - postgres_db
+      - sendgrid_api_key
 
   postgres:
     image: postgres:14.2-alpine3.15
     command: postgres -c 'max_connections=200'
     restart: always
     shm_size: 256MB
     environment:
-      # - PGHOST=postgres
+      - POSTGRES_USER_FILE=/run/secrets/postgres_user
+      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
+      - POSTGRES_PORT=5432
+      - POSTGRES_DB_FILE=/run/secrets/postgres_db
       - PGPORT=${POSTGRES_PORT:-5432}
+      # - PGHOST=postgres
       # - PGDATABASE=${POSTGRES_DB}
       # - PGUSER=${POSTGRES_USER}
       # - PGPASSWORD=${POSTGRES_PASSWORD}
       # - PGOPTIONS=${POSTGRES_INITDB_ARGS}
+      - POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 --auth-local=scram-sha-256 --data-checksums
+      - POSTGRES_HOST_AUTH_METHOD=scram-sha-256
+    secrets:
+      - postgres_user
+      - postgres_password
+      - postgres_db
 
   redis:
     build: ./compose/redis
@@ -45,3 +74,23 @@ services:
   memcached:
     image: memcached:1.6.14-alpine3.15
     restart: always
+
+secrets:
+  # just to keep `docker compose` from complaining about
+  # `Service "postgres" uses an undefined secret`
+  secret_key:
+    external: true
+  django_superuser_username:
+    external: true
+  django_superuser_password:
+    external: true
+  django_superuser_email:
+    external: true
+  postgres_user:
+    external: true
+  postgres_password:
+    external: true
+  postgres_db:
+    external: true
+  sendgrid_api_key:
+    external: true

docs/deployment/self-hosted.md

@@ -22,11 +22,11 @@ Thanks for using HackTJ Live for your event! Here's a quick guide to running Liv
 #### Setup
 
 1.  Clone the Live repository and enter the directory: `git clone https://github.com/HackTJ/live ~/live && cd ~/live`
-2.  Prepare the secrets file. (two options)
+2.  Prepare the secrets files. (two options)
     -   `docker compose exec django poetry run python manage.py createsecrets`
-    -   manually edit the template
-        1.  copy the template: `cp .env.local .env`
-        2.  update the information in `.env` (superuser information, SendGrid API key, Postgres password). You can generate passwords using something like `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1`
+    -   manually create the files
+        1.  copy the template: `cp ./compose/secrets/development/*.txt ./compose/secrets/production`
+        2.  update the secrets. All files in the [`./compose/secrets`](./compose/secrets) directory are configurable. You can generate secrets using something like `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1`
 3.  Spin up the instance: `./start_live prod` and grab a cup of coffee or something as the system initializes. Once that's all started, the next step is to set up the webserver.
 
 ##### Nginx Setup

docs/development.md

@@ -34,15 +34,15 @@ poetry run pre-commit install
 #### Database Setup
 
 ```sh
-export POSTGRES_USER=$(awk -F "\'" '/POSTGRES_USER/ { print $2; }' .env.local)
+export POSTGRES_USER="$(cat ./compose/secrets/development/postgres_user.txt)"
 dropdb "$POSTGRES_USER"
-export POSTGRES_PASSWORD=$(awk -F "\'" '/POSTGRES_PASSWORD/ { print $2; }' .env.local)
+export POSTGRES_PASSWORD="$(cat ./compose/secrets/development/postgres_password.txt)"
 pg_ctl --pgdata=/usr/local/var/postgres initdb -U "$POSTGRES_USER" -P $POSTGRES_PASSWORD
 pg_ctl --pgdata=/usr/local/var/postgres start
 createdb "$POSTGRES_USER"
 psql "$POSTGRES_USER" --file=./docs/setup.sql
 
-export SECRET_KEY=$(awk -F "\'" '/SECRET_KEY/ { print $2; }' .env.local)
+export SECRET_KEY="$(cat ./compose/secrets/development/secret_key.txt)"
 poetry run python manage.py migrate
 
 unset POSTGRES_USER
@@ -52,7 +52,6 @@ unset SECRET_KEY
 
 ### Making Changes
 
--   If you update the [.env.local](./.env.local) file, make sure you surround all values with single quotes. If you rename the file, make sure to update this document with the correct name.
 -   If you have direct commit/push access, be careful. Create PRs for big changes or any changes that might require testing, e.g., CI configuration updates.
 -   ["Squash and merge"](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/about-pull-request-merges#squash-and-merge-your-pull-request-commits) all PRs.
--   If you make any changes to the front-end/templates, make sure to run `pushd tailwind/ && NODE_ENV=production yarn build && popd`.
+-   If you make any changes to the front-end (the Django templates), make sure to run `pushd tailwind/ && NODE_ENV=production yarn run build && popd`.