Merge pull request #73 from GateNLP/docker_deploy_2

deploy in docker container

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+frontend/node_modules

.gitignore

@@ -2,6 +2,9 @@
 node_modules
 /dist
 
+/static
+frontend/static
+
 # Vue generated html template
 base-vue.html
 
@@ -12,6 +15,7 @@ cypress/videos
 # local env files
 .env.local
 .env.*.local
+saved-env.*
 
 # Log files
 npm-debug.log*
@@ -168,4 +172,4 @@ cython_debug/
 
 webpack-stats\.json
 
-annotation_tool/settings/secret.py
+*secret.py

Dockerfile

@@ -0,0 +1,40 @@
+FROM continuumio/miniconda3:latest
+
+ENV PYTHONUNBUFFERED 1
+
+RUN apt-get --allow-releaseinfo-change update && \
+    apt-get -y install gcc libpq-dev libmagic1 && \
+    rm -rf /var/lib/apt/lists/*
+
+ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /sbin/tini
+RUN addgroup --gid 1001 "gate" && \
+      adduser --disabled-password --gecos "GATE User,,," \
+        --home /app --ingroup gate --uid 1001 gate && \
+      chmod +x /sbin/tini
+
+COPY --chown=gate:gate environment.yml /app/
+WORKDIR /app
+RUN chown --recursive gate:gate /opt/conda/
+USER gate:gate
+RUN conda env create --name annotation-tool --file environment.yml
+RUN conda install --name annotation-tool gunicorn
+
+COPY --chown=gate:gate package.json package-lock.json /app/
+COPY --chown=gate:gate frontend/package.json frontend/package-lock.json /app/frontend/
+
+SHELL ["conda", "run", "-n", "annotation-tool", "/bin/bash", "-c"]
+
+RUN npm install --unsafe-perm --only=production
+
+COPY --chown=gate:gate manage.py /app/
+COPY --chown=gate:gate examples/ /app/examples/
+COPY --chown=gate:gate annotation_tool/ /app/annotation_tool/
+COPY --chown=gate:gate backend/ /app/backend
+COPY --chown=gate:gate frontend/ /app/frontend/
+RUN npm run build
+RUN python manage.py collectstatic --noinput
+
+COPY --chown=gate:gate run-server.sh generate-env.sh /app/
+
+USER root
+ENTRYPOINT [ "/app/run-server.sh" ]

README.md

@@ -254,3 +254,19 @@ describe('Example test suite', () => {
 })
 ```
 
+## Deployment
+Deployment is via docker-compose, using nginx to serve static content.
+
+A deployment script is provided to launch the docker-compose stack with the correct environment variables. This also runs `./generate-env.sh` to create a `.env` file containing randomly generated secrets which are mounted as envrionment variables into the container.
+
+First build the images via:
+```bash
+./build-images.sh
+```
+
+then deploy the stack with
+
+```bash
+./deploy.sh production # (or prod) to deploy with production settings
+./deploy.sh staging # (or stag) to deploy with staging settings
+```

annotation_tool/settings/base.py

@@ -31,7 +31,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = ['localhost', '127.0.0.1']
+ALLOWED_HOSTS = ['localhost', '127.0.0.1','0.0.0.0']
 
 
 # Application definition

annotation_tool/settings/production.py

@@ -1,7 +1,36 @@
+import logging
+import sys
 from .base import *
 
 # Enable csrf in production
 MIDDLEWARE.append(
 'django.middleware.csrf.CsrfViewMiddleware'
 )
 
+ALLOWED_HOSTS.append('annotate.gate.ac.uk')
+
+
+LOGGING = {
+   'version': 1,
+   'disable_existing_loggers': False,
+   'formatters': {
+       'verbose': {
+           'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+       },
+   },
+   'handlers': {
+       'console': {
+           'level': 'INFO',
+           'class': 'logging.StreamHandler',
+           'stream': sys.stdout,
+           'formatter': 'verbose'
+       },
+   },
+   'loggers': {
+       '': {
+           'handlers': ['console'],
+           'level': 'INFO',
+           'propagate': True,
+       },
+   },
+}

annotation_tool/settings/secret_docker.py

@@ -0,0 +1,6 @@
+# "Secret" config for use in the docker container, that takes secrets from environment vars
+import os
+
+SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY')
+POSTGRES_USERNAME = os.environ.get('DB_USERNAME')
+POSTGRES_PASSWORD = os.environ.get('DB_PASSWORD')

annotation_tool/settings/staging.py

@@ -4,3 +4,30 @@
 MIDDLEWARE.append(
 'django.middleware.csrf.CsrfViewMiddleware'
 )
+
+ALLOWED_HOSTS.append('annotate-test.gate.ac.uk')
+
+LOGGING = {
+   'version': 1,
+   'disable_existing_loggers': False,
+   'formatters': {
+       'verbose': {
+           'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+       },
+   },
+   'handlers': {
+       'console': {
+           'level': 'INFO',
+           'class': 'logging.StreamHandler',
+           'stream': sys.stdout,
+           'formatter': 'verbose'
+       },
+   },
+   'loggers': {
+       '': {
+           'handlers': ['console'],
+           'level': 'INFO',
+           'propagate': True,
+       },
+   },
+}

build-images.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+docker build -t annotate-backend:latest .
+
+docker build -t annotate-static:latest nginx/

deploy.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+echo "Generating .env file"
+./generate-env.sh
+
+DEPLOY_ENV=$1
+
+case $DEPLOY_ENV in
+
+  production|prod)
+    DJANGO_SETTINGS_MODULE=annotation_tool.settings.production
+    ;;
+
+  staging|stag)
+    DJANGO_SETTINGS_MODULE=annotation_tool.settings.staging
+    ;;
+
+  *)
+    exit 1
+    ;;
+esac
+
+echo "Deploying with DJANGO_SETTINGS_MODULE: $DJANGO_SETTINGS_MODULE"
+
+DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE docker-compose up -d

docker-compose.yml

@@ -0,0 +1,64 @@
+version: "3.9"
+
+services:
+
+  backend:
+    image: annotate-backend:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    environment:
+      - DJANGO_SETTINGS_MODULE
+      - DJANGO_SECRET_KEY
+      - DJANGO_DB_NAME
+      - DB_USERNAME
+      - DB_PASSWORD
+    # depends_on:
+    #   - db
+    volumes:
+      - ./db.sqlite3:/app/db.sqlite3
+      - logs/:/logs/
+    networks:
+      - django-nginx
+
+  nginx:
+    image: annotate-static:latest
+    build:
+      context: .
+      dockerfile: ./nginx/Dockerfile
+    ports:
+      - 80:80
+    depends_on:
+      - backend
+      # - db
+    networks:
+      - django-nginx
+
+
+#   db:
+#     image: postgres:12-alpine
+#     restart: always
+#     expose:
+#       - "5432"
+#     env_file: 
+#       - ./.env
+#     environment:
+#       - POSTGRES_USER=postgres
+#       - POSTGRES_PASSWORD=${PG_SUPERUSER_PASSWORD}
+#       - DJANGO_DB_NAME
+#       - DJANGO_DB_USER=${DB_USERNAME}
+#       - DJANGO_DB_PASSWORD=${DB_PASSWORD}
+#       - DB_BACKUP_USER
+#       - DB_BACKUP_PASSWORD
+#     volumes:
+#       - postgresql-data:/var/lib/postgresql/data
+#       - ./create-django-db.sh:/docker-entrypoint-initdb.d/001-create-django-db.sh
+
+volumes:
+  logs:
+#   postgresql-data:
+
+networks:
+  django-nginx:
+    driver: bridge

frontend/package.json

@@ -9,6 +9,11 @@
     "test": "jest"
   },
   "dependencies": {
+    "@vue/cli-plugin-babel": "~4.5.0",
+    "@vue/cli-plugin-router": "~4.5.0",
+    "@vue/cli-plugin-unit-jest": "^4.5.12",
+    "@vue/cli-plugin-vuex": "~4.5.0",
+    "@vue/cli-service": "~4.5.0",
     "axios": "^0.21.1",
     "bootstrap": "^4.6.0",
     "bootstrap-vue": "^2.21.2",
@@ -19,29 +24,23 @@
     "vue": "^2.6.11",
     "vue-json-pretty": "^1.8.0",
     "vue-router": "^3.2.0",
+    "vue-template-compiler": "^2.6.11",
     "vuex": "^3.4.0",
-    "webpack-bundle-tracker": "^0.4.3"
-  },
-  "devDependencies": {
-    "@testing-library/jest-dom": "^5.12.0",
-    "@testing-library/vue": "^5.6.2",
-    "@vue/cli-plugin-babel": "~4.5.0",
-    "@vue/cli-plugin-router": "~4.5.0",
-    "@vue/cli-plugin-unit-jest": "^4.5.12",
-    "@vue/cli-plugin-vuex": "~4.5.0",
-    "@vue/cli-service": "~4.5.0",
-    "@vue/test-utils": "^1.0.3",
+    "webpack-bundle-tracker": "^0.4.3",
     "css-loader": "^5.2.1",
     "sass": "^1.32.8",
     "sass-loader": "^10.1.1",
     "style-loader": "^2.0.0",
-    "vue-template-compiler": "^2.6.11",
     "webpack": "^4.44.2",
     "webpack-bundle-analyzer": "^3.9.0",
-    "webpack-bundle-tracker": "^0.4.3",
     "webpack-cli": "^3.3.12",
     "webpack-dev-server": "^3.11.2"
   },
+  "devDependencies": {
+    "@testing-library/jest-dom": "^5.12.0",
+    "@testing-library/vue": "^5.6.2",
+    "@vue/test-utils": "^1.0.3"
+  },
   "browserslist": [
     "> 1%",
     "last 2 versions",

generate-env.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# Creates a .env file with random passwords and Django secret key
+#
+
+if [ -f .env ]; then
+  BAKNAME=$(date +%Y-%m-%d-%H-%M-%S)
+  echo "Existing .env found - backing up as saved-env.$BAKNAME"
+  cp .env saved-env.$BAKNAME
+fi
+
+cat > .env <<EOF
+PG_SUPERUSER_PASSWORD=$(openssl rand -base64 16)
+DJANGO_DB_NAME=annotations_db
+DB_USERNAME=gate
+DB_PASSWORD=$(openssl rand -base64 16)
+DJANGO_SECRET_KEY=$(openssl rand -base64 42)
+DB_BACKUP_USER=backup
+DB_BACKUP_PASSWORD=$(openssl rand -base64 16)
+EOF

nginx/Dockerfile

@@ -0,0 +1,5 @@
+FROM nginx:stable-alpine
+RUN rm /etc/nginx/conf.d/default.conf
+COPY production.conf /etc/nginx/nginx.conf
+COPY --from=backend:latest /app/static /usr/share/nginx/html
+COPY --from=backend:latest /app/frontend/ /usr/share/nginx/html/frontend

nginx/production.conf

@@ -0,0 +1,32 @@
+user  nginx;
+worker_processes  1;
+
+events {
+  worker_connections  1024;
+}
+
+http{
+
+  include /etc/nginx/mime.types;
+
+  upstream annotation_tool {
+      server backend:8000;
+  }
+
+  server {
+
+      listen 80;
+
+      location / {
+          proxy_pass http://annotation_tool;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header Host $host;
+          proxy_redirect off;
+      }
+
+      location /static/ {
+          alias /usr/share/nginx/html/;
+      }
+
+  }
+}