changed: docker setup overhaul

.dockerignore

@@ -0,0 +1,6 @@
+Dockerfile
+node_modules
+docker-compose.yml
+setup/docker/envfile.txt
+setup/docker/secrets
+

.gitignore

@@ -15,8 +15,8 @@ config/config.json
 .DS_Store
 .env
 enketo-main.rdb
-setup/docker/envfile.txt
-setup/docker/secrets
+setup/docker/secrets/*
+setup/docker/redis_main_data/*
 logs/submissions*
 logs/logrotate
 public/temp-client-config.json

CHANGELOG.md

@@ -4,6 +4,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 [Unreleased]
 ---------------------
+##### Changed
+- Docker setup.
+
 ##### Fixed
 - Excessive submission logging.
 

Dockerfile

@@ -1,35 +1,44 @@
-FROM ubuntu:trusty
+FROM phusion/baseimage:latest
 
-EXPOSE 8005
-CMD ["bash", "/srv/enketo-express/setup/docker/entrypoint.bash"]
+ENV ENKETO_SRC_DIR=/srv/src/enketo_express
 
 ################
 # apt installs #
 ################
 
-WORKDIR /srv
+# Install Node.
+ADD https://deb.nodesource.com/setup_4.x /tmp/
+RUN bash /tmp/setup_4.x
+
+COPY ./setup/docker/apt_requirements.txt ${ENKETO_SRC_DIR}/setup/docker/
+WORKDIR ${ENKETO_SRC_DIR}/
 RUN apt-get update && \
-    apt-get upgrade -y
-RUN apt-get install -y curl && \
-    curl -sL https://deb.nodesource.com/setup_4.x | bash -
-COPY ./setup/docker/apt_packages.txt /srv/
-RUN apt-get install -y $(cat apt_packages.txt)
+    apt-get upgrade -y && \
+    apt-get install -y $(cat setup/docker/apt_requirements.txt) && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 # Non-interactive equivalent of `dpkg-reconfigure -plow unattended-upgrades` (see https://blog.sleeplessbeastie.eu/2015/01/02/how-to-perform-unattended-upgrades/).
 RUN cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
-
 ###############################
 # Enketo Express Installation #
 ###############################
 
 RUN npm install -g grunt-cli pm2
-# Checks out a fresh copy of the repo.
-RUN git clone https://github.com/enketo/enketo-express.git
-WORKDIR /srv/enketo-express
-RUN npm cache clean &&\
-    npm install
+COPY ./package.json ${ENKETO_SRC_DIR}/
+RUN npm install --production
+
+COPY . ${ENKETO_SRC_DIR}
+ENV PATH $PATH:${KPI_SRC_DIR}/node_modules/.bin
 
 # Persist the `secrets` directory so the encryption key remains consistent.
-RUN mkdir -p /srv/enketo-express/setup/docker/secrets
-VOLUME /srv/enketo-express/setup/docker/secrets
+RUN mkdir -p ${ENKETO_SRC_DIR}/setup/docker/secrets
+VOLUME ${ENKETO_SRC_DIR}/setup/docker/secrets
+
+# Prepare for execution.
+RUN ln -s "${ENKETO_SRC_DIR}/setup/docker/01_setup_enketo.bash" /etc/my_init.d/ && \
+    mkdir -p /etc/service/enketo_express && \
+    ln -s "${ENKETO_SRC_DIR}/setup/docker/run_enketo.bash" /etc/service/enketo_express/run
 
+
+EXPOSE 8005

README.md

@@ -30,10 +30,13 @@ _\* sometimes `vagrant up` fails for reasons beyond our control - e.g. if extern
 
 #### Using Docker:
 1. Install [Docker Compose](http://docs.docker.com/compose/install/).
-2. Set Enketo Express's API key, the linked form and data server, and any additional desired configurations in the file `setup/docker/envfile.txt`.
-3. Run `docker-compose up` from project folder and wait until it completes.
+2. Create a [config file](./config/) at `config/config.json` specifying at minimum an API key.
+3. **(Optional)** For HTTPS, copy your SSL certificate and key files to `setup/docker/secrets/ssl.crt` and `setup/docker/secrets/ssl.key` respectively (take care not to commit these files back to any public git repository). Plain HTTP requests to Enketo Express will be automatically redirected to HTTPS.
+4. Execute `docker-compose up -d` from the [`setup/docker`](./setup/docker) directory and wait to see e.g. `Worker 1 ready for duty...`.
+5. To stop, execute `docker-compose stop` from the [`setup/docker`](./setup/docker) directory. Database dumps from the main Redis instance will be mapped into the directory `setup/docker/redis_main_data/`.
+
+The app should now be running on [localhost](http://localhost).
 
-The app should now be running on [localhost:8005](http://localhost:8005).
 
 ### How to install a production server
 

config/default-config.json

@@ -23,7 +23,7 @@
         "submissions": false
     },
     "support": {
-        "email": "info@kobotoolbox.org"
+        "email": "support@kobotoolbox.org"
     },
     "analytics": "google",
     "google": {

setup/docker/01_setup_enketo.bash

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+source /etc/profile
+
+cd ${ENKETO_SRC_DIR}/
+
+# Create a config. file if necessary.
+python setup/docker/create_config.py
+
+# Build.
+grunt
+# FIXME: Now that the client config. has been built, build again as a workaround to a `grunt` task sequencing issue.
+grunt

setup/docker/create_config.py

@@ -5,55 +5,56 @@
 
 CURRENT_DIR_PATH= os.path.abspath(os.path.dirname(__file__))
 PROJECT_ROOT_PATH= os.path.abspath(os.path.join(CURRENT_DIR_PATH, '../..'))
-print CURRENT_DIR_PATH
-print os.path.abspath(os.path.join(PROJECT_ROOT_PATH, 'setup/docker'))
-assert CURRENT_DIR_PATH == os.path.abspath(os.path.join(PROJECT_ROOT_PATH, 'setup/docker'))
 
-
-def get_encryption_key():
+def get_or_create_encryption_key():
     '''Automate the inconvenient task of generating and maintaining a consistent 
        encryption key.'''
     # Attempt to get the key from an environment variable.
     encryption_key= os.environ.get('ENKETO_ENCRYPTION_KEY')
 
     # If the key wasn't in the environment, attempt to get it from disk.
-    encryption_key_file_path= os.path.join(CURRENT_DIR_PATH, 'secrets/enketo_encryption_key.txt')
-    if not encryption_key and os.path.isfile(os.path.join(encryption_key_file_path)):
+    secrets_dir_path= os.path.join(CURRENT_DIR_PATH, 'secrets/')
+    encryption_key_file_path= os.path.join(secrets_dir_path, 'enketo_encryption_key.txt')
+    if not encryption_key and os.path.isfile(encryption_key_file_path):
         with open(encryption_key_file_path, 'r') as encryption_key_file:
             encryption_key= encryption_key_file.read().strip()
-    # If the key couldn't be retrieved, generate and store a new one.
+    # If the key couldn't be retrieved from disk, generate and store a new one.
     elif not encryption_key:
         encryption_key= base64.b64encode(os.urandom(256))
+        if not os.path.isdir(secrets_dir_path):
+            os.mkdir(secrets_dir_path)
         with open(encryption_key_file_path, 'w') as encryption_key_file:
             encryption_key_file.write(encryption_key)
 
     return encryption_key
 
 
 def create_config():
-    config= dict()
-
-    offline_enabled= os.environ.get('ENKETO_OFFLINE_SURVEYS', 'True').lower() == 'true'
-    if offline_enabled:
-        config['offline enabled']= 'True'
-
-    config['linked form and data server']= dict()
-    config['linked form and data server']['api key']= os.environ['ENKETO_API_KEY']
-    config['linked form and data server']['server url']= os.environ.get('ENKETO_FORM_DATA_SERVER_URL', 'kobocat')
-    config['linked form and data server']['encryption key']= get_encryption_key()
-
-    config['redis']= dict()
-    config['redis']['main']= {'host': os.environ.get('ENKETO_REDIS_MAIN_HOST', 'redis_main'), 
-                              'port': os.environ.get('ENKETO_REDIS_MAIN_PORT', '6379'),
-                              'password': os.environ.get('ENKETO_REDIS_MAIN_PASSWORD', None),
-                             }
-
-    config['redis']['cache']= {'host': os.environ.get('ENKETO_REDIS_CACHE_HOST', 'redis_cache'),
-                               'port': os.environ.get('ENKETO_REDIS_CACHE_PORT', '6379'),
-                               'password': os.environ.get('ENKETO_REDIS_CACHE_PASSWORD', None),
-                              }
 
     CONFIG_FILE_PATH= os.path.join(PROJECT_ROOT_PATH, 'config/config.json')
+    if not os.path.isfile(CONFIG_FILE_PATH):
+        raise EnvironmentError('No Enketo Express configuration found at `{}`.'.format(CONFIG_FILE_PATH))
+    else:
+        try:
+            with open(CONFIG_FILE_PATH, 'r') as config_file:
+                config= json.loads(config_file.read())
+        except:
+            raise ValueError('Could not parse JSON content from `{}`.').format(CONFIG_FILE_PATH)
+
+    # Ensure an API key was set, retrieving it from the environment as a fallback.
+    config.setdefault('linked form and data server', dict()).setdefault('api key', os.environ.get('ENKETO_API_KEY'))
+    if not config['linked form and data server']['api key']:
+        raise EnvironmentError('An API key for Enketo Express is required.')
+
+    # Retrieve/generate the encryption key if not present.
+    config['linked form and data server'].setdefault('encryption key', get_or_create_encryption_key())
+
+    # Set the Docker Redis settings.
+    config.setdefault('redis', dict()).setdefault('main', dict()).setdefault('host', 'redis_main')
+    config['redis'].setdefault('cache', dict()).setdefault('host', 'redis_cache')
+    config['redis']['cache'].setdefault('port', '6379')
+
+    # Write the potentially-updated config file to disk.
     with open(CONFIG_FILE_PATH, 'w') as config_file:
         config_file.write(json.dumps(config, indent=4))
 

setup/docker/docker-compose.yml

@@ -0,0 +1,43 @@
+nginx:
+  image: nginx
+  links:
+    - enketo_express
+  ports:
+    - "80:80"
+    - "443:443"
+  env_file:
+    - ./envfile.txt
+  volumes:
+    - ./nginx/:/tmp/enketo_express_nginx/:ro
+    - ./secrets/:/tmp/enketo_express_secrets/:ro
+    - ../../config/:/srv/src/enketo_express/config/
+  command: bash /tmp/enketo_express_nginx/nginx_command.bash
+  restart: always
+
+enketo_express:
+  # image: kobotoolbox/enketo_express
+  build: ../..
+  links:
+    - redis_main
+    - redis_cache
+  env_file:
+    - ./envfile.txt
+  volumes:
+    - ../../config/:/srv/src/enketo_express/config
+    - ./secrets/:/srv/src/enketo_express/setup/docker/secrets/
+  restart: always
+
+redis_main:
+  image: redis:2.6
+  # Map our "main" Redis config into the container.
+  volumes:
+    - ../redis/conf/redis-enketo-main.conf:/etc/redis/redis.conf:ro
+    - ./redis_main_data/:/data/
+  restart: always
+
+redis_cache:
+  image: redis:2.6
+  # Map our "cache" Redis config into the container.
+  volumes:
+    - ../redis/conf/redis-enketo-cache.conf:/etc/redis/redis.conf:ro
+  restart: always

setup/docker/envfile.txt

@@ -1,15 +1,6 @@
-# Necessary variable.
-ENKETO_API_KEY=
-
-# Optional variables (and their default values).
-#ENKETO_OFFLINE_SURVEYS=True
-#ENKETO_FORM_DATA_SERVER_URL=kobocat
-#ENKETO_REDIS_MAIN_HOST=redis_main
-#ENKETO_REDIS_MAIN_PORT=6379
-#ENKETO_REDIS_MAIN_PASSWORD=
-#ENKETO_REDIS_CACHE_HOST=redis_cache
-#ENKETO_REDIS_CACHE_PORT=6379
-#ENKETO_REDIS_CACHE_PASSWORD=
+# Required, if not specified in `config/config.json`.
+#ENKETO_API_KEY=
 
 # Autogenerated if missing.
 #ENKETO_ENCRYPTION_KEY=
+

setup/docker/nginx/enketo_express_location.conf.tmpl

@@ -0,0 +1,13 @@
+# `envsubst` template.
+# Context: server
+
+# When using a root URI prefix, redirect requests to the new root that aren't terminated with a '/'.
+${INCLUDE_ROOT_URI_REDIRECT}
+
+location ${ROOT_URI} {
+    include /tmp/enketo_express_nginx/enketo_express_proxy_pass.conf;
+
+    # FIXME: Remove once Enketo Express is relocatable.
+    # Optionally include rules for rewriting root-relative references in the response.
+    ${INCLUDE_REWRITE_RULES}
+}

setup/docker/nginx/enketo_express_location_uri_prefix.conf.tmpl

@@ -0,0 +1,7 @@
+# `envsubst` template.
+# Context: server
+
+# Redirect requests not terminated with a '\'.
+location /${ENKETO_EXPRESS_URI_PREFIX} {
+    return 301 $scheme://$http_host/${ENKETO_EXPRESS_URI_PREFIX}/;
+}

setup/docker/nginx/enketo_express_proxy_pass.conf

@@ -0,0 +1,8 @@
+# Context: location
+
+proxy_pass  http://enketo_express:8005/;
+proxy_redirect off;
+proxy_set_header Host $http_host;
+proxy_set_header X-Real-IP $remote_addr ;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for ;
+proxy_set_header X-Forwarded-Proto https ;

setup/docker/nginx/enketo_express_rewrite_response_reference_rules.conf.tmpl

@@ -0,0 +1,14 @@
+# `envsubst` template.
+# Context: location
+
+# Rewrite root-relative references in the response.
+proxy_set_header Accept-Encoding ""; # No GZip.
+sub_filter_once off;
+sub_filter_types text/html text/css application/javascript application/json;
+sub_filter 'href="/'            'href="/${ENKETO_EXPRESS_URI_PREFIX}/';
+sub_filter 'src="/'             'src="/${ENKETO_EXPRESS_URI_PREFIX}/';
+sub_filter 'src: url("/'        'src: url("/${ENKETO_EXPRESS_URI_PREFIX}/';
+sub_filter 'url:"/'             'url:"/${ENKETO_EXPRESS_URI_PREFIX}/';
+sub_filter '="/transform/xform' '="/${ENKETO_EXPRESS_URI_PREFIX}/transform/xform';
+sub_filter '="/connection'      '="/${ENKETO_EXPRESS_URI_PREFIX}/connection';
+sub_filter '/_/#'               '/${ENKETO_EXPRESS_URI_PREFIX}/_/#';

setup/docker/nginx/enketo_express_site_http.conf

@@ -0,0 +1,7 @@
+# Context: http
+
+server {
+    listen       80;
+
+    include /tmp/nginx_templates_enabled/enketo_express_location.conf;
+}