Cleanup and simplify configuration/localdev

.gitignore

@@ -121,4 +121,4 @@ celerybeat-schedule
 /.pants.workdir.file_lock*
 
 # Local Docker customizations
-docker-compose.local.yml
+docker-compose.*.yml

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.6
+FROM python:3.9.6 as base
 LABEL maintainer "ODL DevOps <[email protected]>"
 
 # Add package files, install updated node and pip
@@ -13,7 +13,7 @@ RUN apt-get install -y $(grep -vE "^\s*#" apt.txt  | tr "\n" " ")
 RUN curl --silent --location https://bootstrap.pypa.io/get-pip.py | python3 -
 
 # Add, and run as, non-root user.
-RUN mkdir /src
+RUN mkdir /app
 RUN adduser --disabled-password --gecos "" mitodl
 RUN mkdir /var/media && chown -R mitodl:mitodl /var/media
 
@@ -23,16 +23,27 @@ COPY test_requirements.txt /tmp/test_requirements.txt
 RUN pip install -r requirements.txt -r test_requirements.txt
 
 # Add project
-COPY . /src
-WORKDIR /src
-RUN chown -R mitodl:mitodl /src
+COPY . /app
+WORKDIR /app
+RUN chown -R mitodl:mitodl /app
 
 RUN apt-get clean && apt-get purge
-USER mitodl
 
 # Set pip cache folder, as it is breaking pip when it is on a shared volume
 ENV XDG_CACHE_HOME /tmp/.cache
 
+FROM base as django
+
+USER mitodl
+
+FROM base as django-server
+
 EXPOSE 8013
 ENV PORT 8013
 CMD uwsgi uwsgi.ini
+
+FROM base as jupyter-notebook
+
+RUN pip install --force-reinstall jupyter
+
+USER mitodl

config/nginx.conf.erb

@@ -72,7 +72,7 @@ http {
             uwsgi_param SERVER_ADDR $server_addr;
             uwsgi_param SERVER_PORT $server_port;
             uwsgi_param SERVER_NAME $server_name;
-            uwsgi_pass unix:/tmp/nginx.socket;
+            uwsgi_pass <%= ENV["NGINX_UWSGI_PASS"] || "unix:/tmp/nginx.socket" %>;
             uwsgi_pass_request_headers on;
             uwsgi_pass_request_body on;
             uwsgi_ignore_client_abort on;

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: '3.9'
 
 x-environment:
   &py-environment
@@ -17,6 +17,7 @@ x-environment:
   DOCKER_HOST: ${DOCKER_HOST:-missing}
   WEBPACK_DEV_SERVER_HOST: ${WEBPACK_DEV_SERVER_HOST:-localhost}
   WEBPACK_DEV_SERVER_PORT: ${WEBPACK_DEV_SERVER_PORT:-8012}
+  WEBPACK_USE_DEV_SERVER: 'True'
   OPENEDX_API_KEY: ${OPENEDX_API_KEY:-PUT_YOUR_API_KEY_HERE}
 
 x-extra-hosts:
@@ -36,11 +37,19 @@ services:
       - "6379"
 
   nginx:
-    image: nginx:1.21
+    build:
+      context: ./nginx
     ports:
       - "8013:8013"
     links:
       - web
+    volumes:
+      - ./config/nginx.conf.erb:/etc/nginx/templates/nginx.conf.erb
+      - ./:/app
+    environment:
+      NGINX_WORKERS: 1
+      PORT: 8013
+      NGINX_UWSGI_PASS: "web:8011"
     networks:
       default:
         aliases:
@@ -50,6 +59,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
+      target: django-server
     command: >
       /bin/bash -c '
       sleep 3 &&
@@ -68,27 +78,34 @@ services:
     links:
       - db
       - redis
+    volumes:
+      - .:/app
+      - django_media:/var/media
     extra_hosts: *default-extra-hosts
 
   watch:
     image: node:17.8
-    working_dir: /src
+    working_dir: /app
     command: >
       /bin/bash -c './frontend/public/webpack_dev_server.sh --install'
     ports:
       - "8012:8012"
     environment:
-      NODE_ENV: 'production'
+      NODE_ENV: 'development'
       DOCKER_HOST: ${DOCKER_HOST:-missing}
       CONTAINER_NAME: 'watch'
       WEBPACK_HOST: 0.0.0.0
       WEBPACK_PORT: 8012
     env_file: .env
+    volumes:
+      - .:/app
+      - yarn-cache:/home/mitodl/.cache/yarn
 
   celery:
     build:
       context: .
       dockerfile: Dockerfile
+      target: django
     environment: *py-environment
     env_file: .env
     command: >
@@ -98,11 +115,14 @@ services:
     links:
       - db
       - redis
+    volumes:
+      - .:/app
+      - django_media:/var/media
     extra_hosts: *default-extra-hosts
 
   refine:
     image: node:17.8
-    working_dir: /src
+    working_dir: /app
     command: sh -c 'yarn install --immutable && yarn workspace mitx-online-staff-dashboard run dev'
     ports:
        - "8016:8016"
@@ -113,8 +133,28 @@ services:
       REACT_APP_OIDC_CONFIG_REDIRECT_URI: ${MITX_ONLINE_ADMIN_BASE_URL:-http://mitxonline.odl.local:8016}/oauth2/login/
       REACT_APP_MITX_ONLINE_DATASOURCE: ${MITX_ONLINE_BASE_URL:-http://mitxonline.odl.local:8013}/api
     volumes:
-      - .:/src
+      - .:/app
       - npm-cache:/root/.npm
 
+  notebook:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: jupyter-notebook
+    profiles: ["notebook"]
+    volumes:
+      - .:/app
+    environment:
+      << : *py-environment
+    env_file: .env
+    command: >
+      /bin/bash -c '
+      sleep 3 &&
+      jupyter notebook --no-browser --ip=0.0.0.0 --port=8080'
+    ports:
+      - "8080:8080"
+
 volumes:
-  npm-cache: {}
+  npm-cache: {}
+  django_media: {}
+  yarn-cache: {}

localdev/app.ipynb.example

@@ -11,7 +11,7 @@
     "import django\n",
     "from django.contrib.auth import get_user_model\n",
     "\n",
-    "ABS_PROJECT_ROOT_PATH = \"/src\"\n",
+    "ABS_PROJECT_ROOT_PATH = \"/app\"\n",
     "\n",
     "if not ABS_PROJECT_ROOT_PATH in sys.path:\n",
     "    sys.path.insert(0, ABS_PROJECT_ROOT_PATH)\n",

nginx/20-compile-nginx-conf-erb.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# Based on https://github.com/nginxinc/docker-nginx/blob/6f0396c1e06837672698bc97865ffcea9dc841d5/mainline/debian/20-envsubst-on-templates.sh
+
+set -e
+
+ME=$(basename $0)
+
+auto_erb_build() {
+  local template="/etc/nginx/templates/nginx.conf.erb"
+  local output_path="/etc/nginx/nginx.conf"
+
+  echo >&3 "$ME: Running erb on $template to $output_path"
+  erb "$template" > "$output_path"
+}
+
+auto_erb_build
+
+exit 0

nginx/Dockerfile

@@ -0,0 +1,22 @@
+# NOTE: this dockerfile is primarilty for local development only
+#       it's primary purpose is to emulate heroku-buildpack-nginx's
+#       functionality that compiles config/nginx.conf.erb
+# See https://github.com/heroku/heroku-buildpack-nginx/blob/fefac6c569f28182b3459cb8e34b8ccafc403fde/bin/start-nginx
+FROM nginx:1.21
+
+# Logs are configured to a relatic path under /etc/nginx
+# but the container expects /var/log
+RUN mkdir -p /etc/nginx/logs && ln -sf /var/log/nginx /etc/nginx/logs/
+
+# erb unfortunately needs a whole ruby install
+RUN apt-get update && apt-get install -y ruby
+
+# this gets run automatically by the nginx container's entrypoint
+COPY 20-compile-nginx-conf-erb.sh /docker-entrypoint.d
+
+# NOTE: this removes the args "-g daemon off" because nginx is impolite
+#       and treats a cli flag and a config flag as duplicate:
+#
+#       nginx: [emerg] "daemon" directive is duplicate in /etc/nginx/nginx.conf:3
+#
+CMD ["nginx"]