feat!: Use Dockerfile based on upstream [DSRE-1116] (#1638)

* CI runtime decreased by 70-80%

* docker-compose setup time decreased significantly. **Local use: takes a few seconds as opposed to a few minutes before**. Airflow Variables and Connections are loaded via `import` CLI command; replaces `bin/run` script using Airflow CLI.

* Improved dev secrets security by dynamically generating a Fernet key in makefile
  * `.env` file is generated from `make up`, contains UID and Fernet key. `.env` is automatically loaded into environment variables by `docker-compose`

* Retire shell scripts to use builtin features in our stack
  * `bin/run` replaced by docker-compose and CI
  * `bin/test-dag-tags` replaced by pytest unit tests
  * `bin/test-parse` replaced by pytest unit tests

Author: mikaeld

Diff for: .circleci/config.yml

@@ -16,43 +16,30 @@ orbs:
 
 jobs:
   unit-tests:
-    executor:
+    executor: &python-executor
       name: python/default
       tag: 3.8.12
     steps:
       - checkout
       - python/install-packages:
           pip-dependency-file: requirements.txt
           pkg-manager: pip
-      - run:
-          name: 🧪 Verify requirements.txt is sync'ed with requirements.in
-          command: |
-            pip install pip-tools
-            pip-compile --quiet
-            git diff --exit-code requirements.txt
       - run:
           name: 🧪 Pytest
-          command: python -m pytest
-      - run:
-          name: 🧪 Compile DAGs
-          command: python -m py_compile dags/*.py
-      - run:
-          name: 🧹 Clean up
-          command: find . -name *.pyc -delete
+          command: python -m pytest --junitxml=test-results/junit.xml
+      - store_test_results:
+          path: test-results
 
-  integration-tests:
-    executor:
-      name: docker/machine
-      dlc: true
+  validate-requirements:
+    executor: *python-executor
     steps:
       - checkout
-      - run: docker-compose pull
-      - run: docker-compose build
-      - run: sudo chown -R 10001:10001 .
       - run:
-          name: Check DAGs can be parsed and are correctly tagged
-          # Valid DAG tags are defined in: `bin/test-dag-tags.py`
-          command: bash bin/test-parse
+          name: 🧪 Verify requirements.txt is sync'ed with requirements.in
+          command: |
+            pip install pip-tools
+            pip-compile --quiet
+            git diff --exit-code requirements.txt
 
   sync_gcs:
     docker:
@@ -72,7 +59,7 @@ workflows:
   ci:
     jobs:
       - docker/publish:
-          name: Docker build test
+          name: 🛠️ Docker build test
           before_build: &version
             - run:
                 name: Generate build version.json
@@ -95,13 +82,14 @@ workflows:
               ignore: /.*/
 
       - unit-tests:
-          name: Unit tests
+          name: 🧪 Unit tests
           filters: *ci-filter
 
-      - integration-tests:
-          name: Integration tests
+      - validate-requirements:
+          name: 🧪 Validate requirements
           filters: *ci-filter
 
+
   publish:
     jobs:
       - docker/publish:

Diff for: .dockerignore

@@ -14,14 +14,15 @@ __pycache__/
 
 # Airflow stuff
 logs/
-{AIRFLOW_HOME}/
-examples/
 
 # Virtual environment
 .env/
 .venv/
 venv/
 
+# Airflow dev resources
+resources/
+
 # IDE
 .idea
 .vscode/

Diff for: Dockerfile

@@ -1,79 +1,36 @@
-# Use buster image because the default bullseye image has updated coreutils that require a newer
-# linux kernel than provided by CircleCI, per
-# https://forums.docker.com/t/multiple-projects-stopped-building-on-docker-hub-operation-not-permitted/92570/6
-# and https://forums.docker.com/t/multiple-projects-stopped-building-on-docker-hub-operation-not-permitted/92570/11
-FROM python:3.8.12-slim-buster
-MAINTAINER Harold Woo <[email protected]>
+FROM apache/airflow:slim-2.3.3-python3.8
 
-ARG AIRFLOW_UID=10001
-ARG AIRFLOW_GID=10001
-ARG PROJECT_DIR="/app"
+ARG PROJECT_DIR="/opt/airflow"
 
-# add a non-privileged user for installing and running the application
-RUN mkdir $PROJECT_DIR && \
-    chown $AIRFLOW_UID:$AIRFLOW_GID $PROJECT_DIR && \
-    groupadd --gid $AIRFLOW_GID app && \
-    useradd --no-create-home --uid $AIRFLOW_UID --gid $AIRFLOW_GID --home-dir $PROJECT_DIR app
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH="$PYTHONPATH:$PROJECT_DIR"
+ENV AIRFLOW_HOME=$PROJECT_DIR
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        apt-transport-https build-essential curl git libpq-dev python-dev \
-        default-libmysqlclient-dev gettext sqlite3 libffi-dev \
-        lsb-release gnupg vim screen procps default-mysql-client && \
+USER root
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends build-essential default-libmysqlclient-dev
+
+# Legacy docker image dependencies to be reviewed
+RUN apt-get install -y --no-install-recommends \
+    lsb-release gnupg curl && \
     CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
     echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
     apt-get update -y && apt-get install google-cloud-sdk -y && apt-get install google-cloud-sdk-gke-gcloud-auth-plugin && \
-    apt-get remove -y lsb-release gnupg && \
-    apt-get autoremove -y && \
+    apt-get remove -y lsb-release gnupg
+
+RUN apt-get autoremove -yqq --purge && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    rm -rf /var/lib/apt/lists/*
 
-# Install Python dependencies
-COPY requirements.txt /tmp/
-# Switch to /tmp to install dependencies outside home dir
-WORKDIR /tmp
+USER airflow
 
-RUN pip install --upgrade pip
-RUN export SLUGIFY_USES_TEXT_UNIDECODE=yes && pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt /
+RUN pip install --no-cache-dir -r /requirements.txt
 
-# Switch back to home directory
 WORKDIR $PROJECT_DIR
 
-COPY . $PROJECT_DIR
-
-RUN chown -R $AIRFLOW_UID:$AIRFLOW_GID $PROJECT_DIR
-
-USER $AIRFLOW_UID
-
-ENV PYTHONUNBUFFERED=1 \
-    PORT=8000\
-    PYTHONPATH="$PYTHONPATH:$PROJECT_DIR"
-    # AWS_ACCESS_KEY_ID= \
-    # AWS_SECRET_ACCESS_KEY= \
-    # DEPLOY_ENVIRONMENT =
-
-ENV AIRFLOW_HOME=$PROJECT_DIR \
-    AIRFLOW_EMAIL_BACKEND="airflow.utils.email.send_email_smtp"
-    # AIRFLOW_AUTHENTICATE= \
-    # AIRFLOW_AUTH_BACKEND= \
-    # AIRFLOW_BROKER_URL= \
-    # AIRFLOW_RESULT_URL= \
-    # AIRFLOW_FLOWER_PORT= \
-    # AIRFLOW_DATABASE_URL= \
-    # AIRFLOW__CORE__FERNET_KEY= \
-    # AIRFLOW_SECRET_KEY= \
-    # AIRFLOW_SMTP_HOST= \
-    # AIRFLOW_SMTP_USER= \
-    # AIRFLOW_SMTP_PASSWORD= \
-    # AIRFLOW_SMTP_FROM= \
-
-
-EXPOSE $PORT
-
-# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
-# where volume-mounted files do not have execute bits set.
-# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
-ENTRYPOINT ["/bin/bash", "/app/bin/run"]
+# deploylib expects /app/version.json, copy the file if it exists
+COPY *version.json /app/version.json
 
-CMD ["web"]
+COPY . .

Diff for: Makefile

@@ -1,32 +1,27 @@
-.PHONY: build clean migrate redis-cli run secret shell stop up
+.PHONY: build clean redis-cli run shell stop up
 
 
 help:
 	@echo "Welcome to the Telemetry Airflow\n"
 	@echo "The list of commands for local development:\n"
 	@echo "  build      		Builds the docker images for the docker-compose setup"
-	@echo "  build-linux		Builds the docker images using the current user ID and group ID"
 	@echo "  clean      		Stops and removes all docker containers"
 	@echo "  pip-compile      	Compile dependencies from 'requirements.in' into 'requirements.txt'"
 	@echo "  pip-install-local	Install pip project requirements to your local environment"
-	@echo "  migrate    		Runs the Django database migrations"
 	@echo "  redis-cli  		Opens a Redis CLI"
 	@echo "  shell      		Opens a Bash shell"
-	@echo "  up         		Runs the whole stack, served under http://localhost:8000/"
+	@echo "  up         		Runs the whole stack, served under http://localhost:8080/"
 	@echo "  gke        		Create a sandbox gke cluster for testing"
 	@echo "  clean-gke  		Delete the sandbox gke cluster"
 	@echo "  stop       		Stops the docker containers"
 
 build:
 	docker-compose build
 
-build-linux:
-	docker-compose build --build-arg AIRFLOW_UID="$$(id -u)" --build-arg AIRFLOW_GID="$$(id -g)"
-
 pip-compile:
 	pip-compile
 
-clean:	stop
+clean: stop
 	docker-compose rm -f
 	rm -rf logs/*
 	if [ -f airflow-worker.pid ]; then rm airflow-worker.pid; fi
@@ -35,7 +30,7 @@ pip-install-local: pip-compile
 	pip install -r requirements.txt
 
 shell:
-	docker-compose run web bash
+	docker-compose run airflow-webserver bash
 
 redis-cli:
 	docker-compose run redis redis-cli -h redis
@@ -45,7 +40,11 @@ stop:
 	docker-compose stop
 
 up:
-	docker-compose up
+	grep -qF 'AIRFLOW_UID=' .env || echo "AIRFLOW_UID=$$(id -u)" >> .env
+	grep -qF 'FERNET_KEY=' .env || echo "FERNET_KEY=$$(python -c "from cryptography.fernet import Fernet; fernet_key = Fernet.generate_key(); print(fernet_key.decode())")" >> .env
+	docker-compose up --wait
+	docker-compose exec airflow-webserver airflow variables import dev_variables.json
+	docker-compose exec airflow-webserver airflow connections import dev_connections.json
 
 gke:
 	bin/start_gke