docker-compose.yml

---
x-airflow-common:
  &airflow-common
  build:
    context: .
    dockerfile: ./build/dev/Containerfile
  environment:
    &airflow-common-env
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-db/airflow
    # THIS IS A FAKE CREDENTIAL FOR DEMONSTRATION PURPOSES
    # Generate with the following code
    # python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key())'
    AIRFLOW__CORE__FERNET_KEY: "Ly8wMU8r5K7jPy58M3GpkZbXDNyJz8HiJll3pu8DbIM="
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@airflow-db/airflow
    AIRFLOW__CELERY__BROKER_URL: sqs://user:pass@celery-broker:9324/
    AIRFLOW__WEBSERVER__INSTANCE_NAME: "${AIRFLOW__WEBSERVER__INSTANCE_NAME:-deploy-airflow-on-ecs-fargate}"
    AIRFLOW__LOGGING__LOGGING_LEVEL: DEBUG
    PYTHONPATH: "/opt/airflow/deploy_airflow_on_ecs_fargate:$PYTHONPATH"
    # Gotcha: Even though we set this to "True" in airflow.cfg, an environment variable overrides it
    AIRFLOW__CORE__LOAD_EXAMPLES: True
  volumes:
    - ./dags:/opt/airflow/dags
    - ./plugins:/opt/airflow/plugins
    - ./build/dev/airflow.cfg:/opt/airflow/airflow.cfg
    - ./deploy_airflow_on_ecs_fargate:/opt/airflow/deploy_airflow_on_ecs_fargate
    - ./scripts:/opt/airflow/scripts
  user: "50000:0"
  depends_on:
    &airflow-common-depends-on
    celery-broker:
      condition: service_started
    airflow-db:
      condition: service_healthy

services:
  airflow-db:
    image: postgres:13
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - airflow-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: [ "CMD", "pg_isready", "-U", "airflow" ]
      interval: 5s
      retries: 5
    restart: always

  # For environment parity, use backend that implements SQS interface
  # https://github.com/roribio/alpine-sqs
  celery-broker:
    image: roribio16/alpine-sqs:latest
    expose:
      - 9324
      - 9325
    restart: always

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - 8080:8080
    healthcheck:
      test:
        [
          "CMD",
          "curl",
          "--fail",
          "http://localhost:8080/health"
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  airflow-worker:
    <<: *airflow-common
    command: celery worker
    # SQS does not support worker remote control commands.
    # This means we can't use the 'ping' command as a healthcheck.
    # healthcheck:
    #   test:
    #     - "CMD-SHELL"
    #     - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
    #   interval: 35s
    #   timeout: 30s
    #   retries: 5
    environment:
      <<: *airflow-common-env
      # Required to handle warm shutdown of the celery workers properly
      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
      DUMB_INIT_SETSID: "0"
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  # AIP-40: Deferrable ("Async") Operators
  # https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=177050929
  airflow-triggerer:
    <<: *airflow-common
    command: triggerer
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  airflow-cli:
    <<: *airflow-common
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  # A service to translate statsd input into prometheus compatible metrics
  # https://hub.docker.com/r/prom/statsd-exporter
  statsd-exporter:
    image: prom/statsd-exporter
    command: "--statsd.listen-udp=:8125 --web.listen-address=:9102"
    ports:
      # You can open localhost:9102/metrics to see what is being exported
      - 9102:9102
    restart: always

  prometheus:
    image: prom/prometheus
    ports:
      - 9090:9090
    volumes:
      - ./build/dev/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-volume:/prometheus
    restart: always

volumes:
  airflow-db-volume:
  prometheus-volume: