docker-compose.yml

x-airflow-common: &airflow-common
  build:
    context: ./docker/airflow
    dockerfile: airflow.Dockerfile
  env_file:
    - .env
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW__LOGGING__LOGGING_LEVEL: 'WARNING'
    AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC: 30
    AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME: 5
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth'
    GE_JUPYTER_CMD: "jupyter lab --allow-root --ip 0.0.0.0 --port 18888"
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/dag_generated_configs:/opt/airflow/dag_generated_configs
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    - ./airflow/data_raw:/opt/airflow/data_raw
    - ./airflow/backups:/opt/airflow/backups
    - ./airflow/dbt:/opt/airflow/dbt
    - ./airflow/great_expectations:/opt/airflow/great_expectations
    - ./airflow/tests:/opt/airflow/tests
    - ./airflow/pytest.ini:/opt/airflow/pytest.ini
    - ./profiles.yml:/home/airflow/.dbt/profiles.yml
  user: "${AIRFLOW_UID:-50000}:0"
  networks:
    - re_etl_net
  depends_on: &airflow-common-depends-on
    airflow_db:
      condition: service_healthy

x-superset-common: &superset-common
  build:
    context: ./docker/superset
    dockerfile: superset.Dockerfile
  networks:
    - re_etl_net
  env_file:
    - .env.superset
  environment:
    SUPERSET_LOAD_EXAMPLES: false
    REDIS_HOST: redis
    REDIS_PORT: 6379

  depends_on: &superset-common-depends-on
    redis:
      condition: service_healthy
    ss_db:
      condition: service_healthy

services:
  om_db:
    image: adwh_om_db:17.0
    build:
      context: ./docker/openmetadata
      dockerfile: postgres.Dockerfile
    command: "--work_mem=10MB"
    env_file:
      - .env.om_db
    networks:
      - re_etl_net
    ports:
      - "45432:5432"
    volumes:
      - re_om_db:/var/lib/postgresql/data/
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "pg_isready -h ss_db -d $${POSTGRES_DB} -U $${POSTGRES_USER}"
        ]
      interval: 5s
      retries: 5
    restart: always

  om_elastic:
    image: adwh_om_elasticsearch:8.15.2
    build:
      context: ./docker/elastic
      dockerfile: elastic.Dockerfile
    env_file:
      - .env.elastic
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
      - xpack.security.enabled=false
      - bootstrap.memory_lock=true
      - logger.level=INFO
    networks:
      - re_etl_net
    ports:
      - "9200:9200"
      - "9300:9300"
    healthcheck:
      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 10
    volumes:
      - re_om_elastic:/usr/share/elasticsearch/data

  om_init:
    image: adwh_om_server:1.5.8
    build:
      context: ./docker/openmetadata
      dockerfile: om_server.Dockerfile
    command: "./bootstrap/openmetadata-ops.sh migrate"
    env_file:
      - .env.om_db
      - .env.elastic
      - .env.om_server
    networks:
      - re_etl_net
    depends_on:
      om_elastic:
        condition: service_healthy
      om_db:
        condition: service_healthy

  om_server:
    image: adwh_om_server:1.5.8
    build:
      context: ./docker/openmetadata
      dockerfile: om_server.Dockerfile
    env_file:
      - .env.om_db
      - .env.elastic
      - .env.om_server
    networks:
      - re_etl_net
    expose:
      - 8585
      - 8586
    ports:
      - "8585:8585"
      - "8586:8586"
    depends_on:
      om_elastic:
        condition: service_healthy
      om_db:
        condition: service_healthy
      om_init:
        condition: service_completed_successfully
    volumes:
      - ./config/public_key.der:/opt/openmetadata/conf/public_key.der:ro
      - ./config/private_key.der:/opt/openmetadata/conf/private_key.der:ro
    healthcheck:
      test: [ "CMD", "wget", "-q", "--spider",  "http://localhost:8586/healthcheck" ]
    restart: always

  superset:
    <<: *superset-common
    image: adwh_superset:2.1.0
    command: [ "/app/docker/docker-bootstrap.sh", "app-gunicorn" ]
    restart: unless-stopped
    ports:
      - "8088:8088"
    depends_on:
      <<: *superset-common-depends-on
      ss_init:
        condition: service_completed_successfully
    volumes:
      - ss_core_vol:/app/superset_home

  ss_db:
    image: adwh_ss_db:14.5
    build:
      context: ./docker/superset
      dockerfile: postgres.Dockerfile
    env_file:
      - .env.superset
    networks:
      - re_etl_net
    ports:
      - "5442:5432"
    volumes:
      - re_ss_db_vol:/var/lib/postgresql/data/
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "pg_isready -h ss_db -d $${POSTGRES_DB} -U $${POSTGRES_USER}"
        ]
      interval: 5s
      retries: 5
    restart: always

  ss_init:
    <<: *superset-common
    command: [ "/app/docker/docker-init.sh" ]
    depends_on:
      - redis
      - ss_db
    volumes:
      - ss_core_vol:/app/superset_home
    healthcheck:
      disable: true

  redis:
    image: adwh_redis:7.0.8
    build:
      context: ./docker/redis
      dockerfile: redis.Dockerfile
    ports:
      - "6379:6379"
    volumes:
      - re_redis_vol:/data
    networks:
      - re_etl_net
    restart: unless-stopped
    healthcheck:
      test: [ "CMD", "redis-cli", "ping" ]
      interval: 5s
      timeout: 30s
      retries: 50

  airflow_db:
    image: adwh_airflow_pg_db:14.5
    build:
      context: ./docker/airflow
      dockerfile: airflow_db.Dockerfile
    shm_size: 4gb
    env_file:
      - .env
    networks:
      - re_etl_net
    ports:
      - "5432:5432"
    volumes:
      - re_airflow_db:/var/lib/postgresql/data/
    healthcheck:
      test:
        [
          "CMD",
          "pg_isready",
          "-d",
          "${POSTGRES_DB}",
          "-U",
          "${POSTGRES_USER}"
        ]
      interval: 5s
      retries: 5
    restart: always

  dwh_db:
    image: adwh_dwh_postgis_db:15.3.3
    build:
      context: ./docker/postgis
      dockerfile: postgis.Dockerfile
    shm_size: 4gb
    env_file:
      - .env.dwh
    networks:
      - re_etl_net
    ports:
      - "5431:5432"
    volumes:
      - re_dwh_db:/var/lib/postgresql/data/
      - ./docker/postgis/postgresql.conf:/var/lib/postgresql/data/postgresql.conf
    healthcheck:
      test:
        [
          "CMD",
          "pg_isready",
          "-h",
          "dwh_db",
          "-d",
          "${POSTGRES_DB}",
          "-U",
          "${POSTGRES_USER}"
        ]
      interval: 5s
      retries: 5
    restart: always

  db_admin:
    image: adwh_pgadmin:7.7
    build:
      context: ./docker/pgadmin
      dockerfile: pgadmin4.Dockerfile
    env_file:
      - .env
    restart: always
    depends_on:
      - dwh_db
    networks:
      - re_etl_net
    ports:
      - "5678:80"
    volumes:
      - re_etl_admin:/var/lib/pgadmin

  dbt_proj:
    image: adwh_dbt:1.6.3
    build:
      context: ./docker/dbt
      dockerfile: dbt.Dockerfile
    env_file:
      - .env
    stdin_open: true
    tty: true
    volumes:
      - ./airflow/dbt:/usr/app
      - ./profiles.yml:/root/.dbt/profiles.yml
    networks:
      - re_etl_net
    ports:
      - "18080:18080"
    depends_on:
      - dwh_db

  airflow-webserver:
    <<: *airflow-common
    image: adwh_airflow_webserver:2.9.3
    command: webserver
    ports:
      - "8080:8080"
    healthcheck:
      test:
        [
          "CMD",
          "curl",
          "--fail",
          "http://localhost:8080/health"
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    image: adwh_airflow_scheduler:2.9.3
    command: scheduler
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    ports:
      - "18888:18888"
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-triggerer:
    <<: *airflow-common
    image: adwh_airflow_triggerer:2.9.3
    command: triggerer
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    image: adwh_airflow_init:2.9.3
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - -c
      - |
        function ver() {
          printf "%04d%04d%04d%04d" $${1//./ }
        }
        mkdir -p /sources/logs /sources/dags /sources/plugins /sources/data_raw /sources/dbt
        mkdir -p /sources/great_expectations /sources/backups
        chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins,data_raw,dbt}
        chown -R "${AIRFLOW_UID}:0" /sources/{great_expectations,backups}
        exec /entrypoint airflow version
    # yamllint enable rule:line-length
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
      _PIP_ADDITIONAL_REQUIREMENTS: ''
    user: "0:0"
    volumes:
      - ./airflow:/sources

  airflow-cli:
    <<: *airflow-common
    image: adwh_airflow_cli:2.9.3
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

networks:
  re_etl_net:


volumes:
  re_airflow_db:
  re_etl_admin:
  re_dwh_db:
  re_ss_db_vol:
  re_redis_vol:
  re_om_db:
  re_om_elastic:
  ss_core_vol: