docker-compose.yaml

x-common-config: &common-config
  init: true
  restart: unless-stopped

x-dagster-common-config: &dagster-common-config
  <<: *common-config
  env_file: ./dagster/.env
  build: ./dagster
  image: unicef/giga-dataops-platform/dagster
  restart: unless-stopped
  environment:
    DAGSTER_HOME: /app
    COMMIT_SHA: ${COMMIT_SHA}
    DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS: 300

x-spark-common-config: &spark-common-config
  <<: *common-config
  user: "1001:1001"
  image: unicef/giga-dataops-platform/spark
  env_file: ./dagster/.env

x-spark-worker-common-config: &spark-worker-common-config
  <<: *spark-common-config
  build:
    context: .
    dockerfile: spark/Dockerfile
  working_dir: /opt/bitnami/spark/app
  volumes:
    - ./dagster:/opt/bitnami/spark/app

x-spark-worker-common-env: &spark-worker-common-env
  SPARK_MODE: worker
  SPARK_MASTER_URL: spark://spark-master:7077
  SPARK_WORKER_HOST: 0.0.0.0
  SPARK_WORKER_CORES: ${SPARK_WORKER_CORES:-2}
  SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY:-2731M}
  SPARK_PUBLIC_DNS: localhost
  PYTHONPATH: /opt/bitnami/spark/app

volumes:
  dagster-storage:
  hms-db-storage:

networks:
  default:
    name: giga-dataops
    external: true

services:
  dagster-storage:
    <<: *common-config
    image: bitnami/postgresql:14.10.0-debian-11-r31
    env_file: ./dagster/.env
    volumes:
      - dagster-storage:/bitnami/postgresql

  dagster-webserver:
    <<: *dagster-common-config
    volumes:
      - ./dagster:/app
      - ./dagster/.tmp/warehouse:/opt/spark/warehouse
      - ./dagster/.venv-docker:/app/.venv
    depends_on:
      - dagster-storage
    ports:
      - "3001:3002"

  spark-master:
    <<: *spark-common-config
    build:
      context: .
      dockerfile: spark/Dockerfile
    environment:
      SPARK_MODE: master
      SPARK_MASTER_HOST: 0.0.0.0
      SPARK_MASTER_PORT: 7077
      SPARK_MASTER_WEBUI_PORT: 8070
      SPARK_PUBLIC_DNS: localhost
      THRIFT_PORT: 10000
    ports:
      - "7077:7077"
      - "8070:8070"
      - "10000:10000"

  spark-worker-1:
    <<: [ *spark-common-config, *spark-worker-common-config ]
    environment:
      <<: *spark-worker-common-env
      SPARK_WORKER_WEBUI_PORT: 8071
    ports:
      - "8071:8071"

  spark-worker-2:
    <<: [ *spark-common-config, *spark-worker-common-config ]
    environment:
      <<: *spark-worker-common-env
      SPARK_WORKER_WEBUI_PORT: 8072
    ports:
      - "8072:8072"

  spark-worker-3:
    <<: [ *spark-common-config, *spark-worker-common-config ]
    environment:
      <<: *spark-worker-common-env
      SPARK_WORKER_WEBUI_PORT: 8073
    ports:
      - "8073:8073"

  hive-metastore:
    <<: *common-config
    build: ./hive
    image: unicef/giga-dataops-platform/hive-metastore
    env_file: ./hive/.env
    environment:
      DB_DRIVER: postgres
      SERVICE_NAME: metastore
      METASTORE_HOME: /opt/hive
    volumes:
      - ./hive/hive-site.template.xml:/opt/hive/tpl/hive-site.template.xml
      - ./hive/metastore-site.template.xml:/opt/hive/tpl/metastore-site.template.xml
      - ./hive/hms-entrypoint.sh:/opt/hive/bin/hms-entrypoint.sh
    ports:
      - "9083:9083"
    depends_on:
      hms-database:
        condition: service_healthy

  hms-database:
    <<: *common-config
    image: bitnami/postgresql:9.6.24-debian-10-r70
    env_file: ./hive/.env
    environment:
      POSTGRESQL_USERNAME: ${HMS_POSTGRESQL_USERNAME}
      POSTGRESQL_PASSWORD: ${HMS_POSTGRESQL_PASSWORD}
      POSTGRESQL_DATABASE: ${HMS_POSTGRESQL_DATABASE}
    volumes:
      - hms-db-storage:/bitnami/postgresql
    healthcheck:
      test: [ "CMD", "pg_isready", "-U", "metastore" ]
      timeout: 3s
      start_period: 5s
      retries: 5

#  authproxy:
#    <<: *common-config
#    image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
#    env_file: ./oauth2-proxy/.env
#    environment:
#      OAUTH2_PROXY_PROVIDER: azure
#      OAUTH2_PROXY_INSECURE_OIDC_SKIP_ISSUER_VERIFICATION: true
#      OAUTH2_PROXY_SCOPE: "openid profile email"
#      OAUTH2_PROXY_AZURE_TENANT: ${AZURE_TENANT_ID}
#      OAUTH2_PROXY_OIDC_EMAIL_CLAIM: preferred_username
#      OAUTH2_PROXY_OIDC_ISSUER_URL: https://login.microsoftonline.com/common/v2.0
#      OAUTH2_PROXY_CLIENT_ID: ${AZURE_CLIENT_ID}
#      OAUTH2_PROXY_CLIENT_SECRET: ${AZURE_CLIENT_SECRET}
#      OAUTH2_PROXY_CODE_CHALLENGE_METHOD: S256
#      OAUTH2_PROXY_PROXY_PREFIX: /auth
#      OAUTH2_PROXY_REDIRECT_URL: http://localhost:3001/auth/callback
#      OAUTH2_PROXY_HTTP_ADDRESS: 0.0.0.0:3001
#      OAUTH2_PROXY_UPSTREAMS: "http://dagster-webserver:3002,http://spark-master:8070/spark"
#      OAUTH2_PROXY_EMAIL_DOMAINS: "thinkingmachin.es,unicef.org"
#      OAUTH2_PROXY_API_ROUTES: "^/graphql,^/favicon.ico"
#      OAUTH2_PROXY_PROXY_WEBSOCKETS: true
#      OAUTH2_PROXY_COOKIE_NAME: _oauth2_proxy
#      OAUTH2_PROXY_COOKIE_EXPIRE: 168h
#      OAUTH2_PROXY_COOKIE_SECURE: false
#      OAUTH2_PROXY_COOKIE_HTTPONLY: false
#      OAUTH2_PROXY_CUSTOM_SIGN_IN_LOGO: /app/static/dagster.svg
#      OAUTH2_PROXY_SHOW_DEBUG_ON_ERROR: true
#    volumes:
#      - ./oauth2-proxy/dagster.svg:/app/static/dagster.svg
#    ports:
#      - "3001:3001"