docker-compose.yml

version: "3.9"

services:
  # This service runs the postgres DB used by dagster for run storage, schedule storage,
  # and event log storage.
  dagster_postgresql:
    image: postgres:11
    container_name: dagster_postgresql
    environment:
      POSTGRES_USER: "postgres_user"
      POSTGRES_PASSWORD: "postgres_password"
      POSTGRES_DB: "postgres_db"
    networks:
      - dagster_network

  # This service runs the gRPC server that loads your user code, in both dagster-webserver
  # and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
  # run launcher to use this same image when launching runs in a new container as well.
  # Multiple containers like this can be deployed separately - each just needs to run on
  # its own port, and have its own entry in the workspace.yaml file that's loaded by the
      # webserver.
  pipeline_x:
    image: pipeline-x
    restart: always
    environment:
      DAGSTER_POSTGRES_USER: "postgres_user"
      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
      DAGSTER_POSTGRES_DB: "postgres_db"
      DAGSTER_CURRENT_IMAGE: "pipeline-x"
      AWS_ACCESS_KEY_ID: "yourAWSKeyID"
      AWS_SECRET_ACCESS_KEY: "yourAWSSecretAccessKey"
      DATABASE_IP: "192.168.136.166"
      DATABASE_PORT: 5432
      DATABASE_USER: "etl_user"
      DATABASE_PASSWORD: "etl_password"
      DATABASE_NAME: "etl_db"
    networks:
      - dagster_network
    volumes:
      - /opt/dagster/dagster_home:/tmp/dagster-data

  pipeline_y:
    image: pipeline-y
    restart: always
    environment:
      DAGSTER_POSTGRES_USER: "postgres_user"
      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
      DAGSTER_POSTGRES_DB: "postgres_db"
      DAGSTER_CURRENT_IMAGE: "pipeline-y"
      AWS_ACCESS_KEY_ID: "yourAWSKeyID"
      AWS_SECRET_ACCESS_KEY: "yourAWSSecretAccessKey"
      DATABASE_IP: "192.168.136.166"
      DATABASE_PORT: 5432
      DATABASE_USER: "etl_user"
      DATABASE_PASSWORD: "etl_password"
      DATABASE_NAME: "etl_db"
    networks:
      - dagster_network

  # This service runs dagster-webserver, which loads your user code from the user code container.
  # Since our instance uses the QueuedRunCoordinator, any runs submitted from the webserver will be put on
  # a queue and later dequeued and launched by dagster-daemon.
  dagster_webserver:
    image: dagster
    entrypoint:
      - dagster-webserver
      - -h
      - "0.0.0.0"
      - -p
      - "3000"
      - -w
      - workspace.yaml
    container_name: dagster_webserver
    expose:
      - "3000"
    ports:
      - "3000:3000"
    environment:
      DAGSTER_POSTGRES_USER: "postgres_user"
      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
      DAGSTER_POSTGRES_DB: "postgres_db"
    volumes: # Make docker client accessible, so we can terminate containers from the webserver
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - dagster_network
    depends_on:
      - dagster_postgresql
      - pipeline_x
      - pipeline_y

  # This service runs the dagster-daemon process, which is responsible for taking runs
  # off of the queue and launching them, as well as creating runs from schedules or sensors.
  dagster_daemon:
    image: dagster
    entrypoint:
      - dagster-daemon
      - run
    container_name: dagster_daemon
    restart: on-failure
    environment:
      DAGSTER_POSTGRES_USER: "postgres_user"
      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
      DAGSTER_POSTGRES_DB: "postgres_db"
      AWS_ACCESS_KEY_ID: "yourAWSKeyID"
      AWS_SECRET_ACCESS_KEY: "yourAWSSecretAccessKey"
      DATABASE_IP: "192.168.136.166"
      DATABASE_PORT: 5432
      DATABASE_USER: "etl_user"
      DATABASE_PASSWORD: "etl_password"
      DATABASE_NAME: "etl_db"
    volumes: # Make docker client accessible, so we can launch containers using host docker
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - dagster_network
    depends_on:
      - dagster_postgresql
      - pipeline_y
      - pipeline_x

networks:
  dagster_network:
    driver: bridge
    name: dagster_network