docker-compose.yaml

services:
  webapp:
    image: wenyixu101/webapp:latest
    container_name: webapp
    ports:
      - "5001:80"
    env_file:
      - ./webapp/.env.dev
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  server:
    image: wenyixu101/server:latest
    container_name: server
    ports:
      - "5002:5002"
    environment:
      - ENV=testing
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  notebook:
    image: wenyixu101/notebook:latest
    container_name: notebook
    ports:
      - "8888:8888"
      - "4040:4040"
    environment:
      - JUPYTER_ENABLE_LAB=no
      - ENVIRONMENT=development
      - PYSPARK_SUBMIT_ARGS=--packages io.delta:delta-spark_2.12:3.0.0 --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog pyspark-shell
    # command: ["jupyter", "notebook", "--ip='0.0.0.0'", "--port=8888", "--no-browser", "--allow-root", "--NotebookApp.token=''", "--NotebookApp.password=''"]
    volumes:
      - ./data/spark-events:/opt/data/spark-events
      - ./data/spark-warehouse:/opt/data/spark-warehouse
      - ./datasets:/opt/data/datasets
      - ./examples:/home/jovyan/work
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  spark-master:
    image: wenyixu101/spark:latest
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MODE=master
      - SPARK_MASTER_PORT=7077
      - SPARK_MASTER_WEBUI_PORT=8080
      - SPARK_HISTORY_OPTS=-Dspark.history.fs.logDirectory=/opt/data/spark-events
    command: "/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master"
    volumes:
      - ./data/spark-events:/opt/data/spark-events
      - ./data/spark-warehouse:/opt/data/spark-warehouse
      - ./datasets:/opt/data/datasets
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  spark-worker-1:
    image: wenyixu101/spark:latest
    container_name: spark-worker
    ports:
      - "8081:8081"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MODE=worker
      - SPARK_WORKER_PORT=8081
      - SPARK_WORKER_WEBUI_PORT=8081
      - SPARK_HISTORY_OPTS=-Dspark.history.fs.logDirectory=/opt/data/spark-events
    command: "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077"
    volumes:
      - ./data/spark-events:/opt/data/spark-events
      - ./data/spark-warehouse:/opt/data/spark-warehouse
      - ./datasets:/opt/data/datasets
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  # spark-worker-2:
  #   image: wenyixu101/spark:latest
  #   container_name: spark-worker-2
  #   ports:
  #     - "8082:8082"
  #   environment:
  #     - SPARK_MASTER_HOST=spark-master
  #     - SPARK_MODE=worker
  #     - SPARK_WORKER_PORT=8082
  #     - SPARK_WORKER_WEBUI_PORT=8082
  #     - SPARK_HISTORY_OPTS=-Dspark.history.fs.logDirectory=/opt/data/spark-events
  #   command: "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077"
  #   volumes:
  #     - ./data/spark-events:/opt/data/spark-events
  #     - ./data/spark-warehouse:/opt/data/spark-warehouse
  #     - ./datasets:/opt/data/datasets
  #   pull_policy: always
  #   deploy:
  #     resources:
  #       limits:
  #         cpus: "1" 
  #         memory: 1g

  history-server:
    image: wenyixu101/history-server:latest
    container_name: history-server
    ports:
      - "18080:18080"
    environment:
      - SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=file:/opt/data/spark-events"
      - SPARK_MODE=history-server
    command: "/opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer "
    volumes:
      - ./data/spark-events:/opt/data/spark-events
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  airflow-webserver:
    image: wenyixu101/airflow:latest
    container_name: airflow-webserver
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=<your_fernet_key>
    ports:
      - "8090:8080"
    command: >
      bash -c "airflow db upgrade &&
               airflow users create --username admin --firstname YourFirstName --lastname YourLastName --role Admin --email example@example.com --password yourpassword &&
               airflow webserver"
    volumes:
      - ./dags:/opt/airflow/dags
      - ./examples:/opt/airflow/examples
      - ./data/spark-events:/opt/data/spark-events
      - ./datasets:/opt/data/datasets
      - ./data/spark-warehouse:/opt/data/spark-warehouse
    pull_policy: always
    depends_on:
      - airflow-scheduler
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  airflow-scheduler:
    image: wenyixu101/airflow:latest
    container_name: airflow-scheduler
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=<your_fernet_key>
    command: scheduler
    volumes:
      - ./dags:/opt/airflow/dags
      - ./examples:/opt/airflow/examples
      - ./data/spark-events:/opt/data/spark-events
      - ./datasets:/opt/data/datasets
      - ./data/spark-warehouse:/opt/data/spark-warehouse
    pull_policy: always
    depends_on:
      - postgres
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g
  
  postgres:
    image: postgres:13
    container_name: postgres
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    ports:
      - "5432:5432"
    volumes:
      # - ./data/postgres:/var/lib/postgresql/data
      - ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
    pull_policy: always
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g

  # For reverse proxy
  nginx:
    build:
      context: ./docker/nginx
      dockerfile: Dockerfile
    container_name: nginx
    ports:
      - "18081:80"
    deploy:
      resources:
        limits:
          cpus: "1" 
          memory: 1g