docker-compose.yml

version: "3.5"

services:
  metastore-db:
    image: postgres
    restart: always
    ports:
    - 5432:5432
    environment:
      POSTGRES_PASSWORD: admin
      POSTGRES_USER: admin
      POSTGRES_DB: metastore_db
    container_name: postgres

  hive-metastore:
    platform: linux/x86_64
    build: .
    image: hive-metastore:latest
    ports:
    - 9083:9083
    volumes:
    - ./scripts/entrypoint.sh:/entrypoint.sh
    - ./conf/metastore-site.xml:/opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml
    container_name: hive-metastore
    depends_on:
      - metastore-db
      - storage
      
  notebook:
    image: jupyter/pyspark-notebook:spark-3.1.2
    ports: 
    - 4040:4040
    - 8888:8888
    volumes:
    - ./notebook:/home/jovyan/work
    container_name: jupyter
    command:
      - jupyter
      - lab
      - --no-browser
      - --ip='*'
      - --NotebookApp.token='' 
      - --NotebookApp.password=''
      
  dremio:
    platform: linux/x86_64
    image: dremio/dremio-oss:19.0
    ports: 
    - 9047:9047
    - 31010:31010
    - 32010:32010
    volumes:
    - ./conf/metastore-site.xml:/opt/dremio/plugins/connectors/hive3.d/conf/core-site.xml
    - ./conf/metastore-site.xml:/opt/dremio/plugins/connectors/hive3-ee.d/conf/core-site.xml
    container_name: dremio
    environment:
    #creates a default user for the Dremio cluster. user: dremio, password: dremio123
      - DREMIO_JAVA_EXTRA_OPTS=-Ddebug.addDefaultUser=true
      - SERVER_GC_OPTS=-XX:+UseG1GC
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9047/"]
      interval: 5s
      retries: 10

  createMinioSource:
    image: curlimages/curl:7.78.0
    depends_on:
      dremio:
        condition: service_healthy
    restart: on-failure
    volumes:
    - ./scripts/firstRun.sh:/firstRun.sh
    entrypoint: >
      /bin/sh -c "/firstRun.sh
      exit 0;
      "

  storage:
    image: minio/minio
    ports:
    - 9000:9000
    - 9001:9001
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
    volumes:
    - ./minio:/data
    container_name: minio
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  createbuckets:
    image: minio/mc
    depends_on:
      storage:
        condition: service_healthy
    restart: on-failure
    entrypoint: >
      /bin/sh -c "
      /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin;
      /usr/bin/mc mb myminio/dremio;
      /usr/bin/mc policy set public myminio/dremio;
      exit 0;
      "

networks:
    default:
      name: iceberg
      driver: bridge