docker-compose.yml

version: "3.8"

services:
  spark-master:
    #    image: docker.io/bitnami/spark:3.4
    build:
      context: ./container/spark
      dockerfile: Dockerfile
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    environment:
      - SPARK_MODE=master
      - SPARK_LOCAL_IP=spark-master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - PANDERA_CACHE_DATAFRAME=True
    volumes:
      - ./container/spark/conf/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
      - .:/opt/spark-playground
    #      - ./container/spark/cn_conf/hive-site.xml:/opt/bitnami/spark/cn_conf/hive-site.xml
    networks:
      - spark-network

  spark-worker-1:
    image: docker.io/bitnami/spark:3.4
    container_name: spark-worker-1
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=6G
      - SPARK_WORKER_CORES=4
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    networks:
      - spark-network

#  spark-worker-2:
#    image: docker.io/bitnami/spark:3.4
#    container_name: spark-worker-2
#    environment:
#      - SPARK_MODE=worker
#      - SPARK_MASTER_URL=spark://spark-master:7077
#      - SPARK_WORKER_MEMORY=6G
#      - SPARK_WORKER_CORES=2
#      - SPARK_RPC_AUTHENTICATION_ENABLED=no
#      - SPARK_RPC_ENCRYPTION_ENABLED=no
#      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
#      - SPARK_SSL_ENABLED=no
#    networks:
#      - spark-network


  #  local-spark:
  #    build:
  #      context: ./container/spark
  #      dockerfile: Dockerfile
  #    container_name: local-spark
  #    ports:
  #      - "8888:8888"
  #      - "4040:4040"
  #    volumes:
  #      - ./container/spark/cn_conf/spark-defaults.cn_conf:/opt/bitnami/spark/cn_conf/spark-defaults.cn_conf
  #      - .:/opt/spark-playground
  #    networks:
  #      - spark-network

  clickhouse:
    image: bitnami/clickhouse:latest
    container_name: clickhouse
    environment:
      - CLICKHOUSE_ADMIN_USER=sdeuser
      - CLICKHOUSE_ADMIN_PASSWORD=sdepassword
    volumes:
      - ./container/clickhouse/s3_storage.xml:/opt/bitnami/clickhouse/etc/config.d/s3_storage.xml:ro
    ports:
      - "8123:8123"
      - "9002:9002"
    networks:
      - spark-network

  minio:
    hostname: minio
    image: minio/minio
    container_name: minio
    ports:
      - "9000:9000"
      - "9001:9001"
    command: [ "server", "/data", "--console-address", ":9001" ]
    volumes:
      - ~/minio/data:/data
    environment:
      - MINIO_ROOT_USER=minio
      - MINIO_ROOT_PASSWORD=minio123
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
    networks:
      - spark-network

  createbuckets:
    image: minio/mc
    container_name: createbuckets
    depends_on:
      - minio
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    entrypoint: >
      /bin/sh -c "
      until(/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo 'waiting for minio' && sleep 1; done;
      /usr/bin/mc rm -r --force minio/sparkplayground;
      /usr/bin/mc mb minio/sparkplayground;
      /usr/bin/mc policy set public minio/sparkplayground;
      /usr/bin/mc cp --recursive /data/log_content minio/sparkplayground/;
      exit 0; 
      "
    volumes:
      - ./data/log_content:/data/log_content
    networks:
      - spark-network


  metadata:
    hostname: postgres
    image: postgres:15
    container_name: metadata
    depends_on:
      - minio
    environment:
      - POSTGRES_USER=sdeuser
      - POSTGRES_PASSWORD=sdepassword
      - POSTGRES_DB=metadatadb
    ports:
      - "5432:5432"
    networks:
      - spark-network

  hive-metastore:
    hostname: metastore
    build:
      context: ./container/hive-metastore
      dockerfile: Dockerfile
    container_name: hive-metastore
    ports:
      - "9083:9083"
    volumes:
      - ./container/hive-metastore/conf/hive-site.xml:/opt/apache-hive-metastore-3.0.0-bin/conf/hive-site.xml
    environment:
      - METASTORE_DB_HOSTNAME=postgres
      - METASTORE_TYPE=postgres
    depends_on:
      - metadata
    networks:
      - spark-network

  starrocks:
    image: starrocks/allin1-ubuntu:3.2-latest
    hostname: starrocks
    container_name: starrocks
    depends_on:
      - createbuckets
      - metadata
      - hive-metastore
    ports:
      - "8030:8030"
      - "8040:8040"
      - "9030:9030"
    networks:
      - spark-network
    environment:
      - STARROCKS_ADMIN_USER=sdeuser
      - STARROCKS_ADMIN_PASSWORD=sdepassword
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1



networks:
  spark-network:
    name: sparknetwork
    driver: bridge