docker-compose.sparkcluster.yml

version: '3.2'
services:

  # spark-master:
  #   build:
  #     context: ./spark
  #     dockerfile: Dockerfile
  #     args:
  #       SPARK_VERSION: "${SPARK_VERSION}"
  #       HADOOP_VERSION_SHORT: "${HADOOP_VERSION_SHORT}"
  #   volumes:
  #     - ./combinelib:/combinelib
  #     - type: volume
  #       source: hdfs
  #       target: /hdfs
  #       read_only: true
  #     - type: volume
  #       source: hadoop_binaries
  #       target: /opt/hadoop
  #       read_only: true
  #     - type: volume
  #       source: spark_binaries
  #       target: /opt/spark
  #       read_only: false # owner of dir
  #   ports:
  #     - 8080:8080
  #     - 7077:7077
  #     - 4040:4040
  #   environment:
  #     - "SPARK_HOME=/opt/spark"
  #     - "SPARK_MASTER_HOST=spark-master"
  #     - "SPARK_DRIVER_HOST=spark-master"
  #     - "SPARK_LOCAL_IP=spark-master"
  #     - "SPARK_DRIVER_CORES=1"
  #     - "SPARK_DRIVER_MEMORY=1G"
  #     - "SPARK_PUBLIC_DNS=10.5.0.1"
  #   command:  /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master
  #   networks:
  #     combinenet:
  #       ipv4_address: 10.5.0.8


  # spark-worker:
  #   build:
  #     context: ./spark
  #     dockerfile: Dockerfile
  #     args:
  #       SPARK_VERSION: "${SPARK_VERSION}"
  #       HADOOP_VERSION_SHORT: "${HADOOP_VERSION_SHORT}"
  #   volumes:
  #     - ./combinelib:/combinelib
  #     - type: volume
  #       source: hdfs
  #       target: /hdfs
  #       read_only: true
  #     - type: volume
  #       source: hadoop_binaries
  #       target: /opt/hadoop
  #       read_only: true
  #   ports:
  #     - 8081:8081
  #   environment:
  #     - "SPARK_HOME=/opt/spark"
  #     - "SPARK_MASTER=spark://spark-master:7077"
  #     - "SPARK_MASTER_HOST=spark-master"
  #     - "SPARK_DRIVER_HOST=spark-master"
  #     - "SPARK_WORKER_CORES=2"
  #     - "SPARK_WORKER_MEMORY=4G"
  #     - "SPARK_PUBLIC_DNS=10.5.0.1"
  #   command:  /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark-master:7077
  #   networks:
  #     combinenet:
  #       ipv4_address: 10.5.0.9