docker-compose.yml

version: '3'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181"
  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.1.12 #put machine's external ip or hostname
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
  database:
    image: mongo:latest
    ports:
      - "27017:27017" # Opening for external services
    volumes:
      - /mnt:/data/db
    env_file:
      - secrets.env
  stream-processor:
    build: ./SparkProcessor
    depends_on: 
      - zookeeper
    environment:
      - MASTER=spark://spark:7077
    volumes:
      - "./SparkProcessor/spark_processor.py:/spark-processor/test_script.py"
    command: ["test_script.py"]
  database-processor:
    build: ./DatabaseIngestor
    depends_on: 
      - zookeeper
      - kafka
      - database
    env_file:
      - secrets.env
    entrypoint: ["python3", "MongoIngestor.py", "192.168.1.12:32812,192.168.1.12:32814", "kafka-database-consumer-group-2"]
  spark:
    image: bitnami/spark:latest
    environment:
      - SPARK_MODE=master
    expose:
      - "7077"
    ports:
      - '8080:8080'
  spark-worker:
    image: bitnami/spark:latest
    depends_on: 
      - spark
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
  mongo-express:
    image: mongo-express
    ports:
      - "8080:8081"
    env_file:
      - secrets.env
    environment: 
      - ME_CONFIG_MONGODB_SERVER=database
      - ME_CONFIG_MONGODB_ENABLE_ADMIN=true