docker-compose.yml

version: '3'
services:
  spark-submit:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.5 --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/kafka/kafka_example.yaml
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  spark-master:
    image: metorikku/metorikku
    entrypoint:
      - /scripts/entrypoint-master.sh
    logging:
      driver: none
  spark-worker:
    image: metorikku/metorikku
    entrypoint:
      - /scripts/entrypoint-worker.sh
    logging:
      driver: none
  zookeeper:
    image: wurstmeister/zookeeper
  kafka:
    image: wurstmeister/kafka
    environment:
      KAFKA_CREATE_TOPICS: "test:1:1,testAppendOutputMode:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENERS: PLAINTEXT://kafka:9092
    depends_on:
      - zookeeper
  kafka-seed:
    image: wurstmeister/kafka
    volumes:
      - ./scripts:/scripts
      - ./mocks:/mocks
    command: /scripts/kafka_seed.sh
    environment:
      TOPIC: test
      SEED_FILE: /mocks/movies.jsonl
  kafka-consume:
    image: wurstmeister/kafka
    volumes:
      - ./scripts:/scripts
      - ./mocks:/mocks
    command: /scripts/consumer_test.sh
    environment:
      TOPIC: testAppendOutputMode
      NUMBER_OF_EXPECTED_MESSAGES: 5
      MOCK_OUTPUT: /mocks/movies.jsonl