docker-compose.yml

version: "3"

services:

  namenode:
    image: longpt233/hadoop-spark:2.0.0
    container_name: hadoop-master   # = host name = core-site 
    ports: 
      - 8088:8088    # ui yarn
      - 4040:4040   # ui spark local mode
      - 9870:9870   # ui name node  
    tty: true
    volumes:
      - ./data-docker/hadoop/namenode:/root/hdfs/namenode
    networks:
        cluster-hadoop:
          ipv4_address: 172.25.0.2

  datanode1:
    image: longpt233/hadoop-base:2.0.0
    container_name: hadoop-slave1   # phai tuong ung voi file worker ben trong 
    tty: true
    ports:  
      - 9864:9864
    volumes:
      - ./data-docker/hadoop/datanode-1:/root/hdfs/datanode
    networks:
        cluster-hadoop:
          ipv4_address: 172.25.0.3

  datanode2:
    image: longpt233/hadoop-base:2.0.0
    container_name: hadoop-slave2 
    tty: true
    volumes:
      - ./data-docker/hadoop/datanode-2:/root/hdfs/datanode
    networks:
        cluster-hadoop:
          ipv4_address: 172.25.0.4

  zookeeper:
    image: zookeeper:3.4.9
    container_name: zookeeper-name 
    hostname: zookeeper-host
    ports:
      - "2181:2181"
    environment:
      ZOO_MY_ID: 1
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zookeeper-host:2888:3888
    volumes:
      - ./data-docker/zoo/data:/data
      - ./data-docker/zoo/datalog:/datalog
    networks:
      cluster-hadoop:
        ipv4_address: 172.25.0.7

  kafka:
    image: confluentinc/cp-kafka:5.3.0
    container_name: kafka-name
    hostname: kafka-host
    ports:
      - 9092:9092
      - 19092:19092
      - 29092:29092
    environment: 
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-host:19092,LISTENER_DOCKER_EXTERNAL://kafka-host:9092, OUTSIDE://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT, OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-host:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper
    volumes:
      - ./data-docker/kafka/data:/var/lib/kafka/data
    networks:
      cluster-hadoop:
        ipv4_address: 172.25.0.8

  elasticsearch:
    container_name: es-name
    hostname: es-host
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
    environment:
      - xpack.security.enabled=false
      - "discovery.type=single-node"
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"  # không thì mặc định es ăn hết cmn ram luôn
    networks:
      cluster-hadoop:
        ipv4_address: 172.25.0.11
    ports:
      - 9200:9200
    ulimits: 
      memlock: 
        hard: -1
        soft: -1
    volumes:
      - ./data-docker/es/data:/usr/share/elasticsearch/data

  kibana:
    container_name: kb-name
    image: docker.elastic.co/kibana/kibana:7.12.0
    environment:
      - ELASTICSEARCH_HOSTS=http://es-host:9200
    networks:
      cluster-hadoop:
        ipv4_address: 172.25.0.12
    depends_on:
      - elasticsearch
    ports:
      - 5601:5601

  
networks:
  cluster-hadoop:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.25.0.0/16