docker-compose.kafka.yml

version: "3.9"

################
##  NETWORKS  ##
################
networks:
  kafka-net:
    name: kafka-net
    driver: bridge

  proxy-net:
    external: true

  weave-net:
    external: true

###############
##  Volumes  ##
###############

volumes:
  zookeeper-data-0:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/z1/data
      o: bind

  zookeeper-log-0:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/z1/logs
      o: bind

  zookeeper-data-1:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/z2/data
      o: bind

  zookeeper-log-1:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/z2/logs
      o: bind

  kafka-data-0:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/k1/data
      o: bind

  kafka-data-1:
    driver: local
    driver_opts:
      type: none
      device: /home/sinless777/services/helix/zookeeper/k2/data
      o: bind

services:
  akhq:
    container_name: akhq
    networks:
      - kafka-net
      - proxy-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    image: tchiotludo/akhq
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            docker-kafka-server-1:
              properties:
                bootstrap.servers: "kafka-0:29092"
              schema-registry:
                url: "http://schema-registry-0:8085"
              connect:
                - name: connect-1
                  url: "http://connect-0:8083"
                - name: connect-2
                  url: "http://connect-1:8084"
            docker-kafka-server-2:
              properties:
                bootstrap.servers: "kafka-1:29072"
    ports:
      - 8800:8080
    links:
      - kafka-0
      - schema-registry-0
      - connect-0
      - connect-1
      - kafka-1

  zookeeper-0:
    container_name: zookeeper-0
    image: confluentinc/cp-zookeeper:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    volumes:
      - zookeeper-data-0:/var/lib/zookeeper/data:Z
      - zookeeper-log-0:/var/lib/zookeeper/log:Z
    environment:
      ZOOKEEPER_CLIENT_PORT: '2181'
      ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false'

  kafka-0:
    container_name: kafka-0
    image: confluentinc/cp-kafka:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    volumes:
      - kafka-data-0:/var/lib/kafka:Z
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: '0'
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-0:2181'
      KAFKA_NUM_PARTITIONS: '12'
      KAFKA_COMPRESSION_TYPE: 'gzip'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
      KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
      KAFKA_JMX_PORT: '9091'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
      KAFKA_LISTENERS: 'INTERNAL://kafka-0:29092,OUTSIDE://kafka-0:9092'
      KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka-0:29092,OUTSIDE://localhost:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL'
    links:
      - zookeeper-0

  schema-registry-0:
    container_name: schema-registry-0
    image: confluentinc/cp-schema-registry:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    ports:
      - 8085:8085
    depends_on:
      - kafka-0
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka-0:29092'
      SCHEMA_REGISTRY_HOST_NAME: 'schema-registry-0'
      SCHEMA_REGISTRY_LISTENERS: 'http://schema-registry-0:8085'
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO'

  connect-0:
    container_name: connect-0
    image: confluentinc/cp-kafka-connect:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    ports:
      - 8083:8083
    depends_on:
      - kafka-0
      - schema-registry-0
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka-0:29092'
      CONNECT_REST_PORT: '8083'
      CONNECT_HOST_NAME: 'connect-0'
      CONNECT_REST_LISTENERS: 'http://connect-0:8083'
      CONNECT_LISTENERS: 'http://connect-0:8083'
      CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-0'
      CONNECT_CONFIG_STORAGE_TOPIC: '__connect-0-config'
      CONNECT_OFFSET_STORAGE_TOPIC: '__connect-0-offsets'
      CONNECT_STATUS_STORAGE_TOPIC: '__connect-0-status'
      CONNECT_GROUP_ID: 'kafka-connect-0'
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true'
      CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085'
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true'
      CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085'
      CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_PLUGIN_PATH: ' /usr/share/java/'

  connect-1:
    container_name: connect-1
    image: confluentinc/cp-kafka-connect:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    ports:
      - 8084:8084
    depends_on:
      - kafka-0
      - schema-registry-0
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka-0:29092'
      CONNECT_REST_PORT: '8084'
      CONNECT_HOST_NAME: 'connect-1'
      CONNECT_REST_LISTENERS: 'http://connect-1:8084'
      CONNECT_LISTENERS: 'http://connect-1:8084'
      CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-1'
      CONNECT_CONFIG_STORAGE_TOPIC: '__connect-1-config'
      CONNECT_OFFSET_STORAGE_TOPIC: '__connect-1-offsets'
      CONNECT_STATUS_STORAGE_TOPIC: '__connect-1-status'
      CONNECT_GROUP_ID: 'kafka-connect-1'
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true'
      CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085'
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true'
      CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085'
      CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_PLUGIN_PATH: ' /usr/share/java/'

  # ANOTHER CLUSTER
  zookeeper-1:
    container_name: zookeeper-1
    image: confluentinc/cp-zookeeper:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    volumes:
      - zookeeper-data-1:/var/lib/zookeeper/data:Z
      - zookeeper-log-1:/var/lib/zookeeper/log:Z
    environment:
      ZOOKEEPER_CLIENT_PORT: '2171'
      ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false'

  kafka-1:
    container_name: kafka-1
    image: confluentinc/cp-kafka:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    networks:
      - kafka-net
      - weave-net
    labels:
      com.centurylinklabs.watchtower.enable: "true"
    volumes:
      - kafka-data-1:/var/lib/kafka:Z
    ports:
      - 9072:9072
    environment:
      KAFKA_BROKER_ID: '0'
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2171'
      KAFKA_NUM_PARTITIONS: '12'
      KAFKA_COMPRESSION_TYPE: 'gzip'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
      KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
      KAFKA_JMX_PORT: '9071'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
      KAFKA_LISTENERS: 'INTERNAL://kafka-1:29072,OUTSIDE://kafka-1:9072'
      KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka-1:29072,OUTSIDE://localhost:9072'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL'
    links:
      - zookeeper-1