docker-compose.yml

name: nussknacker

services:

  nginx:
    image: nginx:1.26.0-alpine
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    ports:
      - 8080:8080
    depends_on:
      grafana:
        condition: service_healthy
      designer:
        condition: service_healthy
    healthcheck:
      test: [ "CMD-SHELL", "curl -f http://localhost:8080 || exit 1" ]
      interval: 10s
      retries: 10
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./nginx/additional:/etc/nginx/additional
    deploy:
      resources:
        limits:
          memory: 64M

  ### Nussknacker designer service

  designer:
    build:
      context: ./designer/
      args:
        NUSSKNACKER_VERSION: ${NUSSKNACKER_VERSION:?NUSSKNACKER_VERSION must be defined}
        FLINK_VERSION: "1.19.1-scala_2.12-java11"
        FLINK_VERSION_SHORT: 1.19
        HADOOP_VERSION: 2.8.3
        FLINK_HADOOP_VERSION: "2.8.3-10.0"
        AWS_SDK_VERSION: 2.26.10
        ICEBERG_VERSION: 1.6.1
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      CONFIG_FILE: "/opt/nussknacker/conf/application.conf,/opt/nussknacker/conf/application-customizations.conf"
      DB_URL: "jdbc:postgresql://postgres:5432/nu-db"
      DB_DRIVER: "org.postgresql.Driver"
      DB_USER: "nu"
      DB_PASSWORD: "nupassword"
      NUSSKNACKER_LOG_LEVEL: "INFO"
      KAFKA_ADDRESS: "kafka:9092"
      SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      INFLUXDB_URL: "http://influxdb:8086"
      FLINK_REST_URL: "http://flink-jobmanager:8081"
      NESSIE_URL: "http://nessie:19120/api/v2"
      S3_ENDPOINT: "http://minio:9000"
      WAREHOUSE_URL: "s3://warehouse"
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      AWS_DEFAULT_REGION: us-east-1
      JDK_JAVA_OPTIONS: "-Xmx2048M"
      USAGE_REPORTS_SOURCE: "iceberg-docker-compose"
    depends_on:
      postgres:
        condition: service_healthy
      grafana:
        condition: service_healthy
      kafka:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
      flink-jobmanager:
        condition: service_healthy
      flink-taskmanager:
        condition: service_started
      nessie:
        condition: service_started
    # For some reasons this dependency makes minio-setup stuck in Waiting state. Because of that only dependency to minio is used
#      minio-setup:
#        condition: service_completed_successfully
      minio:
        condition: service_started
    healthcheck:
      test: [ "CMD-SHELL", "curl localhost:8080/api/app/healthCheck" ]
      interval: 10s
      retries: 10
    volumes:
      - ./designer/application-customizations.conf:/opt/nussknacker/conf/application-customizations.conf
    volumes_from:
      - flink-jobmanager
    deploy:
      resources:
        limits:
          memory: 2048M

  postgres:
    image: postgres:13
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      POSTGRES_DB: "nu-db"
      POSTGRES_USER: "nu"
      POSTGRES_PASSWORD: "nupassword"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d nu-db -U nu"]
      interval: 10s
      retries: 10
    volumes:
      - nussknacker_designer_data:/var/lib/postgresql/data
    deploy:
      resources:
        limits:
          memory: 256M

  postgres-setup:
    image: postgres:13
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - nussknacker-iceberg-net
    environment:
      PGHOST: "postgres"
      PGDATABASE: "nu-db"
      PGUSER: "nu"
      PGPASSWORD: "nupassword"
      DB: nessie
      USER: nessie
      PASSWORD: nessie
    volumes:
      - ./postgres/create-db.sh:/create-db.sh
    entrypoint: ["bash", "/create-db.sh"]

  ### Nussknacker metrics-related services:

  grafana:
    image: grafana/grafana:10.1.10
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_SERVER_ROOT_URL: "%(protocol)s://%(domain)s:/grafana"
      GF_SECURITY_ALLOW_EMBEDDING: "true"
    depends_on:
      influxdb:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "nc -z localhost 3000"]
      interval: 10s
      retries: 10
    volumes:
      - ./grafana:/etc/grafana/provisioning
      - ./grafana/dashboards:/var/lib/grafana/dashboards
    deploy:
      resources:
        limits:
          memory: 256M

  influxdb:
    image: influxdb:1.8.10
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      INFLUXDB_DB: "nussknacker_metrics"
    healthcheck:
      test: [ "CMD-SHELL", "influx -execute 'SHOW DATABASES'" ]
      interval: 10s
      retries: 10
    deploy:
      resources:
        limits:
          memory: 128M

  ### KAFKA-related services:

  zookeeper:
    image: bitnami/zookeeper:3.9
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_4LW_COMMANDS_WHITELIST: "ruok"
    healthcheck:
      test: [ "CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok' ]
      interval: 5s
      retries: 5
    deploy:
      resources:
        limits:
          memory: 256M

  kafka:
    image: bitnami/kafka:3.7.0
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    hostname: nu-kafka
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181"
    depends_on:
      zookeeper:
        condition: service_healthy
    healthcheck:
      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list" ]
      interval: 10s
      retries: 10
    deploy:
      resources:
        limits:
          memory: 512M

  schema-registry:
    image: bitnami/schema-registry:7.6.1
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081"
      SCHEMA_REGISTRY_KAFKA_BROKERS: "PLAINTEXT://kafka:9092"
    depends_on:
      kafka:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", 'echo -e "GET /subjects HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" | nc localhost 8081 | grep -q "HTTP/1.1 200 OK" | exit 0']
      interval: 10s
      retries: 10
    deploy:
      resources:
        limits:
          memory: 384M

  akhq:
    image: tchiotludo/akhq:0.24.0
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    environment:
      AKHQ_CONFIGURATION: |
        micronaut:
          server:
            context-path: /akhq
        akhq:
          connections:
            nussknacker-kafka:
              properties:
                bootstrap.servers: "kafka:9092"
              schema-registry:
                url: "http://schema-registry:8081"
    depends_on:
      kafka:
        condition: service_healthy
    healthcheck:
      test: [ "CMD-SHELL", "curl -f http://localhost:8080" ]
      interval: 10s
      retries: 10
    deploy:
      resources:
        limits:
          memory: 256M

  ### FLINK-related services

  flink-jobmanager:
    build:
      context: ./flink/
      args:
        FLINK_VERSION: "1.19.1-scala_2.12-java11"
        FLINK_VERSION_SHORT: 1.19
        HADOOP_VERSION: 2.8.3
        FLINK_HADOOP_VERSION: "2.8.3-10.0"
        AWS_SDK_VERSION: 2.26.10
        ICEBERG_VERSION: 1.6.1
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    command: jobmanager
    environment:
      JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager"
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      AWS_DEFAULT_REGION: us-east-1
    healthcheck:
      test: [ "CMD-SHELL", "curl -f http://localhost:8081/jobs/overview" ]
      interval: 10s
      retries: 10
    volumes:
      - nussknacker_flink_data:/opt/flink/data
    deploy:
      resources:
        limits:
          memory: 2048M

  flink-taskmanager:
    build:
      context: ./flink/
      args:
        FLINK_VERSION: "1.19.1-scala_2.12-java11"
        FLINK_VERSION_SHORT: 1.19
        HADOOP_VERSION: 2.8.3
        FLINK_HADOOP_VERSION: "2.8.3-10.0"
        AWS_SDK_VERSION: 2.26.10
        ICEBERG_VERSION: 1.6.1
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    command: taskmanager
    environment:
      JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager"
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      AWS_DEFAULT_REGION: us-east-1
    depends_on:
      flink-jobmanager:
        condition: service_healthy
      telegraf:
        condition: service_healthy
    volumes_from:
      - flink-jobmanager
    deploy:
      resources:
        limits:
          memory: 2048M

  telegraf:
    image: telegraf:1.30.2
    restart: unless-stopped
    networks:
      - nussknacker-iceberg-net
    healthcheck:
      test: [ "CMD-SHELL", "curl -f http://localhost:8087/write" ]
      interval: 10s
      retries: 10
    volumes:
      - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf
    deploy:
      resources:
        limits:
          memory: 256M

  minio:
    image: minio/minio:RELEASE.2024-06-26T01-06-18Z
    restart: unless-stopped
    networks:
      nussknacker-iceberg-net:
        aliases:
          - warehouse.minio
    hostname: minio
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
      MINIO_DOMAIN: minio
      MINIO_REGION_NAME: us-east-1
      MINIO_REGION: us-east-1
    ports:
      - 9001:9001
      - 9000:9000
    volumes:
      - nussknacker_minio_data:/data
    command: [ "server", "/data", "--console-address", ":9001" ]

  minio-setup:
    depends_on:
      minio:
        condition: service_started
    image: minio/mc:RELEASE.2024-06-24T19-40-33Z
    networks:
      - nussknacker-iceberg-net
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      AWS_DEFAULT_REGION: us-east-1
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      tail -f /dev/null
      "

  nessie:
    image: projectnessie/nessie:0.76.6
    restart: unless-stopped
    depends_on:
      postgres-setup:
        condition: service_completed_successfully
    networks:
      - nussknacker-iceberg-net
    ports:
      - 19120:19120
    environment:
      - nessie.version.store.type=JDBC
      - quarkus.datasource.jdbc.url=jdbc:postgresql://postgres:5432/nessie
      - quarkus.datasource.username=nessie
      - quarkus.datasource.password=nessie

volumes:
  nussknacker_designer_data:
    name: nussknacker_designer_data
  nussknacker_flink_data:
    name: nussknacker_flink_data
  nussknacker_minio_data:
    name: nussknacker_minio_data

networks:
  nussknacker-iceberg-net: