[NU-1726] new quickstart (#176)

.github/workflows/pr-newquickstart-test.yml

@@ -0,0 +1,36 @@
+name: QUICKSTART CI
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - release/*
+  push:
+    branches:
+      - main
+      - staging
+      - release/*
+
+jobs:
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: Test Quickstart start
+        timeout-minutes: 10
+        run: |
+          ./new-quickstart/start.sh || (docker-compose logs && exit 1) 
+      - name: Clean up Quickstart
+        run: |
+          ./new-quickstart/stop-and-clean.sh 
+  

new-quickstart/.env

@@ -0,0 +1 @@
+NUSSKNACKER_VERSION=1.16.0-RC1

new-quickstart/designer/application-customizations.conf

@@ -0,0 +1,16 @@
+## To customize Nu Designer configuration see https://nussknacker.io/documentation/docs/installation_configuration_guide/Common/
+
+scenarioTypes {
+  "streaming" {
+    # customize Flink streaming scenario type
+    # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file
+  }
+  "streaming-lite-embedded" {
+    # customize Lite streaming scenario type
+    # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file
+  }
+  "request-response-embedded": {
+    # customize Lite request-response scenario type
+    # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file
+  }
+}

new-quickstart/docker-compose.yml

@@ -0,0 +1,269 @@
+name: nussknacker
+
+services:
+
+  ### Quickstart setup container 
+  quickstart-setup:
+    build:
+      context: quickstart-setup/
+      dockerfile: Dockerfile
+    depends_on:
+      nginx:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+
+  ### Nussknacker-related services
+
+  nginx:
+    image: nginx:1.26.0-alpine
+    restart: unless-stopped
+    ports:
+      - 8080:8080
+      - 8181:8181
+    depends_on:
+      grafana:
+        condition: service_healthy
+      designer:
+        condition: service_healthy
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:8080 || exit 1" ]
+      interval: 10s
+      retries: 10
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/additional:/etc/nginx/additional
+    deploy:
+      resources:
+        limits:
+          memory: 64M
+
+  ### Nussknacker designer service
+
+  designer:
+    image: touk/nussknacker:${NUSSKNACKER_VERSION:?NUSSKNACKER_VERSION must be defined}_scala-2.12
+    restart: unless-stopped
+    environment:
+      CONFIG_FILE: "/opt/nussknacker/conf/application.conf,/opt/nussknacker/conf/application-customizations.conf"
+      DB_URL: "jdbc:postgresql://postgres:5432/nu-db"
+      DB_DRIVER: "org.postgresql.Driver"
+      DB_USER: "nu"
+      DB_PASSWORD: "nupassword"
+      NUSSKNACKER_LOG_LEVEL: "INFO"
+      KAFKA_ADDRESS: "kafka:9092"
+      SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
+      INFLUXDB_URL: "http://influxdb:8086"
+      FLINK_REST_URL: "http://flink-jobmanager:8081"
+      JDK_JAVA_OPTIONS: "-Xmx1024M"
+      USAGE_REPORTS_SOURCE: "quickstart-docker-compose"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      grafana:
+        condition: service_healthy
+      kafka:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+      flink-jobmanager:
+        condition: service_healthy
+      flink-taskmanager:
+        condition: service_started
+    expose: 
+      - 8181
+    healthcheck:
+      test: [ "CMD-SHELL", "curl localhost:8080/api/app/healthCheck" ]
+      interval: 10s
+      retries: 10
+    volumes:
+      - ./designer/application-customizations.conf:/opt/nussknacker/conf/application-customizations.conf
+    volumes_from:
+      - flink-jobmanager
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+
+  postgres:
+    image: postgres:13
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: "nu-db"
+      POSTGRES_USER: "nu"
+      POSTGRES_PASSWORD: "nupassword"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d nu-db -U nu"]
+      interval: 10s
+      retries: 10
+    volumes:
+      - nussknacker_designer_data:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  ### Nussknacker metrics-related services:
+
+  grafana:
+    image: grafana/grafana:10.1.10
+    restart: unless-stopped
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_SERVER_ROOT_URL: "%(protocol)s://%(domain)s:/grafana"
+      GF_SECURITY_ALLOW_EMBEDDING: "true"
+    depends_on:
+      influxdb:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z localhost 3000"]
+      interval: 10s
+      retries: 10
+    volumes:
+      - ./grafana:/etc/grafana/provisioning
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  influxdb:
+    image: influxdb:1.8.10
+    restart: unless-stopped
+    environment:
+      INFLUXDB_DB: "nussknacker_metrics"
+    healthcheck:
+      test: [ "CMD-SHELL", "influx -execute 'SHOW DATABASES'" ]
+      interval: 10s
+      retries: 10
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+
+  ### KAFKA-related services:
+
+  kafka:
+    image: bitnami/kafka:3.7.0
+    restart: unless-stopped
+    hostname: nu-kafka
+    environment:
+      KAFKA_CFG_NODE_ID: 0
+      KAFKA_CFG_PROCESS_ROLES: "controller,broker"
+      KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka:9093"
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list" ]
+      interval: 10s
+      retries: 10
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+
+  schema-registry:
+    image: bitnami/schema-registry:7.6.1
+    restart: unless-stopped
+    environment:
+      SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081"
+      SCHEMA_REGISTRY_KAFKA_BROKERS: "PLAINTEXT://kafka:9092"
+    depends_on:
+      kafka:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", 'echo -e "GET /subjects HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" | nc localhost 8081 | grep -q "HTTP/1.1 200 OK" | exit 0']
+      interval: 10s
+      retries: 10
+    deploy:
+      resources:
+        limits:
+          memory: 384M
+
+  akhq:
+    image: tchiotludo/akhq:0.24.0
+    restart: unless-stopped
+    environment:
+      AKHQ_CONFIGURATION: |
+        micronaut:
+          server:
+            context-path: /akhq
+        akhq:
+          connections:
+            nussknacker-kafka:
+              properties:
+                bootstrap.servers: "kafka:9092"
+              schema-registry:
+                url: "http://schema-registry:8081"
+    depends_on:
+      kafka:
+        condition: service_healthy
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:8080" ]
+      interval: 10s
+      retries: 10
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  ### FLINK-related services
+
+  flink-jobmanager:
+    build:
+      context: ./flink/
+      args:
+        FLINK_VERSION: "1.18.1-scala_2.12-java11"
+    restart: unless-stopped
+    command: jobmanager
+    environment:
+      JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager"
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:8081/jobs/overview" ]
+      interval: 10s
+      retries: 10
+    volumes:
+      - nussknacker_flink_data:/opt/flink/data
+
+  flink-taskmanager:
+    build:
+      context: ./flink/
+      args:
+        FLINK_VERSION: "1.18.1-scala_2.12-java11"
+    restart: unless-stopped
+    command: taskmanager
+    environment:
+      JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager"
+    depends_on:
+      flink-jobmanager:
+        condition: service_healthy
+      telegraf:
+        condition: service_healthy
+    volumes_from:
+      - flink-jobmanager
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+
+  telegraf:
+    image: telegraf:1.30.2
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:8087/write" ]
+      interval: 10s
+      retries: 10
+    volumes:
+      - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+
+volumes:
+  nussknacker_designer_data:
+    name: nussknacker_designer_data
+  nussknacker_flink_data:
+    name: nussknacker_flink_data

new-quickstart/download-and-start.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+BRANCH_NAME="main"
+NU_QUICKSTART_DIR="nussknacker-quickstart-$BRANCH_NAME"
+
+wget -O "$NU_QUICKSTART_DIR.zip" "https://github.com/TouK/nussknacker-quickstart/archive/refs/heads/$BRANCH_NAME.zip"
+rm -rf "$NU_QUICKSTART_DIR"
+unzip -q "$NU_QUICKSTART_DIR.zip" 
+cd "$NU_QUICKSTART_DIR/new-quickstart"
+./start.sh

new-quickstart/flink/Dockerfile

@@ -0,0 +1,17 @@
+ARG FLINK_VERSION
+
+FROM flink:${FLINK_VERSION}
+
+USER root
+RUN echo '#!/bin/sh' > /ex-docker-entrypoint.sh && \
+    echo 'export FLINK_PROPERTIES=$(cat /opt/flink/conf/flink-properties.yml) && /docker-entrypoint.sh "$@"' >> /ex-docker-entrypoint.sh && \
+    chmod +x /ex-docker-entrypoint.sh
+
+USER flink
+COPY flink-properties.yml /opt/flink/conf/
+RUN mkdir -p /opt/flink/data && \
+    chmod -R 777 /opt/flink/data
+
+VOLUME /opt/flink/data
+
+ENTRYPOINT [ "/ex-docker-entrypoint.sh" ]

new-quickstart/flink/flink-properties.yml

@@ -0,0 +1,28 @@
+taskmanager.numberOfTaskSlots: 8
+
+state.backend: filesystem
+state.checkpoints.dir: file:///opt/flink/data/checkpoints
+state.savepoints.dir: file:///opt/flink/data/savepoints
+
+#Below are base settings for rocksdb metrics, that can be used for grafana dashboards
+state.backend.rocksdb.metrics.estimate-num-keys: true
+state.backend.rocksdb.metrics.estimate-live-data-size: true
+state.backend.rocksdb.metrics.cur-size-all-mem-tables: true
+state.backend.rocksdb.metrics.size-all-mem-tables: true
+# We can have many jobs per cluster, in such setting managed memory is not easy to tune
+state.backend.rocksdb.memory.managed: false
+# For frequent writes increase the value as needed. Currently RocksDB settings can only be changed per Flink cluster
+state.backend.rocksdb.writebuffer.size: 256m
+
+metrics.reporters: influxdb_reporter
+metrics.reporter.influxdb_reporter.factory.class: org.apache.flink.metrics.influxdb.InfluxdbReporterFactory
+metrics.reporter.influxdb_reporter.host: telegraf
+metrics.reporter.influxdb_reporter.port: 8087
+metrics.reporter.influxdb_reporter.db: nussknacker_metrics
+metrics.reporter.influxdb_reporter.scope.variables.excludes: tm_id;job_id;task_id;task_attempt_id;operator_id;task_attempt_num;task_name
+metrics.scope.jm: local.<host>.jobmanagerGlobal
+metrics.scope.jm.job: local.<host>.jobmanagerJob.<job_name>
+metrics.scope.tm: local.<host>.taskmanagerGlobal.<tm_id>
+metrics.scope.tm.job: local.<host>.taskmanagerJob.<tm_id>.<job_name>
+metrics.scope.task: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<task_name>.<subtask_index>
+metrics.scope.operator: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<operator_name>.<subtask_index>