create helm chart

.gitignore

@@ -66,3 +66,4 @@ workspace/
 /srp*gz
 /observer/*
 
+src/helm/output/

build-full.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "This script requires Bash. Use: bash $0 $*"
+    exit 1
+fi
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+pushd "$SCDIR" > /dev/null || exit 1
+./mvnw -s .settings.xml install -Pfull,asciidoctordocs,restdocs -B $*
+popd > /dev/null || exit 1

src/add-deps/gradle.properties

@@ -1,6 +1,6 @@
 appGroup=org.springframework.cloud
 appName=spring-cloud-dataflow-server
-appVersion=2.11.3
+appVersion=2.11.4
 appFolder=.
 repoUrl=https://my.private.repo
 repoUser=repoUserName

src/deploy/helm/add-local-registry-secrets.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "This script requires Bash. Use: bash $0 $*"
+    exit 1
+fi
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+function check_env() {
+  eval ev='$'$1
+  if [ "$ev" == "" ]; then
+    echo "env var $1 not defined"
+    exit 1
+  fi
+}
+
+
+if [ "$4" = "" ]; then
+    echo "Arguments: <secret-name> <registry-name> <registry-user> <registry-password>"
+    exit 1
+fi
+SECRET_NAME=$1
+REGISTRY_NAME=$2
+REGISTRY_USER=$3
+REGISTRY_PWD=$4
+SECRET_NS=$NS
+if [ "$5" != "" ]; then
+    SECRET_NS=$5
+fi
+check_env SECRET_NAME
+check_env SECRET_NS
+
+kubectl create secret docker-registry "$SECRET_NAME" \
+    --docker-server="$REGISTRY_NAME" \
+    --docker-username="$REGISTRY_USER" \
+    --docker-password="$REGISTRY_PWD" \
+    --namespace "$SECRET_NS"
+
+
+if [ -f ./scdf-helm-values.yml ]; then
+    REG_SECRET=$(yq '.global.registry.secret.ref' ./scdf-helm-values.yml)
+    if [ "$REG_SECRET" = "" ]; then
+        yq '.global.registry.secret.ref = strenv(SECRET_NAME)' -i ./scdf-helm-values.yml
+    fi
+    PULL_SECRET=$(yq '.global.imagePullSecrets | .[] | select(. == strenv(SECRET_NAME))' ./scdf-helm-values.yml)
+    if [ "$PULL_SECRET" = "" ]; then
+        PULL_SECRET=$(yq '.global.imagePullSecrets += strenv(SECRET_NAME)' ./scdf-helm-values.yml)
+    fi
+fi

src/deploy/helm/add-roles.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+function add_role() {
+    ROLE=$1
+    ROLE_NAME=$(echo "rolebinding-$NS-default-$ROLE" | sed 's/:/-/g')
+    echo "ROLE_NAME=$ROLE_NAME into $NS"
+    set +e
+    kubectl create rolebinding "$ROLE_NAME" \
+      --namespace $NS \
+      "--clusterrole=$ROLE" \
+      "--user=system:serviceaccount:$NS:default"
+
+    CROLE_NAME=$(echo "cluster-$NS-$ROLE" | sed 's/:/-/g')
+    echo "CROLE_NAME=$CROLE_NAME into $NS"
+    kubectl delete clusterrolebinding "cluster-$NS-${ROLE/:/-}"
+    kubectl create clusterrolebinding "$CROLE_NAME" \
+      --clusterrole=$ROLE \
+      --group=system:authenticated --namespace $NS
+}
+if [ "$NS" == "" ]; then
+  echo "NS not defined"
+  exit 1
+fi
+for role in "$@"; do
+  echo "Adding Role: $role"
+  add_role "$role"
+done

src/deploy/helm/configure-database.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "This script requires Bash. Use: bash $0 $*"
+    exit 1
+fi
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+if [ "$DEBUG" == "true" ]; then
+    echo "DEBUG: configure-database.sh $*"
+fi
+if [ "$4" = "" ]; then
+    echo "<app> <database> <url> <username> <password>"
+    echo " OR"
+    echo "<app> <database> <url> <secret-name> [secret-username-key] [secret-password-key]"
+    echo " - app: dataflow|skipper|global"
+    echo " - secret-username-key: key name in secret. The default is username"
+    echo " - secret-password-key: key name in secret. The default is password"
+    echo "  If username / password is provided it will be assigned to the values file."
+    exit 1
+fi
+
+case $1 in
+"dataflow")
+    APP=server
+    ;;
+"skipper")
+    APP=skipper
+    ;;
+"global")
+    APP=global
+    ;;
+*)
+    echo "Invalid application: $1"
+    exit 1
+    ;;
+esac
+if [ "$DEBUG" == "true" ]; then
+    echo "DEBUG: APP=$APP"
+fi
+case $2 in
+"postgresql" | "postgres")
+    DATABASE=postgresql
+    ;;
+"mariadb" | "maria")
+    DATABASE=mariadb
+    ;;
+"mysql57")
+    DATABASE=mysql57
+    ;;
+"oracle")
+    DATABASE=oracle
+    ;;
+*)
+    echo "Unsupported or invalid database $2"
+    exit 1
+    ;;
+esac
+set +e
+export JDBC_URL="$3"
+if [ "$APP" = "global" ]; then
+    yq '.configuration.database.url = strenv(JDBC_URL)' -i ./scdf-helm-values.yml
+else
+    yq ".${APP}.config.database.url = strenv(JDBC_URL)" -i ./scdf-helm-values.yml
+fi
+
+if [ "$DEBUG" == "true" ]; then
+    echo "DEBUG: DATABASE=$DATABASE"
+fi
+case $DATABASE in
+"mariadb" | "mysql57")
+    JDBC_DRIVER_CLASS=org.mariadb.jdbc.Driver
+    ;;
+"postgresql")
+    JDBC_DRIVER_CLASS=org.postgresql.Driver
+    ;;
+"oracle")
+    JDBC_DRIVER_CLASS=oracle.jdbc.OracleDriver
+    ;;
+*)
+    echo "Unsupported $DATABASE."
+    ;;
+esac
+if [ "$DEBUG" == "true" ]; then
+    echo "DEBUG: JDBC_DRIVER_CLASS=$JDBC_DRIVER_CLASS"
+fi
+
+if [ "$JDBC_DRIVER_CLASS" != "" ]; then
+    export JDBC_DRIVER_CLASS
+    if [ "$APP" = "global" ]; then
+        yq '.configuration.database.driverClassName = strenv(JDBC_DRIVER_CLASS)' -i ./scdf-helm-values.yml
+    else
+        yq ".${APP}.config.database.driverClassName = strenv(JDBC_DRIVER_CLASS)" -i ./scdf-helm-values.yml
+    fi
+fi
+
+if [ "$DIALECT" = "" ] && [ "$DATABASE" = "mariadb" ]; then
+    DIALECT="org.hibernate.dialect.MariaDB106Dialect"
+fi
+if [ "$DIALECT" != "" ]; then
+    if [ "$DEBUG" == "true" ]; then
+        echo "DEBUG: DIALECT=$DIALECT"
+    fi
+    export DIALECT
+    if [ "$APP" = "global" ]; then
+        yq '.configuration.database.dialect = strenv(DIALECT)' -i ./scdf-helm-values.yml
+    else
+        yq ".${APP}.config.database.dialect = strenv(DIALECT)" -i ./scdf-helm-values.yml
+    fi
+fi
+if [ "$6" != "" ]; then
+    SECRET_NAME=$4
+    SECRET_USERNAME_KEY="$5"
+    SECRET_PASSWORD_KEY="$6"
+elif [ "$5" != "" ]; then
+    USERNAME="$4"
+    PASSWORD="$5"
+else
+    SECRET_NAME=$4
+    SECRET_USERNAME_KEY=username
+    SECRET_PASSWORD_KEY=password
+fi
+if [ "$SECRET_NAME" != "" ]; then
+    if [ "$DEBUG" == "true" ]; then
+        echo "DEBUG: SECRET_NAME=$SECRET_NAME, SECRET_USERNAME_KEY=$SECRET_USERNAME_KEY, SECRET_PASSWORD_KEY=$SECRET_PASSWORD_KEY"
+    fi
+    export SECRET_NAME
+    export SECRET_USERNAME_KEY
+    export SECRET_PASSWORD_KEY
+    if [ "$APP" = "global" ]; then
+        yq '.configuration.database.usernameSecret.name = strenv(SECRET_NAME)' -i ./scdf-helm-values.yml
+        yq '.configuration.database.usernameSecret.key = strenv(SECRET_USERNAME_KEY)' -i ./scdf-helm-values.yml
+        yq '.configuration.database.passwordSecret.name = strenv(SECRET_NAME)' -i ./scdf-helm-values.yml
+        yq '.configuration.database.passwordSecret.key = strenv(SECRET_PASSWORD_KEY)' -i ./scdf-helm-values.yml
+    else
+        yq ".${APP}.config.database.usernameSecret.name = strenv(SECRET_NAME)" -i ./scdf-helm-values.yml
+        yq ".${APP}.config.database.usernameSecret.key = strenv(SECRET_USERNAME_KEY)" -i ./scdf-helm-values.yml
+        yq ".${APP}.config.database.passwordSecret.name = strenv(SECRET_NAME)" -i ./scdf-helm-values.yml
+        yq ".${APP}.config.database.passwordSecret.key = strenv(SECRET_PASSWORD_KEY)" -i ./scdf-helm-values.yml
+    fi
+else
+    if [ "$USERNAME" = "" ]; then
+        echo "Expected $USERNAME"
+        exit 1
+    fi
+    if [ "$PASSWORD" = "" ]; then
+        echo "Expected $PASSWORD"
+        exit 1
+    fi
+    export USERNAME
+    export PASSWORD
+    if [ "$APP" = "global" ]; then
+        yq '.configuration.database.username = strenv(USERNAME)' -i ./scdf-helm-values.yml
+        yq '.configuration.database.password = strenv(PASSWORD)' -i ./scdf-helm-values.yml
+    else
+        yq ".${APP}.config.database.username = strenv(USERNAME)" -i ./scdf-helm-values.yml
+        yq ".${APP}.config.database.password = strenv(PASSWORD)" -i ./scdf-helm-values.yml
+
+    fi
+fi
+
+echo "Set ${APP} JDBC url: $JDBC_URL"
+echo "Set ${APP} JDBC class: $JDBC_DRIVER_CLASS"
+echo "Configured ${APP} $DATABASE"

src/deploy/helm/configure-prometheus-proxy.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+if [ "$2" = "" ]; then
+    echo "Usage is: <host> <port> [step]"
+    echo "Where <step> is the frequency of published metrics. Default is 10s"
+    exit 1
+fi
+export HOST=$1
+export PORT=$2
+if [ "$3" != "" ]; then
+    STEP=$3
+else
+    STEP=10s
+fi
+export PROMETHEUS_URL="http://$HOST:$PORT"
+yq "global.management.metrics.export.prometheus.rsocket.host = strenv(HOST)" -i ./scdf-helm-values.yml
+yq "global.management.metrics.export.prometheus.pushgateway.base-url = strenv(PROMETHEUS_URL)" -i ./scdf-helm-values.yml
+yq "global.management.metrics.export.prometheus.pushgateway.enabled = true" -i ./scdf-helm-values.yml
+yq "global.management.metrics.export.prometheus.pushgateway.shutdown-operation = \"PUSH\"" -i ./scdf-helm-values.yml
+yq "global.management.metrics.export.prometheus.step = \"$STEP\"" -i ./scdf-helm-values.yml

src/deploy/helm/create-image-pull-secrets.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "This script requires Bash. Use: bash $0 $*"
+    exit 1
+fi
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+
+function check_env() {
+    eval ev='$'$1
+    if [ "$ev" == "" ]; then
+        echo "env var $1 not defined"
+        exit 1
+    fi
+}
+check_env DOCKER_HUB_USERNAME
+check_env DOCKER_HUB_PASSWORD
+
+$SCDIR/add-local-registry-secret.sh reg-creds-dockerhub docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD" "$NS"
+
+if [ "$SCDF_TYPE" = "pro" ]; then
+    check_env TANZU_DOCKER_USERNAME
+    check_env TANZU_DOCKER_PASSWORD
+    $SCDIR/add-local-registry-secret.sh reg-creds-dev-registry dev.registry.tanzu.vmware.com "$TANZU_DOCKER_USERNAME" "$TANZU_DOCKER_PASSWORD"
+fi

src/deploy/helm/delete-scdf.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+function check_env() {
+  eval ev='$'$1
+  if [ "$ev" == "" ]; then
+    echo "env var $1 not defined"
+    if ((sourced != 0)); then
+      return 1
+    else
+      exit 1
+    fi
+  fi
+}
+check_env NS
+check_env PACKAGE_VERSION
+if [ "$1" != "" ]; then
+    RELEASE_NAME="$1"
+else
+    RELEASE_NAME=dataflow
+fi
+
+echo "Deleting $RELEASE_NAME from $NS"
+helm uninstall $RELEASE_NAME --namespace $NS --wait
+
+kubectl delete apps --all --namespace="$NS"
+kubectl delete deployments --all --namespace="$NS"
+kubectl delete statefulsets --all --namespace="$NS"
+kubectl delete svc --all --namespace="$NS"
+kubectl delete all --all --namespace="$NS"
+kubectl delete pods --all --namespace="$NS"
+kubectl delete pvc --all --namespace="$NS"
+kubectl delete configmaps --all --namespace="$NS"
+kubectl delete secrets --all --namespace="$NS"
+kubectl delete namespace $NS