From 1d6a205d78ea9abf5f752464e3c79a40de3ed84f Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Fri, 23 Aug 2024 13:39:30 +0200 Subject: [PATCH 1/3] Update to 2.11.5-SNAPSHOT --- .../appendix-local-k8s-development.adoc | 4 +-- src/deploy/README.html | 4 +-- src/deploy/carvel/prepare-cluster.sh | 28 ++++++++++++++++++- src/deploy/carvel/register-apps.sh | 19 ++++--------- .../images/pull-composed-task-runner.sh | 2 +- src/deploy/images/pull-dataflow.sh | 2 +- src/deploy/images/pull-grafana.sh | 2 +- .../images/pull-single-step-batch-job.sh | 2 +- src/deploy/images/pull-skipper.sh | 2 +- src/deploy/k8s/configure-k8s.sh | 2 +- src/deploy/k8s/deploy-scdf.sh | 8 +++--- src/deploy/k8s/load-images.sh | 4 +-- src/deploy/k8s/register-apps.sh | 3 +- src/deploy/k8s/update-scdf.sh | 4 +-- src/deploy/shell/register-apps.sh | 10 +++++-- src/deploy/versions.yaml | 10 +++---- .../grafana/grafana-deployment.yaml | 2 +- src/local/register-apps.sh | 1 + 18 files changed, 67 insertions(+), 42 deletions(-) diff --git a/spring-cloud-dataflow-docs/src/main/asciidoc/appendix-local-k8s-development.adoc b/spring-cloud-dataflow-docs/src/main/asciidoc/appendix-local-k8s-development.adoc index a79a5f1d54..6f83aa4f84 100644 --- a/spring-cloud-dataflow-docs/src/main/asciidoc/appendix-local-k8s-development.adoc +++ b/spring-cloud-dataflow-docs/src/main/asciidoc/appendix-local-k8s-development.adoc @@ -139,8 +139,8 @@ This example shows the versions of the current development snapshot. [source,shell] .... -export DATAFLOW_VERSION=2.11.4-SNAPSHOT -export SKIPPER_VERSION=2.11.4-SNAPSHOT +export DATAFLOW_VERSION=2.11.5-SNAPSHOT +export SKIPPER_VERSION=2.11.5-SNAPSHOT .... Before you can install SCDF you will need to pull the following images to ensure they are present for uploading to the k8s cluster. diff --git a/src/deploy/README.html b/src/deploy/README.html index d062a612df..b21db7dbf6 100644 --- a/src/deploy/README.html +++ b/src/deploy/README.html @@ -1280,8 +1280,8 @@
Configure Database
-
export DATAFLOW_VERSION=2.11.4-SNAPSHOT
-export SKIPPER_VERSION=2.11.4-SNAPSHOT
+
export DATAFLOW_VERSION=2.11.5-SNAPSHOT
+export SKIPPER_VERSION=2.11.5-SNAPSHOT
diff --git a/src/deploy/carvel/prepare-cluster.sh b/src/deploy/carvel/prepare-cluster.sh index a6dba31b38..8764309754 100755 --- a/src/deploy/carvel/prepare-cluster.sh +++ b/src/deploy/carvel/prepare-cluster.sh @@ -1,10 +1,36 @@ #!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +K8S=$(realpath $SCDIR/../k8s) bold="\033[1m" dim="\033[2m" end="\033[0m" CERT_MANAGER_VERSION=v1.14.2 -SECRETGEN_CONTROLLER_VERSION=v0.17.0 +SECRETGEN_CONTROLLER_VERSION=v0.17.3 KAPP_CONTROLLER_VERSION=v0.50.0 +wget -q -O cert-manager-release.yml "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" +IMAGES=$(yq '.spec.template.spec.containers | .[] | .image' cert-manager-release.yml) +for image in $IMAGES; do + if [[ "$image" != "---"* ]]; then + $K8S/load-image.sh "$image" + fi +done +#$K8S/load-image.sh "quay.io/jetstack/cert-manager-controller:$CERT_MANAGER_VERSION" +#$K8S/load-image.sh "quay.io/jetstack/cert-manager-cainjector:$CERT_MANAGER_VERSION" +#$K8S/load-image.sh "quay.io/jetstack/cert-manager-webhook:$CERT_MANAGER_VERSION" +wget -q -O secretgen-release.yml "https://github.com/carvel-dev/secretgen-controller/releases/download/$SECRETGEN_CONTROLLER_VERSION/release.yml" +IMAGES=$(yq '.spec.template.spec.containers | .[] | .image' secretgen-release.yml) +for image in $IMAGES; do + if [[ "$image" != "---"* ]]; then + $K8S/load-image.sh "$image" + fi +done +wget -q -O kapp-controller-release.yml https://github.com/carvel-dev/kapp-controller/releases/download/$KAPP_CONTROLLER_VERSION/release.yml +IMAGES=$(yq '.spec.template.spec.containers | .[] | .image' kapp-controller-release.yml) +for image in $IMAGES; do + if [[ "$image" != "---"* ]]; then + $K8S/load-image.sh "$image" + fi +done start_time=$(date +%s) echo "Deploying cert-manager $CERT_MANAGER_VERSION" kapp deploy --yes --wait --wait-check-interval 10s --app cert-manager \ diff --git a/src/deploy/carvel/register-apps.sh b/src/deploy/carvel/register-apps.sh index 4163a410d4..ef2095be1c 100755 --- a/src/deploy/carvel/register-apps.sh +++ b/src/deploy/carvel/register-apps.sh @@ -43,10 +43,13 @@ else # unfortunately different in docker image names and registration link. BROKER_NAME=rabbitmq fi - if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - STREAM_URI="https://dataflow.spring.io/$BROKER_NAME-${TYPE}-latest" -elif [[ "$STREAM_APPS_RT_VERSION" = *"SNAPSHOT"* ]]; then + # export STREAM_APPS_RT_VERSION=2022.0.0-SNAPSHOT + # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x + export STREAM_APPS_RT_VERSION=2024.0.0 +fi +echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" +if [[ "$STREAM_APPS_RT_VERSION" = *"SNAPSHOT"* ]]; then STREAM_APPS_DL_VERSION=$STREAM_APPS_RT_VERSION META_DATA="https://repo.spring.io/snapshot/org/springframework/cloud/stream/app/stream-applications-descriptor/${STREAM_APPS_RT_VERSION}/maven-metadata.xml" echo "Downloading $META_DATA" @@ -72,16 +75,6 @@ fi echo "Registering Stream applications at $DATAFLOW_URL using $STREAM_URI" wget -qO- $DATAFLOW_URL/apps --post-data="uri=$STREAM_URI" -# replace with individual invocations of register_app for only those applications used. -#register_app "source/file" "docker:springcloudstream/file-source-$BROKER_NAME:3.2.1" -#register_app "source/ftp" "docker:springcloudstream/ftp-source-$BROKER_NAME:3.2.1" -#register_app "processor/aggregator" "docker:springcloudstream/aggregator-processor-$BROKER_NAME:3.2.1" -#register_app "processor/filter" "docker:springcloudstream/filter-processor-$BROKER_NAME:3.2.1" -#register_app "processor/groovy" "docker:springcloudstream/groovy-processor-$BROKER_NAME:3.2.1" -#register_app "processor/script" "docker:springcloudstream/script-processor-$BROKER_NAME:3.2.1" -#register_app "processor/splitter" "docker:springcloudstream/splitter-processor-$BROKER_NAME:3.2.1" -#register_app "processor/transform" "docker:springcloudstream/transform-processor-$BROKER_NAME:3.2.1" - TASK_URI=https://dataflow.spring.io/task-${TYPE}-latest echo "Registering Task applications at $DATAFLOW_URL using $TASK_URI" wget -qO- "$DATAFLOW_URL/apps" --post-data="uri=$TASK_URI" diff --git a/src/deploy/images/pull-composed-task-runner.sh b/src/deploy/images/pull-composed-task-runner.sh index c330c546ca..2276dd3a32 100755 --- a/src/deploy/images/pull-composed-task-runner.sh +++ b/src/deploy/images/pull-composed-task-runner.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ "$DATAFLOW_VERSION" = "" ]; then - DATAFLOW_VERSION=2.11.4-SNAPSHOT + DATAFLOW_VERSION=2.11.5-SNAPSHOT fi docker pull "springcloud/spring-cloud-dataflow-composed-task-runner:$DATAFLOW_VERSION" diff --git a/src/deploy/images/pull-dataflow.sh b/src/deploy/images/pull-dataflow.sh index 46cbb5cfb6..6e75a92f7e 100755 --- a/src/deploy/images/pull-dataflow.sh +++ b/src/deploy/images/pull-dataflow.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ "$DATAFLOW_VERSION" = "" ]; then - DATAFLOW_VERSION=2.11.4-SNAPSHOT + DATAFLOW_VERSION=2.11.5-SNAPSHOT fi docker pull "springcloud/spring-cloud-dataflow-server:$DATAFLOW_VERSION" diff --git a/src/deploy/images/pull-grafana.sh b/src/deploy/images/pull-grafana.sh index 634da9dcd6..6c81eece02 100755 --- a/src/deploy/images/pull-grafana.sh +++ b/src/deploy/images/pull-grafana.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ "$DATAFLOW_VERSION" = "" ]; then - DATAFLOW_VERSION=2.11.4-SNAPSHOT + DATAFLOW_VERSION=2.11.5-SNAPSHOT fi docker pull "springcloud/spring-cloud-dataflow-grafana-prometheus:$DATAFLOW_VERSION" diff --git a/src/deploy/images/pull-single-step-batch-job.sh b/src/deploy/images/pull-single-step-batch-job.sh index 7819c27e15..df3b6b5f8a 100755 --- a/src/deploy/images/pull-single-step-batch-job.sh +++ b/src/deploy/images/pull-single-step-batch-job.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ "$DATAFLOW_VERSION" = "" ]; then - DATAFLOW_VERSION=2.11.4-SNAPSHOT + DATAFLOW_VERSION=2.11.5-SNAPSHOT fi docker pull "springcloud/spring-cloud-dataflow-single-step-batch-job:$DATAFLOW_VERSION" diff --git a/src/deploy/images/pull-skipper.sh b/src/deploy/images/pull-skipper.sh index 8d9f21c15f..c867043457 100755 --- a/src/deploy/images/pull-skipper.sh +++ b/src/deploy/images/pull-skipper.sh @@ -1,6 +1,6 @@ #!/bin/bash if [ "$SKIPPER_VERSION" = "" ]; then - SKIPPER_VERSION=2.11.4-SNAPSHOT + SKIPPER_VERSION=2.11.5-SNAPSHOT fi docker pull "springcloud/spring-cloud-skipper-server:$SKIPPER_VERSION" diff --git a/src/deploy/k8s/configure-k8s.sh b/src/deploy/k8s/configure-k8s.sh index 44d02e7773..df144a5bd4 100755 --- a/src/deploy/k8s/configure-k8s.sh +++ b/src/deploy/k8s/configure-k8s.sh @@ -22,7 +22,7 @@ if [ "$1" != "" ]; then export K8S_VERSION="$1" else if [ "$K8S_VERSION" == "" ]; then - export K8S_VERSION="1.28" + export K8S_VERSION="1.30" fi fi set +e diff --git a/src/deploy/k8s/deploy-scdf.sh b/src/deploy/k8s/deploy-scdf.sh index e56719d2b6..0ef119b65d 100755 --- a/src/deploy/k8s/deploy-scdf.sh +++ b/src/deploy/k8s/deploy-scdf.sh @@ -17,10 +17,10 @@ if [ "$DATAFLOW_PRO_VERSION" = "" ]; then DATAFLOW_PRO_VERSION=1.6.1-SNAPSHOT fi if [ "$DATAFLOW_VERSION" = "" ]; then - export DATAFLOW_VERSION=2.11.4-SNAPSHOT + export DATAFLOW_VERSION=2.11.5-SNAPSHOT fi if [ "$SKIPPER_VERSION" = "" ]; then - export SKIPPER_VERSION=2.11.4-SNAPSHOT + export SKIPPER_VERSION=2.11.5-SNAPSHOT fi case $BROKER in @@ -85,11 +85,11 @@ if [ "$USE_PRO" = "" ]; then fi if [ "$DATAFLOW_VERSION" = "" ]; then - DATAFLOW_VERSION=2.11.4-SNAPSHOT + DATAFLOW_VERSION=2.11.5-SNAPSHOT fi if [ "$SKIPPER_VERSION" = "" ]; then - SKIPPER_VERSION=2.11.4-SNAPSHOT + SKIPPER_VERSION=2.11.5-SNAPSHOT fi YAML_PATH=$(realpath $SCDIR/yaml) diff --git a/src/deploy/k8s/load-images.sh b/src/deploy/k8s/load-images.sh index 29e8e37935..25ca0b1c85 100755 --- a/src/deploy/k8s/load-images.sh +++ b/src/deploy/k8s/load-images.sh @@ -18,9 +18,9 @@ case $BROKER in esac if [ "$STREAM_APPS_VERSION" == "" ]; then if [[ "$DATAFLOW_VERSION" == *"SNAPSHOT"* ]]; then - STREAM_APPS_VERSION="4.0.0-SNAPSHOT" + STREAM_APPS_VERSION="5.0.1-SNAPSHOT" else - STREAM_APPS_VERSION="4.0.0" + STREAM_APPS_VERSION="5.0.0" fi fi diff --git a/src/deploy/k8s/register-apps.sh b/src/deploy/k8s/register-apps.sh index ee04f2f03a..97f9105ac6 100755 --- a/src/deploy/k8s/register-apps.sh +++ b/src/deploy/k8s/register-apps.sh @@ -46,8 +46,7 @@ fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then # export STREAM_APPS_RT_VERSION=2022.0.0-SNAPSHOT # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - # export STREAM_APPS_RT_VERSION=2022.0.0 # release for Boot 3.x - export STREAM_APPS_RT_VERSION=2022.0.0 + export STREAM_APPS_RT_VERSION=2024.0.0 fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" if [ "$PLATFORM_TYPE" != "kubernetes" ]; then diff --git a/src/deploy/k8s/update-scdf.sh b/src/deploy/k8s/update-scdf.sh index eef4a50dfa..8414270734 100755 --- a/src/deploy/k8s/update-scdf.sh +++ b/src/deploy/k8s/update-scdf.sh @@ -13,10 +13,10 @@ if [ "$DATAFLOW_PRO_VERSION" = "" ]; then DATAFLOW_PRO_VERSION=1.6.1-SNAPSHOT fi if [ "$DATAFLOW_VERSION" = "" ]; then - export DATAFLOW_VERSION=2.11.4-SNAPSHOT + export DATAFLOW_VERSION=2.11.5-SNAPSHOT fi if [ "$SKIPPER_VERSION" = "" ]; then - export SKIPPER_VERSION=2.11.4-SNAPSHOT + export SKIPPER_VERSION=2.11.5-SNAPSHOT fi case $BROKER in diff --git a/src/deploy/shell/register-apps.sh b/src/deploy/shell/register-apps.sh index 339a2188ec..71b8df868d 100644 --- a/src/deploy/shell/register-apps.sh +++ b/src/deploy/shell/register-apps.sh @@ -5,14 +5,20 @@ if [ -z "$BASH_VERSION" ]; then fi SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +if [ "$BROKER" == "rabbitmq" ]; then + BROKER_NAME=rabbit +else + BROKER_NAME=$BROKER +fi + cat > $SCDIR/register-apps.shell < Date: Mon, 26 Aug 2024 13:21:14 +0200 Subject: [PATCH 2/3] [Ci] Cleanup after comments. --- src/deploy/carvel/prepare-cluster.sh | 3 --- src/deploy/carvel/register-apps.sh | 3 +-- src/deploy/k8s/register-apps.sh | 3 +-- src/local/register-apps.sh | 4 +--- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/deploy/carvel/prepare-cluster.sh b/src/deploy/carvel/prepare-cluster.sh index 8764309754..732ea70e35 100755 --- a/src/deploy/carvel/prepare-cluster.sh +++ b/src/deploy/carvel/prepare-cluster.sh @@ -14,9 +14,6 @@ for image in $IMAGES; do $K8S/load-image.sh "$image" fi done -#$K8S/load-image.sh "quay.io/jetstack/cert-manager-controller:$CERT_MANAGER_VERSION" -#$K8S/load-image.sh "quay.io/jetstack/cert-manager-cainjector:$CERT_MANAGER_VERSION" -#$K8S/load-image.sh "quay.io/jetstack/cert-manager-webhook:$CERT_MANAGER_VERSION" wget -q -O secretgen-release.yml "https://github.com/carvel-dev/secretgen-controller/releases/download/$SECRETGEN_CONTROLLER_VERSION/release.yml" IMAGES=$(yq '.spec.template.spec.containers | .[] | .image' secretgen-release.yml) for image in $IMAGES; do diff --git a/src/deploy/carvel/register-apps.sh b/src/deploy/carvel/register-apps.sh index ef2095be1c..55d5848673 100755 --- a/src/deploy/carvel/register-apps.sh +++ b/src/deploy/carvel/register-apps.sh @@ -44,9 +44,8 @@ else BROKER_NAME=rabbitmq fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - # export STREAM_APPS_RT_VERSION=2022.0.0-SNAPSHOT # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - export STREAM_APPS_RT_VERSION=2024.0.0 + export STREAM_APPS_RT_VERSION=2022.0.0 fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" if [[ "$STREAM_APPS_RT_VERSION" = *"SNAPSHOT"* ]]; then diff --git a/src/deploy/k8s/register-apps.sh b/src/deploy/k8s/register-apps.sh index 97f9105ac6..ecb8a30016 100755 --- a/src/deploy/k8s/register-apps.sh +++ b/src/deploy/k8s/register-apps.sh @@ -44,9 +44,8 @@ else BROKER_NAME=$BROKER fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - # export STREAM_APPS_RT_VERSION=2022.0.0-SNAPSHOT # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - export STREAM_APPS_RT_VERSION=2024.0.0 + export STREAM_APPS_RT_VERSION=2022.0.0 fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" if [ "$PLATFORM_TYPE" != "kubernetes" ]; then diff --git a/src/local/register-apps.sh b/src/local/register-apps.sh index 42295eaa5e..5f9ac446f0 100755 --- a/src/local/register-apps.sh +++ b/src/local/register-apps.sh @@ -39,10 +39,8 @@ else BROKER_NAME=$BROKER fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - export STREAM_APPS_RT_VERSION=2022.0.0 # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - # export STREAM_APPS_RT_VERSION=2022.0.0 # release for Boot 3.x - export STREAM_APPS_RT_VERSION=2024.0.0 + export STREAM_APPS_RT_VERSION=2022.0.0 fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" TYPE=maven From 9ad7bc2b08e94ff60ec011fec2834ffc00717db5 Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Wed, 28 Aug 2024 13:59:17 +0200 Subject: [PATCH 3/3] Ensure STREAM_APPS_RT_VERSION=2021.1.2 --- src/deploy/carvel/register-apps.sh | 4 ++-- src/deploy/k8s/register-apps.sh | 4 ++-- src/local/register-apps.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/deploy/carvel/register-apps.sh b/src/deploy/carvel/register-apps.sh index 55d5848673..fa5b203885 100755 --- a/src/deploy/carvel/register-apps.sh +++ b/src/deploy/carvel/register-apps.sh @@ -44,8 +44,8 @@ else BROKER_NAME=rabbitmq fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - export STREAM_APPS_RT_VERSION=2022.0.0 + export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x + # export STREAM_APPS_RT_VERSION=2022.0.0 # release for Boot 3.x apps fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" if [[ "$STREAM_APPS_RT_VERSION" = *"SNAPSHOT"* ]]; then diff --git a/src/deploy/k8s/register-apps.sh b/src/deploy/k8s/register-apps.sh index ecb8a30016..f5bec5f540 100755 --- a/src/deploy/k8s/register-apps.sh +++ b/src/deploy/k8s/register-apps.sh @@ -44,8 +44,8 @@ else BROKER_NAME=$BROKER fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - export STREAM_APPS_RT_VERSION=2022.0.0 + export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x + # export STREAM_APPS_RT_VERSION=2022.0.0 # release for Boot 3.x apps fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" if [ "$PLATFORM_TYPE" != "kubernetes" ]; then diff --git a/src/local/register-apps.sh b/src/local/register-apps.sh index 5f9ac446f0..873e9866db 100755 --- a/src/local/register-apps.sh +++ b/src/local/register-apps.sh @@ -39,8 +39,8 @@ else BROKER_NAME=$BROKER fi if [ "$STREAM_APPS_RT_VERSION" = "" ]; then - # export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x - export STREAM_APPS_RT_VERSION=2022.0.0 + export STREAM_APPS_RT_VERSION=2021.1.2 # release for Boot 2.x + # export STREAM_APPS_RT_VERSION=2022.0.0 # release for Boot 3.x apps fi echo "STREAM_APPS_RT_VERSION=$STREAM_APPS_RT_VERSION" TYPE=maven