Improve Carvel Deployment Support. (#5921)

* Improve Carvel Deployment Support.

* Allow kapp-controller to select default service account.

* Provide for specifying allocateLoadBalancerNodePorts and loadBalancerClass on skipper and dataflow server services.

build-carvel-package.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+function create_and_clear() {
+    rm -rf "$1"
+    mkdir -p "$1"
+}
+
+SCDIR=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")"))
+set -euxo pipefail
+pushd $SCDIR > /dev/null
+export DATAFLOW_VERSION=$(./mvnw help:evaluate -o -Dexpression=project.version -q -DforceStdout)
+export SKIPPER_VERSION=$(./mvnw help:evaluate -o -Dexpression=spring-cloud-skipper.version -pl spring-cloud-dataflow-parent -q -DforceStdout)
+
+if [ "$PACKAGE_VERSION" = "" ]; then
+  export PACKAGE_VERSION=$DATAFLOW_VERSION
+fi
+
+# you can launch a local docker registry using docker run -d -p 5000:5000 --name registry registry:2.7
+# export REPO_PREFIX="<local-machine-ip>:5000/"
+if [ "$REPO_PREFIX" = "" ]; then
+  REPO_PREFIX="docker.io/"
+fi
+
+export PACKAGE_BUNDLE_REPOSITORY="${REPO_PREFIX}springcloud/scdf-oss-package"
+export REPOSITORY_BUNDLE="${REPO_PREFIX}springcloud/scdf-oss-repo"
+
+export SKIPPER_REPOSITORY="springcloud/spring-cloud-skipper-server"
+export SERVER_REPOSITORY="springcloud/spring-cloud-dataflow-server"
+export CTR_VERSION=$DATAFLOW_VERSION
+export PACKAGE_NAME="scdf"
+export PACKAGE_BUNDLE_TEMPLATE="src/carvel/templates/bundle/package"
+export IMGPKG_LOCK_TEMPLATE="src/carvel/templates/imgpkg"
+export VENDIR_SRC_IN="src/carvel/config"
+export SERVER_VERSION="$DATAFLOW_VERSION"
+
+export PACKAGE_BUNDLE_GENERATED=/tmp/generated/packagebundle
+export IMGPKG_LOCK_GENERATED_IN=/tmp/generated/imgpkgin
+export IMGPKG_LOCK_GENERATED_OUT=/tmp/generated/imgpkgout
+create_and_clear $PACKAGE_BUNDLE_GENERATED
+create_and_clear $IMGPKG_LOCK_GENERATED_IN
+create_and_clear $IMGPKG_LOCK_GENERATED_OUT
+
+echo "bundle-path=$PACKAGE_BUNDLE_GENERATED"
+export SCDF_DIR="$SCDIR"
+
+sh "$SCDIR/.github/actions/build-package-bundle/build-package-bundle.sh"
+
+imgpkg push --bundle "$PACKAGE_BUNDLE_REPOSITORY:$PACKAGE_VERSION" --file "$PACKAGE_BUNDLE_GENERATED"
+
+export REPO_BUNDLE_TEMPLATE="src/carvel/templates/bundle/repo"
+
+export REPO_BUNDLE_RENDERED=/tmp/generated/reporendered
+export REPO_BUNDLE_GENERATED=/tmp/generated/repobundle
+create_and_clear $REPO_BUNDLE_RENDERED
+create_and_clear $REPO_BUNDLE_GENERATED
+
+sh "$SCDIR/.github/actions/build-repository-bundle/build-repository-bundle.sh"
+
+imgpkg push --bundle "$REPOSITORY_BUNDLE:$PACKAGE_VERSION" --file "$REPO_BUNDLE_GENERATED"
+
+popd

build-containers.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+./mvnw install -s .settings.xml -DskipTests -T 1C -am -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server,:spring-cloud-dataflow-composed-task-runner
+./mvnw spring-boot:build-image -s .settings.xml -DskipTests -T 1C -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server,:spring-cloud-dataflow-composed-task-runner

src/carvel/config/dataflow-svc.yml

@@ -1,6 +1,9 @@
 #@ load("@ytt:data", "data")
 #@ load("dataflow.star", "service_spec_type")
-
+#@ load("dataflow.star", "service_spec_type_loadbalancer")
+#@ load("dataflow.star", "service_spec_allocate_load_balancer_node_ports")
+#@ load("dataflow.star", "has_service_spec_load_balancer_class")
+#@ load("dataflow.star", "service_spec_load_balancer_class")
 kind: Service
 apiVersion: v1
 metadata:
@@ -18,5 +21,11 @@ spec:
   - port: 80
     targetPort: 9393
     name: scdf-server
+  #@ if service_spec_type_loadbalancer():
+  allocateLoadBalancerNodePorts: #@ service_spec_allocate_load_balancer_node_ports()
+    #@ if has_service_spec_load_balancer_class():
+  loadBalancerClass: #@ service_spec_load_balancer_class()
+    #@ end
+  #@ end
   selector:
     app: scdf-server

src/carvel/config/dataflow.star

@@ -82,11 +82,30 @@ end
 def image_pull_secrets():
   return [{"name": registry_secret_ref()}]
 end
+def has_service_spec_type():
+  return non_empty_string(data.values.scdf.server.service.type)
+end
+
+def service_spec_type_loadbalancer():
+  return non_empty_string(data.values.scdf.server.service.type) and data.values.scdf.server.service.type == 'LoadBalancer'
+end
 
 def service_spec_type():
   return data.values.scdf.server.service.type
 end
 
+def service_spec_allocate_load_balancer_node_ports():
+  return data.values.scdf.server.service.allocateLoadBalancerNodePorts
+end
+
+def has_service_spec_load_balancer_class():
+  return non_empty_string(data.values.scdf.server.service.loadBalancerClass)
+end
+
+def service_spec_load_balancer_class():
+  return data.values.scdf.server.service.loadBalancerClass
+end
+
 def context_path():
   return data.values.scdf.server.contextPath
 end

src/carvel/config/skipper-svc.yml

@@ -1,5 +1,9 @@
 #@ load("@ytt:data", "data")
 #@ load("skipper.star", "service_spec_type")
+#@ load("skipper.star", "service_spec_type_loadbalancer")
+#@ load("skipper.star", "service_spec_allocate_load_balancer_node_ports")
+#@ load("skipper.star", "has_service_spec_load_balancer_class")
+#@ load("skipper.star", "service_spec_load_balancer_class")
 
 apiVersion: v1
 kind: Service
@@ -16,5 +20,11 @@ spec:
   ports:
   - port: 80
     targetPort: 7577
+  #@ if service_spec_type_loadbalancer():
+  allocateLoadBalancerNodePorts: #@ service_spec_allocate_load_balancer_node_ports()
+    #@ if has_service_spec_load_balancer_class():
+  loadBalancerClass: #@ service_spec_load_balancer_class()
+    #@ end
+  #@ end
   selector:
     app: skipper

src/carvel/config/skipper.star

@@ -90,6 +90,23 @@ def service_spec_type():
   return data.values.scdf.skipper.service.type
 end
 
+def service_spec_type_loadbalancer():
+  return non_empty_string(data.values.scdf.skipper.service.type) and data.values.scdf.skipper.service.type == 'LoadBalancer'
+end
+
+def service_spec_allocate_load_balancer_node_ports():
+  return data.values.scdf.skipper.service.allocateLoadBalancerNodePorts
+  end
+
+def has_service_spec_load_balancer_class():
+  return non_empty_string(data.values.scdf.skipper.service.loadBalancerClass)
+  end
+
+def service_spec_load_balancer_class():
+  return data.values.scdf.skipper.service.loadBalancerClass
+end
+
 def skipper_has_password():
   return non_empty_string(data.values.scdf.skipper.database.password)
-end
+end
+

src/carvel/config/values/values.yml

@@ -15,6 +15,8 @@ scdf:
       digest: ""
     service:
       type: ClusterIP
+      allocateLoadBalancerNodePorts: true
+      loadBalancerClass: ""
     resources:
       limits:
         cpu: ""
@@ -59,6 +61,8 @@ scdf:
       digest: ""
     service:
       type: ClusterIP
+      allocateLoadBalancerNodePorts: true
+      loadBalancerClass: ""
     resources:
       limits:
         cpu: ""

src/carvel/templates/bundle/repo/values-schema.yml

@@ -79,6 +79,12 @@ components:
                         - ClusterIP
                         - ExternalName
                       description: Service type
+                    allocateLoadBalancerNodePorts:
+                      type: boolean
+                      description: Indicates if load balancer should create node ports. Default is true
+                    loadBalancerClass:
+                      type: string
+                      description: Determines a specific configured type of load balancer.
                 resources:
                   type: object
                   properties:
@@ -185,6 +191,12 @@ components:
                         - ClusterIP
                         - ExternalName
                       description: Service type
+                    allocateLoadBalancerNodePorts:
+                      type: boolean
+                      description: Indicates if load balancer should create node ports. Default is true
+                    loadBalancerClass:
+                      type: string
+                      description: Determines a specific configured type of load balancer.
                 resources:
                   type: object
                   properties:

src/carvel/test/ordering.test.ts

@@ -29,7 +29,6 @@ describe('ordering', () => {
     expect(findAnnotation(skipperDeployment, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/skipper');
     expect(findAnnotation(dataflowService, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/server');
     expect(findAnnotation(dataflowDeployment, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/server');
-
     expect(findAnnotations(dataflowService, 'kapp.k14s.io/change-rule')).toContainAnyValues([
       'upsert after upserting scdf.tanzu.vmware.com/skipper'
     ]);

src/carvel/test/servers.test.ts

@@ -37,19 +37,25 @@ describe('servers', () => {
       dataValueYamls: [
         ...DEFAULT_REQUIRED_DATA_VALUES,
         'scdf.server.service.type=LoadBalancer',
-        'scdf.skipper.service.type=LoadBalancer'
+        'scdf.server.service.allocateLoadBalancerNodePorts=false',
+        'scdf.skipper.service.type=ClusterIP',
+        'scdf.skipper.service.allocateLoadBalancerNodePorts=true'
       ]
     });
     expect(result.success, result.stderr).toBeTruthy();
     const yaml = result.stdout;
 
     const dataflowService = findService(yaml, SCDF_SERVER_NAME);
+    console.log(dataflowService);
     expect(dataflowService).toBeTruthy();
     expect(dataflowService?.spec?.type).toBe('LoadBalancer');
 
+    expect(dataflowService?.spec?.allocateLoadBalancerNodePorts).toBe(false);
+
     const skipperService = findService(yaml, SKIPPER_NAME);
     expect(skipperService).toBeTruthy();
-    expect(skipperService?.spec?.type).toBe('LoadBalancer');
+    expect(skipperService?.spec?.type).toBe('ClusterIP');
+    expect(skipperService?.spec?.allocateLoadBalancerNodePorts).toBeFalsy();
   });
 
   it('should have tagged images', async () => {

src/deploy/carvel/carvel-add-package.sh

@@ -15,6 +15,7 @@ fi
 check_env NS
 check_env PACKAGE
 check_env PACKAGE_NAME
+
 echo "Adding $PACKAGE as $PACKAGE_NAME in $NS"
 
 if [ "$DEBUG" = "true" ]; then
@@ -23,5 +24,27 @@ else
     ARGS=""
 fi
 echo "Creating $PACKAGE_NAME for $PACKAGE"
-kctrl package repository add --namespace $NS --repository $PACKAGE_NAME --url $PACKAGE --yes --wait --wait-check-interval 10s $ARGS
+if [ "$REPO_SECRET_REF" = "" ]; then
+  if [[ "$PACKAGE_NAME" == *"pro"* ]]; then
+    REPO_SECRET_REF=reg-creds-dev-registry
+  else
+    REPO_SECRET_REF=reg-creds-dockerhub
+  fi
+fi
+
+echo "Using secretRef=$REPO_SECRET_REF in $PACKAGE_NAME for $PACKAGE"
+
+kubectl apply --namespace $NS -f -  <<EOF
+apiVersion: packaging.carvel.dev/v1alpha1
+kind: PackageRepository
+metadata:
+  name: $PACKAGE_NAME
+spec:
+  fetch:
+    imgpkgBundle:
+      image: $PACKAGE
+      secretRef:
+        name: $REPO_SECRET_REF
+EOF
+kctrl package repository kick --namespace $NS --repository $PACKAGE_NAME --yes --wait --wait-check-interval 10s
 kctrl package repository list --namespace $NS

src/deploy/carvel/carvel-add-registry-secret.sh

@@ -30,7 +30,6 @@ function create_secret() {
 #            --docker-password="$4" \
 #            --namespace "$NS"
     echo "Annotating $SCRT_NAME for image-pull-secret"
-    # kubectl annotate secret "$SCRT_NAME" --namespace "$SCRT_NS"  secretgen.carvel.dev/image-pull-secret=""
 #    kubectl annotate secret "$1" --namespace "$NS"  secretgen.carvel.dev/image-pull-secret=""
     echo "Exporting $SCRT_NAME from $SCRT_NS"
     kubectl apply -f - <<EOF
@@ -40,7 +39,8 @@ metadata:
   name: ${SCRT_NAME}
   namespace: ${SCRT_NS}
 spec:
-  toNamespace: '*'
+  toNamespaces:
+    - "*"
 EOF
 
 }

src/deploy/carvel/carvel-deploy-package.sh

@@ -25,8 +25,6 @@ if [ "$5" != "" ]; then
 fi
 if [ "$6" != "" ]; then
     SA=$6
-else
-    SA=scdf-sa
 fi
 if [ ! -f "$VALUES_FILE" ]; then
     echo "Cannot find $VALUES_FILE"
@@ -43,12 +41,13 @@ if [ "$DEBUG" = "true" ]; then
 else
     ARGS=""
 fi
-if [ "$SA" = "" ]; then
-    SA=scdf-sa
+SA_ARGS=
+if [ "$SA" != "" ]; then
+    SA_ARGS="--service-account-name $SA"
 fi
 echo "Installing $APP_NAME from $PACKAGE_NAME:$PACKAGE_VERSION"
+# $SA_ARGS
 kctrl package install --package-install "$APP_NAME" \
-  --service-account-name "$SA" \
   --package "$PACKAGE_NAME" \
   --values-file "$VALUES_FILE" \
   --version "$PACKAGE_VERSION" --namespace "$NS" --yes \

src/deploy/carvel/carvel-import-secret.sh

@@ -26,30 +26,30 @@ if [ "$NAMESPACE" = "" ]; then
     exit 2
 fi
 if [ "$IMPORT_TYPE" == "import" ]; then
-kubectl apply -f -  <<EOF
+  echo "Creating SecretImport $SECRET_NAME from $FROM_NAMESPACE to $NAMESPACE"
+  kubectl apply --namespace $NAMESPACE -f -  <<EOF
 apiVersion: secretgen.carvel.dev/v1alpha1
 kind: SecretImport
 metadata:
   name: $SECRET_NAME
-  namespace: $NAMESPACE
 spec:
   fromNamespace: $FROM_NAMESPACE
 EOF
-echo "Created SecretImport $SECRET_NAME from $FROM_NAMESPACE to $NAMESPACE"
+  echo "Created SecretImport $SECRET_NAME from $FROM_NAMESPACE to $NAMESPACE"
 else
-kubectl apply -f -  <<EOF
+  echo "Creating Placeholder Secret $SECRET_NAME in $NAMESPACE"
+  kubectl apply --namespace $NAMESPACE -f - <<EOF
 apiVersion: v1
 kind: Secret
 metadata:
   name: $SECRET_NAME
-  namespace: $NAMESPACE
   annotations:
     secretgen.carvel.dev/image-pull-secret: ""
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: e30K
 EOF
-echo "Created Placeholder Secret $SECRET_NAME in $NAMESPACE"
+  echo "Created Placeholder Secret $SECRET_NAME in $NAMESPACE"
 fi
 
 if [ "$DEBUG" = "true" ]; then