Implement Acceptance e2e tests for BTP user journey

.github/actions/ensure-pack/action.yaml

@@ -0,0 +1,13 @@
+name: 'Ensure pack'
+description: 'Action for ensuring pack CLI is installed on the agent'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: ensure pack
+      run: |
+        if ! which pack; then
+          echo "Installing pack..."
+          (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.36.0/pack-v0.36.0-linux.tgz" | sudo tar -C /usr/local/bin/ --no-same-owner -xzv pack)
+        fi
+      shell: bash

.github/workflows/btp-integration-tests.yml

@@ -0,0 +1,73 @@
+name: btp integration tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'release-*'
+
+jobs:
+  e2e-test-btp:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - uses: ./.github/actions/setup-go
+      - uses: ./.github/actions/ensure-pack
+      - name: compute github commit sha
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: build kyma CLI binary
+        run: make build
+      - uses: kyma-project/terraform-module/.github/actions/create-sap-btp-kyma@0635a02ed7c63944f2e544d27603f3d2c72fee2b
+        id: create-btp-resources
+        with:
+          btp_subaccount_name: cli-test-${{ steps.vars.outputs.sha_short }}-${{ github.run_attempt }}
+          btp_backend_url: '${{ secrets.BTP_BACKEND_URL}}'
+          btp_user: '${{ secrets.BTP_BOT_USER}}'
+          btp_password: '${{ secrets.BTP_BOT_PASSWORD}}'
+          btp_global_account: '${{ secrets.BTP_GLOBAL_ACCOUNT }}'
+          btp_idp_tenant: '${{ secrets.BTP_CUSTOM_IAS_TENANT }}'
+          btp_subaccount_region: '${{ secrets.BTP_SUBACCOUNT_REGION }}'
+          btp_kyma_region: '${{ secrets.BTP_KYMA_REGION }}'
+          btp_kyma_plan: '${{ secrets.BTP_KYMA_PLAN }}'
+      - name: ensure btp prerequisites 
+        run: |
+          echo -e "BTP_GLOBAL_ACCOUNT = \"$BTP_GLOBAL_ACCOUNT\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_BOT_USER = \"$BTP_BOT_USER\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_BOT_PASSWORD = \"$BTP_BOT_PASSWORD\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_BACKEND_URL = \"$BTP_BACKEND_URL\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_CUSTOM_IAS_TENANT = \"$BTP_CUSTOM_IAS_TENANT\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_KYMA_SUBACCOUNT_ID = \"$BTP_KYMA_SUBACCOUNT_ID\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_OBJECTSTORE_SUBACCOUNT_ID = \"$BTP_OBJECTSTORE_SUBACCOUNT_ID\"" >> tests/btp/tf/.tfvars
+          echo -e "BTP_HANA_SUBACCOUNT_ID = \"$BTP_HANA_SUBACCOUNT_ID\"" >> tests/btp/tf/.tfvars
+          make -C tests/btp prerequisites
+        env:
+          BTP_BACKEND_URL: '${{ secrets.BTP_BACKEND_URL}}'
+          BTP_BOT_USER: '${{ secrets.BTP_BOT_USER}}'
+          BTP_BOT_PASSWORD: '${{ secrets.BTP_BOT_PASSWORD}}'
+          BTP_CUSTOM_IAS_TENANT: '${{ secrets.BTP_CUSTOM_IAS_TENANT }}'
+          BTP_GLOBAL_ACCOUNT: '${{ secrets.BTP_GLOBAL_ACCOUNT }}'
+          BTP_KYMA_SUBACCOUNT_ID: '${{ steps.create-btp-resources.outputs.subaccount_id }}'
+          BTP_OBJECTSTORE_SUBACCOUNT_ID: '${{ secrets.BTP_OBJECTSTORE_SUBACCOUNT_ID }}'
+          BTP_HANA_SUBACCOUNT_ID: '${{ secrets.BTP_HANA_SUBACCOUNT_ID }}'
+      - name: test 
+        run: |
+          DOMAIN=$DOMAIN \
+          make -C tests/btp e2e-test
+        env:
+          DOMAIN: '${{ steps.create-btp-resources.outputs.domain }}'
+      - name: cleanup
+        run: |
+          make -C tests/btp cleanup
+      - uses: kyma-project/terraform-module/.github/actions/force-delete-sap-btp-subaccount@0635a02ed7c63944f2e544d27603f3d2c72fee2b
+        if: always()
+        with:
+          btp_subaccount_id: ${{ steps.create-btp-resources.outputs.subaccount_id }}
+          btp_backend_url: ${{ secrets.BTP_BACKEND_URL}}
+          btp_user: ${{ secrets.BTP_BOT_USER}}
+          btp_password: ${{ secrets.BTP_BOT_PASSWORD}}
+          btp_global_account: ${{ secrets.BTP_GLOBAL_ACCOUNT }}
+          btp_idp_tenant: ${{ secrets.BTP_CUSTOM_IAS_TENANT }}

.github/workflows/pull.yaml

@@ -28,12 +28,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: ./.github/actions/setup-go
-      - name: ensure pack
-        run: |
-          if ! which pack; then
-            echo "Installing pack..."
-            (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.36.0/pack-v0.36.0-linux.tgz" | sudo tar -C /usr/local/bin/ --no-same-owner -xzv pack)
-          fi
+      - uses: ./.github/actions/ensure-pack
       - name: build kyma binary
         run: make build
       - uses: kyma-project/serverless/.github/actions/create-k3d-cluster@9e8c091842e5c884e1770ef0bcc5b4b4d2894b74

.gitignore

@@ -36,3 +36,15 @@ mod
 /template.yaml
 
 dist/
+tests/btp/sm.env
+tests/btp/hana-admin-api-binding.json
+tests/btp/config.json
+tests/**/node_modules/
+tests/btp/sample-http-db-nodejs/bookstore/package-lock.json
+tests/btp/tf/*.tfvars
+tests/btp/tf/.terraform*
+tests/btp/tf/terraform.tfstate*
+tests/btp/tf/creds.json
+tests/btp/tf/btp-access-credentials-secret.yaml
+tests/btp/tf/hana-admin-creds.json
+tests/btp/k8s-resources/db/books-hdi-initjob.yaml

tests/btp/Makefile

@@ -1,3 +1,12 @@
+.PHONY: prerequisites
+prerequisites: 
+	terraform -chdir=tf init
+	terraform -chdir=tf apply --auto-approve -var-file=.tfvars
+
 .PHONY: e2e-test
-e2e-test: 
-	./integration-test-btp.sh
+e2e-test: |
+	DOMAIN=${DOMAIN} ./integration-test-btp.sh
+
+.PHONY: cleanup
+cleanup:
+	./kyma-cleanup.sh

tests/btp/integration-test-btp.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
+echo -e "\n--------------------------------------------------------------------------------------\n"
 echo "Running kyma integration tests uing connected managed kyma runtime"
 
-# -------------------------------------------------------------------------------------
-echo "Step1: Generating temporary access for new service account"
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step1: Generating temporary access for new service account\n"
 
 ../../bin/kyma alpha kubeconfig generate --clusterrole cluster-admin --serviceaccount test-sa --output /tmp/kubeconfig.yaml --time 2h
 
@@ -12,26 +14,118 @@ if [[ $(kubectl config view --minify --raw | yq '.users[0].name') != 'test-sa' ]
     exit 1
 fi
 echo "Running test in user context of: $(kubectl config view --minify --raw | yq '.users[0].name')"
-# -------------------------------------------------------------------------------------
-echo "Step2: List modules"
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step2: List modules\n"
 ../../bin/kyma alpha module list
 
-# -------------------------------------------------------------------------------------
-echo "Step3: Connecting to remote BTP subaccount"
-# -------------------------------------------------------------------------------------
-echo "Step4: Create Shared Service Instance Reference"
-# -------------------------------------------------------------------------------------
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step3: Connecting to a service manager from remote BTP subaccount\n"
+
+# https://help.sap.com/docs/btp/sap-business-technology-platform/namespace-level-mapping?locale=en-US
+( cd tf ; curl https://raw.githubusercontent.com/kyma-project/btp-manager/main/hack/create-secret-file.sh | bash -s operator remote-service-manager-credentials )
+kubectl create -f tf/btp-access-credentials-secret.yaml || true
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step4: Create service instance reference to a shared object-store service instance\n"
+
+echo "Waiting for CRD btp operator"
+while ! kubectl get crd btpoperators.operator.kyma-project.io; do echo "Waiting for CRD btp operator..."; sleep 1; done
+kubectl wait --for condition=established crd/btpoperators.operator.kyma-project.io
+while ! kubectl get btpoperators.operator.kyma-project.io btpoperator --namespace kyma-system; do echo "Waiting for btpoperator..."; sleep 1; done
+kubectl wait --for condition=Ready btpoperators.operator.kyma-project.io/btpoperator -n kyma-system --timeout=180s
+
+
+# TODO - change after btp operator commands are extracted as btp module cli extension
+../../bin/kyma alpha reference-instance \
+    --btp-secret-name remote-service-manager-credentials \
+    --namespace kyma-system \
+    --offering-name objectstore \
+    --plan-selector standard \
+    --reference-name object-store-reference
+kubectl apply -n kyma-system -f ./k8s-resources/object-store-binding.yaml
+
+while ! kubectl get secret object-store-reference-binding --namespace kyma-system; do echo "Waiting for object-store-reference-binding secret..."; sleep 5; done
+
+
 # Enable Docker Registry
-echo "Step5: Enable Docker Registry from experimental channel (with persistent BTP based storage)"
-../../bin/kyma alpha module add docker-registry --channel experimental --cr-path k8s-resources/exposed-docker-registry.yaml
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step5: Enable Docker Registry from experimental channel (with persistent BTP based storage)\n"
+../../bin/kyma alpha module add docker-registry --channel experimental --cr-path k8s-resources/custom-docker-registry.yaml
 
 echo "..waiting for docker registry"
-kubectl wait --for condition=Installed dockerregistries.operator.kyma-project.io/default -n kyma-system --timeout=360s
-# -------------------------------------------------------------------------------------
-echo "Step6: Map bookstore DB"
-# -------------------------------------------------------------------------------------
-echo "Step7: Push bookstore application (w/o Dockerfile)"
-# -------------------------------------------------------------------------------------
+kubectl wait --for condition=Installed dockerregistries.operator.kyma-project.io/custom-dr -n kyma-system --timeout=360s
+
+sleep 5
+
+dr_external_url=$(../../bin/kyma alpha registry config --externalurl)
+
+# TODO new cli command, for example
+# dr_internal_pull_url=$(../../bin/kyma alpha registry config --internalurl)
+dr_internal_pull_url=$(kubectl get dockerregistries.operator.kyma-project.io -n kyma-system custom-dr -ojsonpath={.status.internalAccess.pullAddress})
+
+../../bin/kyma alpha registry config --output config.json
+
+echo "Docker Registry enabled (URLs: $dr_external_url, $dr_internal_pull_url)"
+echo "config.json for docker CLI access generated"
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step6: Map SAP Hana DB instance with Kyma runtime\n"
+
+../../bin/kyma alpha hana map --credentials-path tf/hana-admin-creds.json
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step7: Pack & push hdi-deploy image\n"
+
+# build hdi-deploy via pack and push it via docker CLI (external url)
+pack build hdi-deploy:latest -p sample-http-db-nodejs/hdi-deploy -B paketobuildpacks/builder:base
+docker tag hdi-deploy:latest $dr_external_url/hdi-deploy:latest
+docker --config . push $dr_external_url/hdi-deploy:latest
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step8: Deploy hdi-deploy (hdi instance & binding, run db initialisation)\n"
+
+echo "Initialising db binding..."
+kubectl set image -f ./k8s-resources/db/books-hdi-initjob-template.yaml bookstore-db=$dr_internal_pull_url/hdi-deploy:latest --local -o yaml > ./k8s-resources/db/books-hdi-initjob.yaml
+kubectl apply -k ./k8s-resources/db
+echo "Waiting for hana-init-job to complete..."
+kubectl wait --for condition=Complete jobs/hana-hdi-initjob --timeout=360s 
+echo "Bookstore db initialised" 
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step9: Pushing bookstore app\n"
+
+# build hdi-deploy via pack and push it via docker CLI (external url)
+pack build bookstore:latest -p sample-http-db-nodejs/bookstore -B paketobuildpacks/builder:base
+docker tag bookstore:latest $dr_external_url/bookstore:latest
+docker --config . push $dr_external_url/bookstore:latest
+
+# TODO check why this fails on GH action with 
+# Importing bookstore:2025-03-13_16-06-26
+# Error:
+#   failed to import image to in-cluster registry
+
+# Error Details:
+#   failed to push image to the in-cluster registry: PUT https://localhost:32137/v2/bookstore/blobs/uploads/c0ae3d32-c861-4ed5-a796-43bf9434b954?_state=REDACTED&digest=sha256%3Acb4197092ed16fbdd56eafda1c0995d527ca3a0621d3b1787a1376e8478d751c: BLOB_UPLOAD_UNKNOWN: blob upload unknown to registry; map[]
+
+
+#../../bin/kyma alpha app push --name bookstore --expose --container-port 3000 --mount-secret hana-hdi-binding --code-path sample-http-db-nodejs/bookstore
+
+../../bin/kyma alpha app push --name bookstore --expose --container-port 3000 --mount-secret hana-hdi-binding --image $dr_internal_pull_url/bookstore:latest --image-pull-secret dockerregistry-config
+
+kubectl wait --for condition=Available deployment bookstore --timeout=60s
+kubectl wait --for='jsonpath={.status.state}=Ready' apirules.gateway.kyma-project.io/bookstore
+
+echo -e "\n--------------------------------------------------------------------------------------\n"
+echo -e "Step10: Verify bookstore app\n"
+sleep 5
+response=$(curl https://bookstore.$DOMAIN/v1/books)
+echo "HTTP response from sample app: $response"
+
+if [[ $response != '[{"id":1,"title":"Dune","author":"Frank Herbert"},{"id":2,"title":"Pippi Goes on Board","author":"Astrid Lindgren"}]' ]]; then
+    exit 1
+fi
 
 
 exit 0

tests/btp/k8s-resources/custom-docker-registry.yaml

@@ -0,0 +1,11 @@
+apiVersion: operator.kyma-project.io/v1alpha1
+kind: DockerRegistry
+metadata:
+  name: custom-dr
+  namespace: kyma-system
+spec:
+  externalAccess:
+    enabled: true
+  storage:
+    btpObjectStore:
+      secretName: object-store-reference-binding

tests/btp/k8s-resources/db/books-hdi-initjob-template.yaml

@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  creationTimestamp: null
+  name: hana-hdi-initjob
+spec:
+  completions: 1
+  manualSelector: false
+  parallelism: 1
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        sidecar.istio.io/inject: "false"
+    spec:
+      containers:
+      - env:
+        - name: SERVICE_BINDING_ROOT
+          value: /bindings
+        - name: EXIT
+          value: "true"
+        - name: EXIT_PROCESS_AFTER_UPLOAD
+          value: "true"
+        image: foo # this will be changed dynamically
+        imagePullPolicy: Always
+        name: bookstore-db
+        resources: {}
+        volumeMounts:
+        - name: hdi
+          mountPath: /bindings/hdi
+          readOnly: true
+      restartPolicy: OnFailure
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+      volumes:
+      - name: hdi
+        secret:
+          secretName: hana-hdi-binding
+      imagePullSecrets:
+        - name: dockerregistry-config
+status: {}

tests/btp/k8s-resources/db/hana-hdi-binding.yaml

@@ -0,0 +1,8 @@
+apiVersion: services.cloud.sap.com/v1
+kind: ServiceBinding
+metadata:
+  name: hana-hdi-binding
+spec:
+  serviceInstanceName: hana-hdi-instance
+  externalName: hana-hdi
+  secretName: hana-hdi-binding

tests/btp/k8s-resources/db/hana-hdi-instance.yaml

@@ -0,0 +1,8 @@
+apiVersion: services.cloud.sap.com/v1
+kind: ServiceInstance
+metadata:
+  name: hana-hdi-instance
+spec:
+  externalName: hana-hdi
+  serviceOfferingName: hana
+  servicePlanName: hdi-shared

tests/btp/k8s-resources/db/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - hana-hdi-instance.yaml
+  - hana-hdi-binding.yaml
+  - books-hdi-initjob.yaml
+

tests/btp/k8s-resources/object-store-binding.yaml

@@ -0,0 +1,8 @@
+apiVersion: services.cloud.sap.com/v1
+kind: ServiceBinding
+metadata:
+  name: object-store-reference-binding
+spec:
+  serviceInstanceName: object-store-reference
+  externalName: object-store-reference-binding
+  secretName: object-store-reference-binding