diff --git a/.github/workflows/main-integration.yaml b/.github/workflows/main-integration.yaml index 4b6147cb2..ab1e826a8 100644 --- a/.github/workflows/main-integration.yaml +++ b/.github/workflows/main-integration.yaml @@ -84,7 +84,7 @@ jobs: with: manager_image: "europe-docker.pkg.dev/kyma-project/prod/api-gateway-manager:${{ needs.wait-for-image-build.outputs.sha }}" gardener_secret: ${{ secrets.GARDENER_TOKEN }} - script: ./tests/integration/scripts/custom-domain-gardener-gcp-gh.sh + script: ./hack/ci/custom-domain-gardener-gcp.sh client_id: ${{ secrets.CLIENT_ID }} client_secret: ${{ secrets.CLIENT_SECRET }} oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration" @@ -103,7 +103,7 @@ jobs: with: manager_image: "europe-docker.pkg.dev/kyma-project/prod/api-gateway-manager:${{ needs.wait-for-image-build.outputs.sha }}" gardener_secret: ${{ secrets.GARDENER_TOKEN }} - script: ./tests/integration/scripts/custom-domain-gardener-aws-gh.sh + script: ./hack/ci/custom-domain-gardener-aws.sh client_id: ${{ secrets.CLIENT_ID }} client_secret: ${{ secrets.CLIENT_SECRET }} oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration" diff --git a/.github/workflows/pull-integration-release.yaml b/.github/workflows/pull-integration-release.yaml index 87ef805f4..2da85addc 100644 --- a/.github/workflows/pull-integration-release.yaml +++ b/.github/workflows/pull-integration-release.yaml @@ -97,7 +97,7 @@ jobs: with: manager_image: "europe-docker.pkg.dev/kyma-project/dev/api-gateway-manager:PR-${{github.event.number}}" gardener_secret: ${{ secrets.GARDENER_TOKEN }} - script: ./tests/integration/scripts/custom-domain-gardener-gcp-gh.sh + script: ./hack/ci/custom-domain-gardener-gcp.sh client_id: ${{ secrets.CLIENT_ID }} client_secret: ${{ secrets.CLIENT_SECRET }} oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration" @@ -116,7 +116,7 @@ jobs: with: manager_image: "europe-docker.pkg.dev/kyma-project/dev/api-gateway-manager:PR-${{github.event.number}}" gardener_secret: ${{ secrets.GARDENER_TOKEN }} - script: ./tests/integration/scripts/custom-domain-gardener-aws-gh.sh + script: ./hack/ci/custom-domain-gardener-aws.sh client_id: ${{ secrets.CLIENT_ID }} client_secret: ${{ secrets.CLIENT_SECRET }} oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration" diff --git a/.github/workflows/pull-unit-lint.yaml b/.github/workflows/pull-unit-lint.yaml index b6fb18f29..e5d0d0723 100644 --- a/.github/workflows/pull-unit-lint.yaml +++ b/.github/workflows/pull-unit-lint.yaml @@ -40,5 +40,5 @@ jobs: PULL_PULL_SHA=${{ github.event.pull_request.head.sha}} \ PULL_BASE_SHA=${{ github.event.pull_request.base.sha}} \ PULL_NUMBER=${{ github.event.number }} \ - ./tests/integration/scripts/code-coverage-guard.sh + ./hack/ci/code-coverage-guard.sh diff --git a/tests/integration/scripts/code-coverage-guard.sh b/hack/ci/code-coverage-guard.sh similarity index 97% rename from tests/integration/scripts/code-coverage-guard.sh rename to hack/ci/code-coverage-guard.sh index 61bdfa498..335ee46a1 100755 --- a/tests/integration/scripts/code-coverage-guard.sh +++ b/hack/ci/code-coverage-guard.sh @@ -17,7 +17,7 @@ if [ -z "$PULL_BASE_SHA" ]; then fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" || exit; pwd)" -cd "${SCRIPT_DIR}/../../../" || exit 1 +cd "${SCRIPT_DIR}/../../" || exit 1 echo -e "Code Coverage Guard - Ensures PRs do not lower code coverage\\n" echo -e "Running tests on: PR-${PULL_NUMBER} (${PULL_PULL_SHA})\\n" diff --git a/tests/integration/scripts/custom-domain-gardener-aws-gh.sh b/hack/ci/custom-domain-gardener-aws.sh similarity index 90% rename from tests/integration/scripts/custom-domain-gardener-aws-gh.sh rename to hack/ci/custom-domain-gardener-aws.sh index 9d3f5d7aa..2774f7399 100755 --- a/tests/integration/scripts/custom-domain-gardener-aws-gh.sh +++ b/hack/ci/custom-domain-gardener-aws.sh @@ -17,4 +17,4 @@ export GARDENER_PROVIDER_SECRET_NAME="aws-gardener-access" export GARDENER_PROJECT_NAME="goats" export GARDENER_CLUSTER_VERSION="1.27.8" -./tests/integration/scripts/custom-domain-gardener-gh.sh +./hack/ci/custom-domain-gardener.sh diff --git a/tests/integration/scripts/custom-domain-gardener-gcp-gh.sh b/hack/ci/custom-domain-gardener-gcp.sh similarity index 90% rename from tests/integration/scripts/custom-domain-gardener-gcp-gh.sh rename to hack/ci/custom-domain-gardener-gcp.sh index 8531cada1..e6007c1ba 100755 --- a/tests/integration/scripts/custom-domain-gardener-gcp-gh.sh +++ b/hack/ci/custom-domain-gardener-gcp.sh @@ -17,4 +17,4 @@ export GARDENER_PROVIDER_SECRET_NAME="goat" export GARDENER_PROJECT_NAME="goats" export GARDENER_CLUSTER_VERSION="1.27.8" -./tests/integration/scripts/custom-domain-gardener-gh.sh +./hack/ci/custom-domain-gardener.sh diff --git a/tests/integration/scripts/custom-domain-gardener-gh.sh b/hack/ci/custom-domain-gardener.sh similarity index 98% rename from tests/integration/scripts/custom-domain-gardener-gh.sh rename to hack/ci/custom-domain-gardener.sh index d5e624fe1..551992997 100755 --- a/tests/integration/scripts/custom-domain-gardener-gh.sh +++ b/hack/ci/custom-domain-gardener.sh @@ -47,7 +47,7 @@ export PATH="${PATH}:${PWD}" CLUSTER_NAME=ag-$(echo $RANDOM | md5sum | head -c 7) export CLUSTER_NAME -./tests/integration/scripts/provision-gardener-gh.sh +./hack/ci/provision-gardener-gh.sh echo "waiting for Gardener to finish shoot reconcile..." kubectl wait --kubeconfig "${GARDENER_KUBECONFIG}" --for=jsonpath='{.status.lastOperation.state}'=Succeeded --timeout=600s "shoots/${CLUSTER_NAME}" diff --git a/tests/integration/scripts/provision-gardener-gh.sh b/hack/ci/provision-gardener.sh similarity index 100% rename from tests/integration/scripts/provision-gardener-gh.sh rename to hack/ci/provision-gardener.sh diff --git a/tests/integration/scripts/shoot_aws.yaml b/hack/ci/shoot_aws.yaml similarity index 100% rename from tests/integration/scripts/shoot_aws.yaml rename to hack/ci/shoot_aws.yaml diff --git a/tests/integration/scripts/shoot_gcp.yaml b/hack/ci/shoot_gcp.yaml similarity index 100% rename from tests/integration/scripts/shoot_gcp.yaml rename to hack/ci/shoot_gcp.yaml diff --git a/tests/integration/scripts/README.md b/tests/integration/scripts/README.md deleted file mode 100644 index c7a9ff206..000000000 --- a/tests/integration/scripts/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Scripts for API Gateway Integration Tests - -Scripts are based on the gardener provision scripts and utilities from `test-infra`: -https://github.com/kyma-project/test-infra/tree/main/prow/scripts - -## Structure - -- `gardener`: scripts for generic gardener de-/provision clusters and IaaS specifics in additional script files -- `jobguard`: job guard script that executes `/prowutils/jobguard` utility from `test-infra/kyma-integration` image, waiting for image build job to complete -- `lib`: general utility functions for logging, kyma cli install, etc. - -## Entrypoint for Integration Tests on Gardener - -API Gateway integration tests are triggered on `presubmit` (PR) and `postsubmit` (main branch) by executing `integration-gardener.sh` script, which provisions a Gardener cluster and starts the tests in `integration-tests.sh`. - -All required environment variables are provided by Prow. diff --git a/tests/integration/scripts/custom-domain-gardener-aws.sh b/tests/integration/scripts/custom-domain-gardener-aws.sh deleted file mode 100755 index 8758917d0..000000000 --- a/tests/integration/scripts/custom-domain-gardener-aws.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# -##Description: This scripts installs and tests api-gateway custom domain test as well as gateway test using the CLI on a real Gardener AWS cluster. -## exit on error, and raise error when variable is not set when used - -set -e - -export MACHINE_TYPE="m5.xlarge" -export DISK_SIZE=50 -export DISK_TYPE="gp2" -export SCALER_MAX=3 -export SCALER_MIN=1 -export GARDENER_PROVIDER="aws" -export GARDENER_REGION="eu-north-1" -export GARDENER_ZONES="eu-north-1b,eu-north-1c,eu-north-1a" - -./tests/integration/scripts/custom-domain-gardener.sh diff --git a/tests/integration/scripts/custom-domain-gardener-gcp.sh b/tests/integration/scripts/custom-domain-gardener-gcp.sh deleted file mode 100755 index e436566df..000000000 --- a/tests/integration/scripts/custom-domain-gardener-gcp.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# -##Description: This scripts installs and tests api-gateway custom domain test as well as gateway test using the CLI on a real Gardener GCP cluster. -## exit on error, and raise error when variable is not set when used - -set -e - -export MACHINE_TYPE="n2-standard-4" -export DISK_SIZE=50 -export DISK_TYPE="pd-standard" -export SCALER_MAX=3 -export SCALER_MIN=1 -export GARDENER_PROVIDER="gcp" -export GARDENER_REGION="europe-west3" -export GARDENER_ZONES="europe-west3-c,europe-west3-b,europe-west3-a" - -./tests/integration/scripts/custom-domain-gardener.sh diff --git a/tests/integration/scripts/custom-domain-gardener.sh b/tests/integration/scripts/custom-domain-gardener.sh deleted file mode 100755 index 5323e9f0f..000000000 --- a/tests/integration/scripts/custom-domain-gardener.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash - -# -##Description: This scripts installs and tests api-gateway custom domain test as well as gateway test using the CLI on a real Gardener GCP cluster. -## exit on error, and raise error when variable is not set when used -## IMG env variable expected (for make deploy), which points to the image in the registry - -set -euo pipefail - -function check_required_vars() { - local requiredVarMissing=false - for var in "$@"; do - if [ -z "${var}" ]; then - >&2 echo "Environment variable ${var} is required but not set" - requiredVarMissing=true - fi - done - if [ "${requiredVarMissing}" = true ] ; then - exit 2 - fi -} - -requiredVars=( - GARDENER_KYMA_PROW_KUBECONFIG - GARDENER_KYMA_PROW_PROJECT_NAME -) - -check_required_vars "${requiredVars[@]}" - -function cleanup() { - kubectl annotate shoot "${CLUSTER_NAME}" confirmation.gardener.cloud/deletion=true \ - --overwrite \ - -n "garden-${GARDENER_KYMA_PROW_PROJECT_NAME}" \ - --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" - - kubectl delete shoot "${CLUSTER_NAME}" \ - --wait="false" \ - --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" \ - -n "garden-${GARDENER_KYMA_PROW_PROJECT_NAME}" -} - -# Cleanup on exit, be it successful or on fail -trap cleanup EXIT INT - -# Add pwd to path to be able to use binaries downloaded in scripts -export PATH="${PATH}:${PWD}" - -# wait for build job -./tests/integration/scripts/jobguard.sh - -CLUSTER_NAME=ag-$(echo $RANDOM | md5sum | head -c 7) -export CLUSTER_NAME -./tests/integration/scripts/provision-gardener.sh - -echo "waiting for Gardener to finish shoot reconcile..." -kubectl wait --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" --for=jsonpath='{.status.lastOperation.state}'=Succeeded --timeout=600s "shoots/${CLUSTER_NAME}" - -cat < patch.yaml -spec: - extensions: - - type: shoot-dns-service - providerConfig: - apiVersion: service.dns.extensions.gardener.cloud/v1alpha1 - dnsProviderReplication: - enabled: true - kind: DNSConfig - syncProvidersFromShootSpecDNS: true - - type: shoot-cert-service - providerConfig: - apiVersion: service.cert.extensions.gardener.cloud/v1alpha1 - kind: CertConfig - shootIssuers: - enabled: true -EOF - -kubectl patch shoot "${CLUSTER_NAME}" --patch-file patch.yaml --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" - -make install-istio -make deploy - -echo "waiting for Gardener to finish shoot reconcile..." -kubectl wait --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" --for=jsonpath='{.status.lastOperation.state}'=Succeeded --timeout=600s "shoots/${CLUSTER_NAME}" - -# KYMA_DOMAIN is required by the tests -export TEST_DOMAIN="${CLUSTER_NAME}.${GARDENER_KYMA_PROW_PROJECT_NAME}.shoot.live.k8s-hana.ondemand.com" -export TEST_CUSTOM_DOMAIN="a.build.kyma-project.io" - -make test-custom-domain - -# Run gateway tests in the same script -export IS_GARDENER=true - -make test-integration-gateway diff --git a/tests/integration/scripts/jobguard.sh b/tests/integration/scripts/jobguard.sh deleted file mode 100755 index c5fc88356..000000000 --- a/tests/integration/scripts/jobguard.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -e - -JOB_NAME_PATTERN=${JOB_NAME_PATTERN:-"(post-.*-build)|(pull-.*-build)"} -TIMEOUT=${JOBGUARD_TIMEOUT:-"10m"} - -if [[ "${BUILD_TYPE}" == "pr" ]]; then - BASE_REF=${PULL_PULL_SHA} -else - BASE_REF=${PULL_BASE_SHA} -fi - -args=( - "-github-endpoint=http://ghproxy" - "-github-endpoint=https://api.github.com" - "-github-token-path=/etc/github/token" - "-fail-on-no-contexts=false" - "-timeout=$TIMEOUT" - "-org=$REPO_OWNER" - "-repo=$REPO_NAME" - "-base-ref=$BASE_REF" - "-expected-contexts-regexp=$JOB_NAME_PATTERN" -) - -jobguard "${args[@]}" diff --git a/tests/integration/scripts/provision-gardener.sh b/tests/integration/scripts/provision-gardener.sh deleted file mode 100755 index 8bb44054e..000000000 --- a/tests/integration/scripts/provision-gardener.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# -##Description: This script provisions a Gardener cluster with config specified in environmental variables - -set -euo pipefail - -function check_required_vars() { - local requiredVarMissing=false - for var in "$@"; do - if [ -z "${var}" ]; then - >&2 echo "Environment variable ${var} is required but not set" - requiredVarMissing=true - fi - done - if [ "${requiredVarMissing}" = true ] ; then - exit 2 - fi -} - -requiredVars=( - CLUSTER_NAME - GARDENER_PROVIDER - GARDENER_REGION - GARDENER_ZONES - GARDENER_KYMA_PROW_KUBECONFIG - GARDENER_KYMA_PROW_PROJECT_NAME - GARDENER_KYMA_PROW_PROVIDER_SECRET_NAME - GARDENER_CLUSTER_VERSION - MACHINE_TYPE - DISK_SIZE - DISK_TYPE - SCALER_MAX - SCALER_MIN -) - -check_required_vars "${requiredVars[@]}" - -# Install Kyma CLI in latest version -echo "--> Install kyma CLI locally to /tmp/bin" -curl -Lo kyma.tar.gz "https://github.com/kyma-project/cli/releases/latest/download/kyma_linux_x86_64.tar.gz" \ -&& tar -zxvf kyma.tar.gz && chmod +x kyma \ -&& rm -f kyma.tar.gz - -kyma version --client -kyma provision gardener ${GARDENER_PROVIDER} \ - --secret "${GARDENER_KYMA_PROW_PROVIDER_SECRET_NAME}" \ - --name "${CLUSTER_NAME}" \ - --project "${GARDENER_KYMA_PROW_PROJECT_NAME}" \ - --credentials "${GARDENER_KYMA_PROW_KUBECONFIG}" \ - --region "${GARDENER_REGION}" \ - --zones "${GARDENER_ZONES}" \ - --type "${MACHINE_TYPE}" \ - --disk-size $DISK_SIZE \ - --disk-type "${DISK_TYPE}" \ - --scaler-max $SCALER_MAX \ - --scaler-min $SCALER_MIN \ - --kube-version="${GARDENER_CLUSTER_VERSION}" \ - --attempts 3 \ - --verbose \ No newline at end of file diff --git a/tests/integration/scripts/test-gardener-aws.sh b/tests/integration/scripts/test-gardener-aws.sh deleted file mode 100755 index 363cc9877..000000000 --- a/tests/integration/scripts/test-gardener-aws.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -#Description: This scripts installs and test Kyma using the CLI on a real Gardener AWS cluster. -# -#Expected common vars: -# - GARDENER_REGION - Gardener compute region -# - GARDENER_ZONES - Gardener compute zones inside the region -# - GARDENER_CLUSTER_VERSION - Version of the Kubernetes cluster -# - GARDENER_KYMA_PROW_KUBECONFIG - Kubeconfig of the Gardener service account -# - GARDENER_KYMA_PROW_PROJECT_NAME - Name of the gardener project where the cluster will be integrated. -# - GARDENER_KYMA_PROW_PROVIDER_SECRET_NAME - Name of the secret configured in the gardener project to access the cloud provider -# - -# exit on error, and raise error when variable is not set when used -set -e - -export MACHINE_TYPE="m5.xlarge" -export DISK_SIZE=50 -export DISK_TYPE="gp2" -export SCALER_MAX=3 -export SCALER_MIN=1 -export GARDENER_PROVIDER="aws" -export GARDENER_REGION="eu-north-1" -export GARDENER_ZONES="eu-north-1b,eu-north-1c,eu-north-1a" - -./tests/integration/scripts/test-gardener.sh diff --git a/tests/integration/scripts/test-gardener-gcp.sh b/tests/integration/scripts/test-gardener-gcp.sh deleted file mode 100755 index e22b9b38b..000000000 --- a/tests/integration/scripts/test-gardener-gcp.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -#Description: This scripts installs and test Kyma using the CLI on a real Gardener GCP cluster. -# -#Expected common vars: -# - GARDENER_REGION - Gardener compute region -# - GARDENER_ZONES - Gardener compute zones inside the region -# - GARDENER_CLUSTER_VERSION - Version of the Kubernetes cluster -# - GARDENER_KYMA_PROW_KUBECONFIG - Kubeconfig of the Gardener service account -# - GARDENER_KYMA_PROW_PROJECT_NAME - Name of the gardener project where the cluster will be integrated. -# - GARDENER_KYMA_PROW_PROVIDER_SECRET_NAME - Name of the secret configured in the gardener project to access the cloud provider -# - -# exit on error, and raise error when variable is not set when used -set -e - -export MACHINE_TYPE="n1-standard-4" -export DISK_SIZE=50 -export DISK_TYPE="pd-standard" -export SCALER_MAX=3 -export SCALER_MIN=1 -export GARDENER_PROVIDER="gcp" -export GARDENER_REGION="europe-west4" -export GARDENER_ZONES="europe-west4-b" - -# Since there is an issue with iptables locking when using Istio CNI and Calico on Gardener GCP clusters, we set need to set -# the concurrency of the tests to 1 to avoid failing tests. -export TEST_CONCURRENCY="1" - -./tests/integration/scripts/test-gardener.sh diff --git a/tests/integration/scripts/test-gardener.sh b/tests/integration/scripts/test-gardener.sh deleted file mode 100755 index 20ac8d4e0..000000000 --- a/tests/integration/scripts/test-gardener.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -#Description: This scripts installs and test Kyma using the CLI on a real Gardener cluster. -# -#Expected common vars: -# - GARDENER_REGION - Gardener compute region -# - GARDENER_ZONES - Gardener compute zones inside the region -# - GARDENER_CLUSTER_VERSION - Version of the Kubernetes cluster -# - GARDENER_KYMA_PROW_KUBECONFIG - Kubeconfig of the Gardener service account -# - GARDENER_KYMA_PROW_PROJECT_NAME - Name of the gardener project where the cluster will be integrated. -# - GARDENER_KYMA_PROW_PROVIDER_SECRET_NAME - Name of the secret configured in the gardener project to access the cloud provider -# - -# exit on error, and raise error when variable is not set when used -set -e - -function check_required_vars() { - local requiredVarMissing=false - for var in "$@"; do - if [ -z "${var}" ]; then - >&2 echo "Environment variable ${var} is required but not set" - requiredVarMissing=true - fi - done - if [ "${requiredVarMissing}" = true ] ; then - exit 2 - fi -} - -requiredVars=( - GARDENER_KYMA_PROW_KUBECONFIG - GARDENER_KYMA_PROW_PROJECT_NAME -) - -check_required_vars "${requiredVars[@]}" - -cleanup() { -kubectl annotate shoot "${CLUSTER_NAME}" confirmation.gardener.cloud/deletion=true \ - --overwrite \ - -n "garden-${GARDENER_KYMA_PROW_PROJECT_NAME}" \ - --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" - -kubectl delete shoot "${CLUSTER_NAME}" \ - --wait="false" \ - --kubeconfig "${GARDENER_KYMA_PROW_KUBECONFIG}" \ - -n "garden-${GARDENER_KYMA_PROW_PROJECT_NAME}" -} - -# Cleanup on exit, be it successful or on fail -trap cleanup EXIT INT - -# Add pwd to path to be able to use binaries downloaded in scripts -export PATH="${PATH}:${PWD}" - -CLUSTER_NAME=$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c10) -export CLUSTER_NAME - -./tests/integration/scripts/provision-gardener.sh -./tests/integration/scripts/jobguard.sh - -attempts_counter=1 -max_attempts=3 -success=false - -while [ $success = false ] && [ $attempts_counter -le $max_attempts ]; do - if make install-istio; then - success=true - else - attempts_counter=$(( attempts_counter + 1 )) - sleep 180 - fi -done - -# KYMA_DOMAIN is required by the tests -export KYMA_DOMAIN="${CLUSTER_NAME}.${GARDENER_KYMA_PROW_PROJECT_NAME}.shoot.live.k8s-hana.ondemand.com" -make test-integration -