From 7eb8864f421975fbfd8fda8af8bbb04917e9c1aa Mon Sep 17 00:00:00 2001 From: klinch0 <68821526+klinch0@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:14:04 +0300 Subject: [PATCH] Add tests refactor (#451) ## Summary by CodeRabbit ## Release Notes - **New Features** - Introduced a new script for managing Helm releases in Kubernetes, including installation and status monitoring. - Added a configuration file for tenant settings, enabling monitoring and SeaweedFS. - Enhanced PostgreSQL initialization script to manage database roles and privileges dynamically. - Added a new local pre-commit hook for version map checks. - **Bug Fixes** - Updated pre-commit hooks for consistent formatting. - **Tests** - Improved testing capabilities for applications in a Kubernetes environment with new Makefile targets. - Enhanced Docker image with tools for YAML and JSON processing. - Updated testing image to the latest version for improved performance. --------- Signed-off-by: Andrei Kvapil Co-authored-by: Andrei Kvapil --- .pre-commit-config.yaml | 13 +- hack/e2e.application.sh | 135 ++++++++++++++++++ hack/values/tenant.yaml | 6 + .../apps/ferretdb/templates/init-script.yaml | 4 +- packages/core/testing/Chart.yaml | 0 packages/core/testing/Makefile | 16 ++- .../testing/images/e2e-sandbox/Dockerfile | 3 +- packages/core/testing/templates/sandbox.yaml | 0 packages/core/testing/values.yaml | 2 +- 9 files changed, 170 insertions(+), 9 deletions(-) create mode 100755 hack/e2e.application.sh create mode 100644 hack/values/tenant.yaml mode change 100644 => 100755 packages/core/testing/Chart.yaml mode change 100644 => 100755 packages/core/testing/Makefile mode change 100644 => 100755 packages/core/testing/images/e2e-sandbox/Dockerfile mode change 100644 => 100755 packages/core/testing/templates/sandbox.yaml mode change 100644 => 100755 packages/core/testing/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 589b9e24..16fb8495 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,5 +12,14 @@ repos: - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.41.0 hooks: - - id: markdownlint - args: [--fix, --disable, MD013, MD041, --] + - id: markdownlint + args: [--fix, --disable, MD013, MD041, --] +- repo: local + hooks: + - id: gen-versions-map + name: Generate versions map and check for changes + entry: sh -c 'make -C packages/apps check-version-map && make -C packages/extra check-version-map' + language: system + types: [file] + pass_filenames: false + description: Run the script and fail if it generates changes diff --git a/hack/e2e.application.sh b/hack/e2e.application.sh new file mode 100755 index 00000000..e908615c --- /dev/null +++ b/hack/e2e.application.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +RED='\033[0;31m' +GREEN='\033[0;32m' +RESET='\033[0m' +YELLOW='\033[0;33m' + + +ROOT_NS="tenant-root" +TEST_TENANT="tenant-e2e" + +function clean() { + kubectl delete helmrelease.helm.toolkit.fluxcd.io $TEST_TENANT -n $ROOT_NS + if true; then + echo -e "${GREEN}Cleanup successful!${RESET}" + return 0 + else + echo -e "${RED}Cleanup failed!${RESET}" + return 1 + fi +} + +function install_helmrelease() { + local release_name="$1" + local namespace="$2" + local chart_path="$3" + local repo_name="$4" + local repo_ns="$5" + local values_file="$6" + + if [[ -z "$release_name" ]]; then + echo -e "${RED}Error: Release name is required.${RESET}" + exit 1 + fi + + if [[ -z "$namespace" ]]; then + echo -e "${RED}Error: Namespace name is required.${RESET}" + exit 1 + fi + + if [[ -z "$chart_path" ]]; then + echo -e "${RED}Error: Chart path name is required.${RESET}" + exit 1 + fi + + local helmrelease_file=$(mktemp /tmp/HelmRelease.XXXXXX.yaml) + { + echo "apiVersion: helm.toolkit.fluxcd.io/v2" + echo "kind: HelmRelease" + echo "metadata:" + echo " labels:" + echo " cozystack.io/ui: \"true\"" + echo " name: \"$release_name\"" + echo " namespace: \"$namespace\"" + echo "spec:" + echo " chart:" + echo " spec:" + echo " chart: \"$chart_path\"" + echo " reconcileStrategy: Revision" + echo " sourceRef:" + echo " kind: HelmRepository" + echo " name: \"$repo_name\"" + echo " namespace: \"$repo_ns\"" + echo " version: '*'" + echo " interval: 1m0s" + echo " timeout: 5m0s" + + if [[ -n "$values_file" && -f "$values_file" ]]; then + echo " values:" + cat "$values_file" | sed 's/^/ /' + fi + } > "$helmrelease_file" + + kubectl apply -f "$helmrelease_file" + + rm -f "$helmrelease_file" +} + +function install_tenant (){ + local release_name="$1" + local namespace="$2" + local values_file="${3:-tenant.yaml}" + local repo_name="cozystack-apps" + local repo_ns="cozy-public" + + install_helmrelease "$release_name" "$namespace" "tenant" "$repo_name" "$repo_ns" "$values_file" +} + +function check_helmrelease_status() { + local release_name="$1" + local namespace="$2" + local timeout=300 # Timeout in seconds + local interval=5 # Interval between checks in seconds + local elapsed=0 + + while [[ $elapsed -lt $timeout ]]; do + local status_output + status_output=$(kubectl get helmrelease "$release_name" -n "$namespace" -o json | jq -r '.status.conditions[-1].reason') + + if [[ "$status_output" == "InstallSucceeded" ]]; then + echo -e "${GREEN}Helm release '$release_name' is ready.${RESET}" + return 0 + elif [[ "$status_output" == "InstallFailed" ]]; then + echo -e "${RED}Helm release '$release_name': InstallFailed${RESET}" + exit 1 + else + echo -e "${YELLOW}Helm release '$release_name' is not ready. Current status: $status_output${RESET}" + fi + + sleep "$interval" + elapsed=$((elapsed + interval)) + done + + echo -e "${RED}Timeout reached. Helm release '$release_name' is still not ready after $timeout seconds.${RESET}" + exit 1 +} + +chart_name="$1" + +if [ -z "$chart_name" ]; then + echo -e "${RED}No chart name provided. Exiting...${RESET}" + exit 1 +fi + +echo "Running tests for chart: $chart_name" +install_tenant $TEST_TENANT $ROOT_NS +check_helmrelease_status $TEST_TENANT $ROOT_NS + +repo_name="cozystack-apps" +repo_ns="cozy-public" + +release_name="$chart_name-e2e" +install_helmrelease "$release_name" "$TEST_TENANT" "$chart_name" "$repo_name" "$repo_ns" + +check_helmrelease_status "$release_name" "$TEST_TENANT" diff --git a/hack/values/tenant.yaml b/hack/values/tenant.yaml new file mode 100644 index 00000000..3e75d05b --- /dev/null +++ b/hack/values/tenant.yaml @@ -0,0 +1,6 @@ +host: "" +etcd: false +monitoring: false +ingress: false +seaweedfs: false +isolated: true diff --git a/packages/apps/ferretdb/templates/init-script.yaml b/packages/apps/ferretdb/templates/init-script.yaml index fb64edbf..35723ede 100644 --- a/packages/apps/ferretdb/templates/init-script.yaml +++ b/packages/apps/ferretdb/templates/init-script.yaml @@ -63,7 +63,7 @@ stringData: DROP USER $user; EOT done - + echo "== create roles" psql -v ON_ERROR_STOP=1 --echo-all <<\EOT SELECT 'CREATE ROLE app_admin NOINHERIT;' @@ -83,7 +83,7 @@ stringData: FOR schema_record IN SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT IN ('pg_catalog', 'information_schema') LOOP -- Changing Schema Ownership EXECUTE format('ALTER SCHEMA %I OWNER TO %I', schema_record.schema_name, 'app_admin'); - + -- Add rights for the admin role EXECUTE format('GRANT ALL ON SCHEMA %I TO %I', schema_record.schema_name, 'app_admin'); EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO %I', schema_record.schema_name, 'app_admin'); diff --git a/packages/core/testing/Chart.yaml b/packages/core/testing/Chart.yaml old mode 100644 new mode 100755 diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile old mode 100644 new mode 100755 index 7e6ae50e..728b7de8 --- a/packages/core/testing/Makefile +++ b/packages/core/testing/Makefile @@ -1,6 +1,7 @@ NAMESPACE=cozy-e2e-tests NAME := sandbox CLEAN := 1 +TESTING_APPS := $(shell find ../../apps -maxdepth 1 -mindepth 1 -type d | awk -F/ '{print $$NF}') include ../../../scripts/common-envs.mk @@ -32,10 +33,19 @@ image-e2e-sandbox: yq -i '.e2e.image = strenv(IMAGE)' values.yaml rm -f images/e2e-sandbox.json -test: wait-for-sandbox ## Run the end-to-end tests in existing sandbox. - cat ../../../hack/e2e.sh | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /e2e.sh && chmod +x /e2e.sh' +copy-hack-dir: + tar -C ../../../ -cf- hack | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- tar -xf- + +test: wait-for-sandbox copy-hack-dir ## Run the end-to-end tests in existing sandbox. helm template -n cozy-system installer ../installer | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /cozystack-installer.yaml' - kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /e2e.sh' + kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /hack/e2e.sh' + +test-applications: wait-for-sandbox copy-hack-dir ## Run the end-to-end tests in existing sandbox for applications. + for app in $(TESTING_APPS); do \ + echo "Running tests for $${app}"; \ + kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "/hack/e2e.application.sh $${app}"; \ + done + kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "kubectl get hr -A | grep -v 'True'" delete: ## Remove sandbox from existing Kubernetes cluster. kubectl delete deploy -n $(NAMESPACE) cozystack-e2e-$(NAME) diff --git a/packages/core/testing/images/e2e-sandbox/Dockerfile b/packages/core/testing/images/e2e-sandbox/Dockerfile old mode 100644 new mode 100755 index a07dccd3..dff51de2 --- a/packages/core/testing/images/e2e-sandbox/Dockerfile +++ b/packages/core/testing/images/e2e-sandbox/Dockerfile @@ -5,7 +5,7 @@ ARG TALOSCTL_VERSION=1.7.6 ARG HELM_VERSION=3.15.4 RUN apt-get update -RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl +RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl jq RUN curl -LO "https://github.com/siderolabs/talos/releases/download/v${TALOSCTL_VERSION}/talosctl-linux-amd64" \ && chmod +x talosctl-linux-amd64 \ && mv talosctl-linux-amd64 /usr/local/bin/talosctl @@ -13,3 +13,4 @@ RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kube && chmod +x kubectl \ && mv kubectl /usr/local/bin/kubectl RUN curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s - --version "v${HELM_VERSION}" +RUN wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq diff --git a/packages/core/testing/templates/sandbox.yaml b/packages/core/testing/templates/sandbox.yaml old mode 100644 new mode 100755 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml old mode 100644 new mode 100755 index a05cd6d1..ab42bfa7 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/aenix-io/cozystack/e2e-sandbox:v0.17.1@sha256:c3390f0076f4a8445273d0cdeda7725ac6f5110ad35e1a286be4b158708c4402 + image: ghcr.io/aenix-io/cozystack/e2e-sandbox:latest@sha256:1a26a511b9e269bcb607e2d80f878d7c2d993b7a2a7a3a2a1042470c8c56b061