diff --git a/Jenkinsfile b/Jenkinsfile index f7059229b..1b1ff2a59 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -280,7 +280,6 @@ pipeline { environment { CLOUDSDK_CORE_DISABLE_PROMPTS = 1 CLEAN_NAMESPACE = 1 - OPERATOR_NS = 'ps-operator' GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', , returnStdout: true).trim() VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}" CLUSTER_NAME = sh(script: "echo jen-ps-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", , returnStdout: true).trim() diff --git a/e2e-tests/functions b/e2e-tests/functions index a8316683b..1af3ce57a 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -35,316 +35,315 @@ create_namespace() { deploy_operator() { destroy_operator - if [[ $OPERATOR_NS ]]; then - create_namespace "${OPERATOR_NS}" - fi + if [[ $OPERATOR_NS ]]; then + create_namespace "${OPERATOR_NS}" + fi - kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml" + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml" - if [ -n "$OPERATOR_NS" ]; then - kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/cw-rbac.yaml" + if [ -n "$OPERATOR_NS" ]; then + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/cw-rbac.yaml" - yq eval \ - "$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \ - "${DEPLOY_DIR}/cw-operator.yaml" \ - | yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \ - | yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \ - | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - - else - kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/rbac.yaml" - - yq eval \ - "$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \ - "${DEPLOY_DIR}/operator.yaml" \ - | yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \ - | yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \ - | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - - fi + yq eval \ + "$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \ + "${DEPLOY_DIR}/cw-operator.yaml" | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' | + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - + else + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/rbac.yaml" + + yq eval \ + "$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \ + "${DEPLOY_DIR}/operator.yaml" | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' | + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - + fi } destroy_operator() { - kubectl -n "${OPERATOR_NS:-$NAMESPACE}" delete deployment percona-server-mysql-operator --force --grace-period=0 || true - if [[ $OPERATOR_NS ]]; then - kubectl delete namespace $OPERATOR_NS --force --grace-period=0 || true - fi + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" delete deployment percona-server-mysql-operator --force --grace-period=0 || true + if [[ $OPERATOR_NS ]]; then + kubectl delete namespace $OPERATOR_NS --force --grace-period=0 || true + fi } deploy_non_tls_cluster_secrets() { - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/secrets.yaml" + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/secrets.yaml" } deploy_tls_cluster_secrets() { - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/ssl-secret.yaml" + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/ssl-secret.yaml" } deploy_client() { - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/client.yaml" + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/client.yaml" } apply_s3_storage_secrets() { - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/minio-secret.yml" - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/cloud-secret.yml" + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/minio-secret.yml" + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/cloud-secret.yml" } deploy_pmm_server() { - if [[ $OPENSHIFT ]]; then - oc create sa pmm-server -n "${NAMESPACE}" || : - oc adm policy add-scc-to-user privileged -z pmm-server -n "${NAMESPACE}" || : - oc create rolebinding pmm-ps-operator-namespace-only --role percona-server-for-mysql-operator-role --serviceaccount=$NAMESPACE:pmm-server -n "${NAMESPACE}" || : - oc patch role/percona-server-for-mysql-operator-role --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n "${NAMESPACE}" || : - - local additional_params="--set platform=openshift --set sa=pmm-server --set supresshttp2=false" - fi - - helm install monitoring -n "${NAMESPACE}" --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} $additional_params https://percona-charts.storage.googleapis.com/pmm-server-$PMM_SERVER_VERSION.tgz - - until kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null"; do - echo "Retry $retry" - sleep 5 - let retry+=1 - if [ $retry -ge 20 ]; then - echo "Max retry count $retry reached. Pmm-server can't start" - exit 1 - fi - done + if [[ $OPENSHIFT ]]; then + oc create sa pmm-server -n "${NAMESPACE}" || : + oc adm policy add-scc-to-user privileged -z pmm-server -n "${NAMESPACE}" || : + oc create rolebinding pmm-ps-operator-namespace-only --role percona-server-for-mysql-operator-role --serviceaccount=$NAMESPACE:pmm-server -n "${NAMESPACE}" || : + oc patch role/percona-server-for-mysql-operator-role --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n "${NAMESPACE}" || : + + local additional_params="--set platform=openshift --set sa=pmm-server --set supresshttp2=false" + fi + helm install monitoring -n "${NAMESPACE}" --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} $additional_params https://percona-charts.storage.googleapis.com/pmm-server-$PMM_SERVER_VERSION.tgz + + until kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null"; do + echo "Retry $retry" + sleep 5 + let retry+=1 + if [ $retry -ge 20 ]; then + echo "Max retry count $retry reached. Pmm-server can't start" + exit 1 + fi + done } get_pmm_api_key() { - local key_name=$1 + local key_name=$1 - if [[ -z $key_name ]]; then - key_name="operator" - fi + if [[ -z $key_name ]]; then + key_name="operator" + fi - local ADMIN_PASSWORD - ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2") - curl --insecure -X POST -H "Content-Type: application/json" -d "{\"name\":\"$key_name\", \"role\": \"Admin\"}" "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq .key + local ADMIN_PASSWORD + ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2") + curl --insecure -X POST -H "Content-Type: application/json" -d "{\"name\":\"$key_name\", \"role\": \"Admin\"}" "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq .key } delete_pmm_api_key() { - local key_name=$1 + local key_name=$1 - if [[ -z $key_name ]]; then - key_name="operator" - fi + if [[ -z $key_name ]]; then + key_name="operator" + fi - local ADMIN_PASSWORD - ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2") + local ADMIN_PASSWORD + ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2") - local key_id - key_id=$(curl --insecure -X GET "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq '.[] | select( .name == "operator").id') - curl --insecure -X DELETE "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys/$key_id" + local key_id + key_id=$(curl --insecure -X GET "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq '.[] | select( .name == "operator").id') + curl --insecure -X DELETE "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys/$key_id" } deploy_minio() { - local access_key - local secret_key - access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" - secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" - - helm uninstall -n "${NAMESPACE}" minio-service || : - helm repo remove minio || : - helm repo add minio https://charts.min.io/ - retry 10 60 helm install minio-service \ - -n "${NAMESPACE}" \ - --version 5.0.14 \ - --set replicas=1 \ - --set mode=standalone \ - --set resources.requests.memory=256Mi \ - --set rootUser=rootuser \ - --set rootPassword=rootpass123 \ - --set "users[0].accessKey"="$(printf '%q' "$(printf '%q' "$access_key")")" \ - --set "users[0].secretKey"="$(printf '%q' "$(printf '%q' "$secret_key")")" \ - --set "users[0].policy"=consoleAdmin \ - --set service.type=ClusterIP \ - --set configPathmc=/tmp/.minio/ \ - --set persistence.size=2G \ - --set securityContext.enabled=false \ - minio/minio - MINIO_POD=$(kubectl -n "${NAMESPACE}" get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}') - wait_pod $MINIO_POD - - # create bucket - kubectl -n "${NAMESPACE}" run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ - bash -c "AWS_ACCESS_KEY_ID='$access_key' AWS_SECRET_ACCESS_KEY='$secret_key' AWS_DEFAULT_REGION=us-east-1 \ + local access_key + local secret_key + access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" + secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" + + helm uninstall -n "${NAMESPACE}" minio-service || : + helm repo remove minio || : + helm repo add minio https://charts.min.io/ + retry 10 60 helm install minio-service \ + -n "${NAMESPACE}" \ + --version 5.0.14 \ + --set replicas=1 \ + --set mode=standalone \ + --set resources.requests.memory=256Mi \ + --set rootUser=rootuser \ + --set rootPassword=rootpass123 \ + --set "users[0].accessKey"="$(printf '%q' "$(printf '%q' "$access_key")")" \ + --set "users[0].secretKey"="$(printf '%q' "$(printf '%q' "$secret_key")")" \ + --set "users[0].policy"=consoleAdmin \ + --set service.type=ClusterIP \ + --set configPathmc=/tmp/.minio/ \ + --set persistence.size=2G \ + --set securityContext.enabled=false \ + minio/minio + MINIO_POD=$(kubectl -n "${NAMESPACE}" get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}') + wait_pod $MINIO_POD + + # create bucket + kubectl -n "${NAMESPACE}" run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ + bash -c "AWS_ACCESS_KEY_ID='$access_key' AWS_SECRET_ACCESS_KEY='$secret_key' AWS_DEFAULT_REGION=us-east-1 \ /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing" } retry() { - local max=$1 - local delay=$2 - shift 2 # cut delay and max args - local n=1 - - until "$@"; do - if [[ $n -ge $max ]]; then - echo "The command ${*} has failed after $n attempts." - exit 1 - fi - ((n++)) - sleep $delay - done + local max=$1 + local delay=$2 + shift 2 # cut delay and max args + local n=1 + + until "$@"; do + if [[ $n -ge $max ]]; then + echo "The command ${*} has failed after $n attempts." + exit 1 + fi + ((n++)) + sleep $delay + done } get_operator_pod() { - kubectl get pods -n "${OPERATOR_NS:-$NAMESPACE}" \ - --selector=app.kubernetes.io/name=percona-server-mysql-operator \ - -o 'jsonpath={.items[].metadata.name}' + kubectl get pods -n "${OPERATOR_NS:-$NAMESPACE}" \ + --selector=app.kubernetes.io/name=percona-server-mysql-operator \ + -o 'jsonpath={.items[].metadata.name}' } get_cr() { - local name_suffix=$1 - - yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" "${DEPLOY_DIR}/cr.yaml" \ - | yq eval "$(printf '.spec.initImage="%s"' "${IMAGE}")" - \ - | yq eval '.spec.secretsName="test-secrets"' - \ - | yq eval '.spec.sslSecretName="test-ssl"' - \ - | yq eval '.spec.upgradeOptions.apply="disabled"' - \ - | yq eval '.spec.mysql.clusterType="async"' - \ - | yq eval "$(printf '.spec.mysql.image="%s"' "${IMAGE_MYSQL}")" - \ - | yq eval "$(printf '.spec.backup.image="%s"' "${IMAGE_BACKUP}")" - \ - | yq eval "$(printf '.spec.orchestrator.image="%s"' "${IMAGE_ORCHESTRATOR}")" - \ - | yq eval "$(printf '.spec.proxy.router.image="%s"' "${IMAGE_ROUTER}")" - \ - | yq eval "$(printf '.spec.toolkit.image="%s"' "${IMAGE_TOOLKIT}")" - \ - | yq eval "$(printf '.spec.proxy.haproxy.image="%s"' "${IMAGE_HAPROXY}")" - \ - | yq eval "$(printf '.spec.pmm.image="%s"' "${IMAGE_PMM_CLIENT}")" - \ - | if [ -n "${MINIKUBE}" ]; then - yq eval '(.. | select(has("antiAffinityTopologyKey")).antiAffinityTopologyKey) |= "none"' - \ - | yq eval '.spec.proxy.haproxy.resources.requests.cpu="300m"' - - else - yq eval - - fi + local name_suffix=$1 + + yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" "${DEPLOY_DIR}/cr.yaml" | + yq eval "$(printf '.spec.initImage="%s"' "${IMAGE}")" - | + yq eval '.spec.secretsName="test-secrets"' - | + yq eval '.spec.sslSecretName="test-ssl"' - | + yq eval '.spec.upgradeOptions.apply="disabled"' - | + yq eval '.spec.mysql.clusterType="async"' - | + yq eval "$(printf '.spec.mysql.image="%s"' "${IMAGE_MYSQL}")" - | + yq eval "$(printf '.spec.backup.image="%s"' "${IMAGE_BACKUP}")" - | + yq eval "$(printf '.spec.orchestrator.image="%s"' "${IMAGE_ORCHESTRATOR}")" - | + yq eval "$(printf '.spec.proxy.router.image="%s"' "${IMAGE_ROUTER}")" - | + yq eval "$(printf '.spec.toolkit.image="%s"' "${IMAGE_TOOLKIT}")" - | + yq eval "$(printf '.spec.proxy.haproxy.image="%s"' "${IMAGE_HAPROXY}")" - | + yq eval "$(printf '.spec.pmm.image="%s"' "${IMAGE_PMM_CLIENT}")" - | + if [ -n "${MINIKUBE}" ]; then + yq eval '(.. | select(has("antiAffinityTopologyKey")).antiAffinityTopologyKey) |= "none"' - | + yq eval '.spec.proxy.haproxy.resources.requests.cpu="300m"' - + else + yq eval - + fi } get_client_pod() { - kubectl -n "${NAMESPACE}" get pods \ - --selector=name=mysql-client \ - -o 'jsonpath={.items[].metadata.name}' + kubectl -n "${NAMESPACE}" get pods \ + --selector=name=mysql-client \ + -o 'jsonpath={.items[].metadata.name}' } run_mysql() { - local command="$1" - local uri="$2" - local pod="$3" + local command="$1" + local uri="$2" + local pod="$3" - client_pod=$(get_client_pod) - wait_pod $client_pod 1>&2 + client_pod=$(get_client_pod) + wait_pod $client_pod 1>&2 - kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- \ - bash -c "printf '%s\n' \"${command}\" | mysql -sN $uri" 2>&1 \ - | sed -e 's/mysql: //' \ - | (grep -v 'Using a password on the command line interface can be insecure.' || :) + kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- \ + bash -c "printf '%s\n' \"${command}\" | mysql -sN $uri" 2>&1 | + sed -e 's/mysql: //' | + (grep -v 'Using a password on the command line interface can be insecure.' || :) } run_mysqlsh() { - local command="$1" - local uri="$2" - local pod="$3" + local command="$1" + local uri="$2" + local pod="$3" - client_pod=$(get_client_pod) - wait_pod $client_pod 1>&2 + client_pod=$(get_client_pod) + wait_pod $client_pod 1>&2 - kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- \ - bash -c "printf '%s\n' \"${command}\" | mysqlsh --sql --quiet-start=2 $uri" 2>&1 \ - | tail -n +2 + kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- \ + bash -c "printf '%s\n' \"${command}\" | mysqlsh --sql --quiet-start=2 $uri" 2>&1 | + tail -n +2 } run_curl() { - kubectl -n "${NAMESPACE}" exec mysql-client -- bash -c "curl -s -k $*" + kubectl -n "${NAMESPACE}" exec mysql-client -- bash -c "curl -s -k $*" } get_innodb_cluster_name() { - echo $(get_cluster_name) | tr -cd '[^a-zA-Z0-9_]+' + echo $(get_cluster_name) | tr -cd '[^a-zA-Z0-9_]+' } get_mysqlsh_uri() { - local idx=${1:-0} + local idx=${1:-0} - echo "root:root_password@$(get_cluster_name)-mysql-${idx}.$(get_cluster_name)-mysql.${NAMESPACE}" + echo "root:root_password@$(get_cluster_name)-mysql-${idx}.$(get_cluster_name)-mysql.${NAMESPACE}" } get_gr_status() { - local uri="$1" - local pod="$2" + local uri="$1" + local pod="$2" - client_pod=$(get_client_pod) + client_pod=$(get_client_pod) - kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- mysqlsh --uri $uri --cluster --result-format json -- cluster status \ - | sed -e 's/mysql: //' \ - | (grep -v 'Using a password on the command line interface can be insecure.' || :) + kubectl -n "${NAMESPACE}" exec "${pod:-mysql-client}" -- mysqlsh --uri $uri --cluster --result-format json -- cluster status | + sed -e 's/mysql: //' | + (grep -v 'Using a password on the command line interface can be insecure.' || :) } get_cluster_name() { - kubectl -n "${NAMESPACE}" get ps -o jsonpath='{.items[0].metadata.name}' + kubectl -n "${NAMESPACE}" get ps -o jsonpath='{.items[0].metadata.name}' } get_mysql_service() { - local cluster=$1 + local cluster=$1 - echo "${cluster}-mysql" + echo "${cluster}-mysql" } get_router_service() { - local cluster=$1 + local cluster=$1 - echo "${cluster}-router" + echo "${cluster}-router" } get_haproxy_svc() { - local cluster=$1 + local cluster=$1 - echo "${cluster}-haproxy" + echo "${cluster}-haproxy" } get_orc_svc() { - local cluster=$1 + local cluster=$1 - echo "${cluster}-orc" + echo "${cluster}-orc" } get_mysql_headless_fqdn() { - local cluster=$1 - local index=$2 + local cluster=$1 + local index=$2 - echo "${cluster}-mysql-${index}.${cluster}-mysql" + echo "${cluster}-mysql-${index}.${cluster}-mysql" } get_orc_headless_fqdn() { - local cluster=$1 - local index=$2 + local cluster=$1 + local index=$2 - echo "${cluster}-orc-${index}.${cluster}-orc" + echo "${cluster}-orc-${index}.${cluster}-orc" } get_metric_values() { - local metric=$1 - local instance=$2 - local user_pass=$3 - local start=$($date -u "+%s" -d "-1 minute") - local end=$($date -u "+%s") - - set +o xtrace - retry=0 - until run_curl "https://${user_pass}@monitoring-service/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" | jq '.data.result[0].values[][1]' | grep '^"[0-9]*"$'; do - sleep 1 - let retry+=1 - if [ $retry -ge 30 ]; then - echo "Max retry count $retry reached. Data about instance $instance was not collected!" - exit 1 - fi - done - set -o xtrace + local metric=$1 + local instance=$2 + local user_pass=$3 + local start=$($date -u "+%s" -d "-1 minute") + local end=$($date -u "+%s") + + set +o xtrace + retry=0 + until run_curl "https://${user_pass}@monitoring-service/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" | jq '.data.result[0].values[][1]' | grep '^"[0-9]*"$'; do + sleep 1 + let retry+=1 + if [ $retry -ge 30 ]; then + echo "Max retry count $retry reached. Data about instance $instance was not collected!" + exit 1 + fi + done + set -o xtrace } get_qan20_values() { - local instance=$1 - local user_pass=$2 - local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute") - local end=$($date -u "+%Y-%m-%dT%H:%M:%S") - local endpoint=monitoring-service - - local payload=$( - cat <&1 || :) | grep -q NotFound; do - sleep 1 - done - if [ "$(kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.type}')" = "ClusterIP" ]; then - kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.clusterIP}' - return - fi - until kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[]}' 2>&1 | egrep -q "hostname|ip"; do - sleep 1 - done - kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].ip}' - kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' + local service=$1 + + while (kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.type}' 2>&1 || :) | grep -q NotFound; do + sleep 1 + done + if [ "$(kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.type}')" = "ClusterIP" ]; then + kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.clusterIP}' + return + fi + until kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[]}' 2>&1 | egrep -q "hostname|ip"; do + sleep 1 + done + kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].ip}' + kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' } wait_cluster_consistency_async() { - local cluster_name=${1} - local cluster_size=${2} - local orc_size=${3} + local cluster_name=${1} + local cluster_size=${2} + local orc_size=${3} - if [ -z "${orc_size}" ]; then - orc_size=3 - fi + if [ -z "${orc_size}" ]; then + orc_size=3 + fi - sleep 7 # wait for two reconcile loops ;) 3 sec x 2 times + 1 sec = 7 seconds - until [[ "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.state}')" == "ready" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.ready}')" == "${cluster_size}" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.orchestrator.ready}')" == "${orc_size}" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.orchestrator.state}')" == "ready" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.state}')" == "ready" ]]; do - echo 'waiting for cluster readyness (async)' - sleep 15 - done + sleep 7 # wait for two reconcile loops ;) 3 sec x 2 times + 1 sec = 7 seconds + until [[ "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.state}')" == "ready" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.ready}')" == "${cluster_size}" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.orchestrator.ready}')" == "${orc_size}" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.orchestrator.state}')" == "ready" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.state}')" == "ready" ]]; do + echo 'waiting for cluster readyness (async)' + sleep 15 + done } wait_cluster_consistency_gr() { - local cluster_name=${1} - local cluster_size=${2} - local router_size=${3} + local cluster_name=${1} + local cluster_size=${2} + local router_size=${3} - if [ -z "${router_size}" ]; then - router_size=3 - fi + if [ -z "${router_size}" ]; then + router_size=3 + fi - sleep 7 # wait for two reconcile loops ;) 3 sec x 2 times + 1 sec = 7 seconds - until [[ "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.state}')" == "ready" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.ready}')" == "${cluster_size}" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.router.ready}')" == "${router_size}" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.router.state}')" == "ready" && - "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.state}')" == "ready" ]]; do - echo 'waiting for cluster readyness (group replication)' - sleep 15 - done + sleep 7 # wait for two reconcile loops ;) 3 sec x 2 times + 1 sec = 7 seconds + until [[ "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.state}')" == "ready" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.mysql.ready}')" == "${cluster_size}" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.router.ready}')" == "${router_size}" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.router.state}')" == "ready" && + "$(kubectl get ps "${cluster_name}" -n "${NAMESPACE}" -o jsonpath='{.status.state}')" == "ready" ]]; do + echo 'waiting for cluster readyness (group replication)' + sleep 15 + done } wait_pod() { - local pod=$1 - - set +o xtrace - retry=0 - echo -n $pod - until kubectl get pod/$pod -n "${NAMESPACE}" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep 'true'; do - sleep 1 - echo -n . - let retry+=1 - if [ $retry -ge 360 ]; then - kubectl describe pod/$pod -n "${NAMESPACE}" - kubectl logs $pod -n "${NAMESPACE}" - kubectl logs $(get_operator_pod) -n "${OPERATOR_NS:-$NAMESPACE}" \ - | grep -v 'level=info' \ - | grep -v 'level=debug' \ - | grep -v 'Getting tasks for pod' \ - | grep -v 'Getting pods from source' \ - | tail -100 - echo max retry count $retry reached. something went wrong with operator or kubernetes cluster - exit 1 - fi - done - set -o xtrace + local pod=$1 + + set +o xtrace + retry=0 + echo -n $pod + until kubectl get pod/$pod -n "${NAMESPACE}" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep 'true'; do + sleep 1 + echo -n . + let retry+=1 + if [ $retry -ge 360 ]; then + kubectl describe pod/$pod -n "${NAMESPACE}" + kubectl logs $pod -n "${NAMESPACE}" + kubectl logs $(get_operator_pod) -n "${OPERATOR_NS:-$NAMESPACE}" | + grep -v 'level=info' | + grep -v 'level=debug' | + grep -v 'Getting tasks for pod' | + grep -v 'Getting pods from source' | + tail -100 + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster + exit 1 + fi + done + set -o xtrace } wait_deployment() { - local name=$1 - local target_namespace=${2:-"$namespace"} - - sleep 10 - set +o xtrace - retry=0 - echo -n $name - until [ -n "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \ - -a "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \ - == "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.readyReplicas}')" ]; do - sleep 1 - echo -n . - let retry+=1 - if [ $retry -ge 360 ]; then - kubectl logs $(get_operator_pod) -c operator \ - | grep -v 'level=info' \ - | grep -v 'level=debug' \ - | tail -100 - echo max retry count $retry reached. something went wrong with operator or kubernetes cluster - exit 1 - fi - done - echo - set -o xtrace + local name=$1 + local target_namespace=${2:-"$namespace"} + + sleep 10 + set +o xtrace + retry=0 + echo -n $name + until [ -n "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \ + -a "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \ + == "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.readyReplicas}')" ]; do + sleep 1 + echo -n . + let retry+=1 + if [ $retry -ge 360 ]; then + kubectl logs $(get_operator_pod) -c operator | + grep -v 'level=info' | + grep -v 'level=debug' | + tail -100 + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster + exit 1 + fi + done + echo + set -o xtrace } check_auto_tuning() { - local RAM_SIZE=$1 - local RDS_MEM_INSTANCE=12582880 - local CUSTOM_INNODB_SIZE=$2 - local CUSTOM_CONNECTIONS=$3 - - local INNODB_SIZE=$(run_mysql \ - 'SELECT @@innodb_buffer_pool_size;' \ - "-h $(get_haproxy_svc "$(get_cluster_name)") -uroot -proot_password") - local CONNECTIONS=$(run_mysql \ - 'SELECT @@max_connections;' \ - "-h $(get_haproxy_svc "$(get_cluster_name)") -uroot -proot_password") - - if [[ -n ${CUSTOM_INNODB_SIZE} ]]; then - if [[ ${INNODB_SIZE} != ${CUSTOM_INNODB_SIZE} ]]; then - echo "innodb_buffer_pool_size is set to ${INNODB_SIZE}, which does not correlate with ${CUSTOM_INNODB_SIZE} from custom config" - exit 1 - fi - else - if [[ ${INNODB_SIZE} != $((RAM_SIZE * 50 / 100)) ]]; then - echo "innodb_buffer_pool_size is set to ${INNODB_SIZE}, which does not correlate with cr.pxc.limits.memory * 0.5" - exit 1 - fi - fi - - if [[ -n ${CUSTOM_CONNECTIONS} ]]; then - if [[ ${CONNECTIONS} != ${CUSTOM_CONNECTIONS} ]]; then - echo "max_connections is set to ${AUTO_CONNECTIONS}, which does not correlate with ${CUSTOM_CONNECTIONS} from custom config" - exit 1 - fi - else - if [[ ${CONNECTIONS} != $((RAM_SIZE / RDS_MEM_INSTANCE)) ]]; then - echo "max_connections is set to ${CONNECTIONS}, which does not correlate with cr.pxc.limits.memory / ${RDS_MEM_INSTANCE}" - exit 1 - fi - fi + local RAM_SIZE=$1 + local RDS_MEM_INSTANCE=12582880 + local CUSTOM_INNODB_SIZE=$2 + local CUSTOM_CONNECTIONS=$3 + + local INNODB_SIZE=$(run_mysql \ + 'SELECT @@innodb_buffer_pool_size;' \ + "-h $(get_haproxy_svc "$(get_cluster_name)") -uroot -proot_password") + local CONNECTIONS=$(run_mysql \ + 'SELECT @@max_connections;' \ + "-h $(get_haproxy_svc "$(get_cluster_name)") -uroot -proot_password") + + if [[ -n ${CUSTOM_INNODB_SIZE} ]]; then + if [[ ${INNODB_SIZE} != ${CUSTOM_INNODB_SIZE} ]]; then + echo "innodb_buffer_pool_size is set to ${INNODB_SIZE}, which does not correlate with ${CUSTOM_INNODB_SIZE} from custom config" + exit 1 + fi + else + if [[ ${INNODB_SIZE} != $((RAM_SIZE * 50 / 100)) ]]; then + echo "innodb_buffer_pool_size is set to ${INNODB_SIZE}, which does not correlate with cr.pxc.limits.memory * 0.5" + exit 1 + fi + fi + + if [[ -n ${CUSTOM_CONNECTIONS} ]]; then + if [[ ${CONNECTIONS} != ${CUSTOM_CONNECTIONS} ]]; then + echo "max_connections is set to ${AUTO_CONNECTIONS}, which does not correlate with ${CUSTOM_CONNECTIONS} from custom config" + exit 1 + fi + else + if [[ ${CONNECTIONS} != $((RAM_SIZE / RDS_MEM_INSTANCE)) ]]; then + echo "max_connections is set to ${CONNECTIONS}, which does not correlate with cr.pxc.limits.memory / ${RDS_MEM_INSTANCE}" + exit 1 + fi + fi } get_mysql_router_service() { - local cluster=$1 + local cluster=$1 - echo "${cluster}-router" + echo "${cluster}-router" } deploy_version_service() { - kubectl create configmap -n "${OPERATOR_NS:-$NAMESPACE}" versions \ - --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.ps-operator.dep.json" \ - --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.ps-operator.json" + kubectl create configmap -n "${OPERATOR_NS:-$NAMESPACE}" versions \ + --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.ps-operator.dep.json" \ + --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.ps-operator.json" - kubectl apply -n "${OPERATOR_NS:-$NAMESPACE}" -f "${TESTS_CONFIG_DIR}/vs.yaml" + kubectl apply -n "${OPERATOR_NS:-$NAMESPACE}" -f "${TESTS_CONFIG_DIR}/vs.yaml" - sleep 5 + sleep 5 } deploy_cert_manager() { - kubectl create namespace cert-manager || : - kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true || : - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml --validate=false || : 2>/dev/null + kubectl create namespace cert-manager || : + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true || : + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml --validate=false || : 2>/dev/null } destroy_cert_manager() { - kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml --validate=false || : 2>/dev/null - kubectl delete --grace-period=0 --force=true namespace cert-manager + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml --validate=false || : 2>/dev/null + kubectl delete --grace-period=0 --force=true namespace cert-manager } get_primary_from_label() { - kubectl -n "${NAMESPACE}" get pods -l mysql.percona.com/primary=true -ojsonpath="{.items[0].metadata.name}" + kubectl -n "${NAMESPACE}" get pods -l mysql.percona.com/primary=true -ojsonpath="{.items[0].metadata.name}" } get_primary_from_haproxy() { - local haproxy_pod=$1 - local haproxy_pod_ip=$(kubectl -n "${NAMESPACE}" get pods ${haproxy_pod} -o jsonpath="{.status.podIP}") + local haproxy_pod=$1 + local haproxy_pod_ip=$(kubectl -n "${NAMESPACE}" get pods ${haproxy_pod} -o jsonpath="{.status.podIP}") - run_mysql "SHOW VARIABLES LIKE '%hostname%';" "-h ${haproxy_pod_ip} -P3306 -uroot -proot_password" | awk '{print $2}' + run_mysql "SHOW VARIABLES LIKE '%hostname%';" "-h ${haproxy_pod_ip} -P3306 -uroot -proot_password" | awk '{print $2}' } get_primary_from_group_replication() { - run_mysql "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" | cut -d'.' -f1 + run_mysql "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" | cut -d'.' -f1 } verify_certificate_sans() { - local certificate=$1 - local expected_sans=$2 - local have=$(mktemp) - local want=$(mktemp) + local certificate=$1 + local expected_sans=$2 + local have=$(mktemp) + local want=$(mktemp) - kubectl -n "${NAMESPACE}" get certificate "${certificate}" -o jsonpath='{.spec.dnsNames}' | jq '.' >"${have}" - echo "${expected_sans}" | jq '.' >"${want}" + kubectl -n "${NAMESPACE}" get certificate "${certificate}" -o jsonpath='{.spec.dnsNames}' | jq '.' >"${have}" + echo "${expected_sans}" | jq '.' >"${want}" - diff "${have}" "${want}" + diff "${have}" "${want}" } check_passwords_leak() { - local secrets - local passwords - local pods - - secrets=$(kubectl get secrets -o json | jq -r '.items[].data | to_entries | .[] | select(.key | (endswith(".crt") or endswith(".key") or endswith(".pub") or endswith(".pem") or endswith(".p12") or test("namespace")) | not) | .value') - passwords="$(for i in $secrets; do - base64 -d <<<$i - echo - done) $secrets" - pods=$(kubectl -n "${NAMESPACE}" get pods -o name | awk -F "/" '{print $2}') - - collect_logs() { - local containers - local count - - NS=$1 - for p in $pods; do - containers=$(kubectl -n "$NS" get pod $p -o jsonpath='{.spec.containers[*].name}') - for c in $containers; do - kubectl -n "$NS" logs $p -c $c >${TEMP_DIR}/logs_output-$p-$c.txt - echo logs saved in: ${TEMP_DIR}/logs_output-$p-$c.txt - for pass in $passwords; do - count=$(grep -c --fixed-strings -- "$pass" ${TEMP_DIR}/logs_output-$p-$c.txt || :) - if [[ $count != 0 ]]; then - echo leaked passwords are found in log ${TEMP_DIR}/logs_output-$p-$c.txt - false - fi - done - done - echo - done - } - - collect_logs $NAMESPACE - if [ -n "$OPERATOR_NS" ]; then - pods=$(kubectl -n "${OPERATOR_NS}" get pods -o name | awk -F "/" '{print $2}') - collect_logs $OPERATOR_NS - fi + local secrets + local passwords + local pods + + secrets=$(kubectl get secrets -o json | jq -r '.items[].data | to_entries | .[] | select(.key | (endswith(".crt") or endswith(".key") or endswith(".pub") or endswith(".pem") or endswith(".p12") or test("namespace")) | not) | .value') + passwords="$(for i in $secrets; do + base64 -d <<<$i + echo + done) $secrets" + pods=$(kubectl -n "${NAMESPACE}" get pods -o name | awk -F "/" '{print $2}') + + collect_logs() { + local containers + local count + + NS=$1 + for p in $pods; do + local containers=$(kubectl -n "$NS" get pod $p -o jsonpath='{.spec.containers[*].name}') + for c in $containers; do + kubectl -n "$NS" logs $p -c $c >${TEMP_DIR}/logs_output-$p-$c.txt + echo logs saved in: ${TEMP_DIR}/logs_output-$p-$c.txt + for pass in $passwords; do + local count=$(grep -c --fixed-strings -- "$pass" ${TEMP_DIR}/logs_output-$p-$c.txt || :) + if [[ $count != 0 ]]; then + echo leaked passwords are found in log ${TEMP_DIR}/logs_output-$p-$c.txt + false + fi + done + done + echo + done + } + + collect_logs $NAMESPACE + if [ -n "$OPERATOR_NS" ]; then + local pods=$(kubectl -n "${OPERATOR_NS}" get pods -o name | awk -F "/" '{print $2}') + collect_logs $OPERATOR_NS + fi } deploy_chaos_mesh() { - destroy_chaos_mesh + destroy_chaos_mesh - helm repo add chaos-mesh https://charts.chaos-mesh.org - if [ -n "${MINIKUBE}" ]; then - helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${NAMESPACE} --set chaosDaemon.runtime=docker --set dashboard.create=false --version 2.5.1 - else - helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${NAMESPACE} --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.5.1 - fi - sleep 10 + helm repo add chaos-mesh https://charts.chaos-mesh.org + if [ -n "${MINIKUBE}" ]; then + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${NAMESPACE} --set chaosDaemon.runtime=docker --set dashboard.create=false --version 2.5.1 + else + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${NAMESPACE} --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.5.1 + fi + sleep 10 } destroy_chaos_mesh() { - local chaos_mesh_ns=$(helm list --all-namespaces --filter chaos-mesh | tail -n1 | awk -F' ' '{print $2}' | sed 's/NAMESPACE//') - - if [ -n "${chaos_mesh_ns}" ]; then - helm uninstall --wait --timeout 60s chaos-mesh --namespace ${chaos_mesh_ns} || : - fi - timeout 30 kubectl delete MutatingWebhookConfiguration $(kubectl get MutatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || : - timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || : - timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'validate-auth' | awk '{print $1}') || : - for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}'); do - kubectl get ${i} --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace \ - | while read -r line; do - local kind=$(echo "$line" | awk '{print $1}') - local name=$(echo "$line" | awk '{print $2}') - local namespace=$(echo "$line" | awk '{print $3}') - kubectl patch $kind $name -n "${namespace}" --type=merge -p '{"metadata":{"finalizers":[]}}' || : - done - timeout 30 kubectl delete ${i} --all --all-namespaces || : - done - timeout 30 kubectl delete crd $(kubectl get crd | grep 'chaos-mesh.org' | awk '{print $1}') || : - timeout 30 kubectl delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'chaos-mesh' | awk '{print $1}') || : - timeout 30 kubectl delete clusterrole $(kubectl get clusterrole | grep 'chaos-mesh' | awk '{print $1}') || : + local chaos_mesh_ns=$(helm list --all-namespaces --filter chaos-mesh | tail -n1 | awk -F' ' '{print $2}' | sed 's/NAMESPACE//') + + if [ -n "${chaos_mesh_ns}" ]; then + helm uninstall --wait --timeout 60s chaos-mesh --namespace ${chaos_mesh_ns} || : + fi + timeout 30 kubectl delete MutatingWebhookConfiguration $(kubectl get MutatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || : + timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || : + timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'validate-auth' | awk '{print $1}') || : + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}'); do + kubectl get ${i} --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace | + while read -r line; do + local kind=$(echo "$line" | awk '{print $1}') + local name=$(echo "$line" | awk '{print $2}') + local namespace=$(echo "$line" | awk '{print $3}') + kubectl patch $kind $name -n "${namespace}" --type=merge -p '{"metadata":{"finalizers":[]}}' || : + done + timeout 30 kubectl delete ${i} --all --all-namespaces || : + done + timeout 30 kubectl delete crd $(kubectl get crd | grep 'chaos-mesh.org' | awk '{print $1}') || : + timeout 30 kubectl delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'chaos-mesh' | awk '{print $1}') || : + timeout 30 kubectl delete clusterrole $(kubectl get clusterrole | grep 'chaos-mesh' | awk '{print $1}') || : } kill_pods() { - local ns=$1 - local selector=$2 - local pod_label=$3 - local label_value=$4 - local chaos_suffix=$5 - - if [ "${selector}" == "pod" ]; then - yq eval ' + local ns=$1 + local selector=$2 + local pod_label=$3 + local label_value=$4 + local chaos_suffix=$5 + + if [ "${selector}" == "pod" ]; then + yq eval ' .metadata.name = "chaos-pod-kill-'${chaos_suffix}'" | del(.spec.selector.pods.test-namespace) | - .spec.selector.pods.'${ns}'[0] = "'${pod_label}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \ - | kubectl apply --namespace ${ns} -f - - elif [ "${selector}" == "label" ]; then - yq eval ' + .spec.selector.pods.'${ns}'[0] = "'${pod_label}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml | + kubectl apply --namespace ${ns} -f - + elif [ "${selector}" == "label" ]; then + yq eval ' .metadata.name = "chaos-kill-label-'${chaos_suffix}'" | .spec.mode = "all" | del(.spec.selector.pods) | - .spec.selector.labelSelectors."'${pod_label}'" = "'${label_value}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \ - | kubectl apply --namespace ${ns} -f - - fi - sleep 5 + .spec.selector.labelSelectors."'${pod_label}'" = "'${label_value}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml | + kubectl apply --namespace ${ns} -f - + fi + sleep 5 } failure_pod() { - local ns=$1 - local pod=$2 - local chaos_suffix=$3 + local ns=$1 + local pod=$2 + local chaos_suffix=$3 - yq eval ' + yq eval ' .metadata.name = "chaos-pod-failure-'${chaos_suffix}'" | del(.spec.selector.pods.test-namespace) | - .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-pod-failure.yml \ - | kubectl apply --namespace ${ns} -f - - sleep 5 + .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-pod-failure.yml | + kubectl apply --namespace ${ns} -f - + sleep 5 } network_loss() { - local ns=$1 - local pod=$2 - local chaos_suffix=$3 + local ns=$1 + local pod=$2 + local chaos_suffix=$3 - yq eval ' + yq eval ' .metadata.name = "chaos-pod-network-loss-'${chaos_suffix}'" | del(.spec.selector.pods.test-namespace) | - .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-network-loss.yml \ - | kubectl apply --namespace ${ns} -f - - sleep 5 + .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-network-loss.yml | + kubectl apply --namespace ${ns} -f - + sleep 5 } renew_certificate() { - certificate="$1" + certificate="$1" - local pod_name - pod_name=$(kubectl get -n "${NAMESPACE}" pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}') + local pod_name + pod_name=$(kubectl get -n "${NAMESPACE}" pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}') - local revision - revision=$(kubectl get -n "${NAMESPACE}" certificate "$certificate" -o 'jsonpath={.status.revision}') + local revision + revision=$(kubectl get -n "${NAMESPACE}" certificate "$certificate" -o 'jsonpath={.status.revision}') - kubectl exec -n "${NAMESPACE}" "$pod_name" -- /tmp/cmctl renew "$certificate" + kubectl exec -n "${NAMESPACE}" "$pod_name" -- /tmp/cmctl renew "$certificate" - # wait for new revision - for i in {1..10}; do - local new_revision - new_revision=$(kubectl get -n "${NAMESPACE}" certificate "$certificate" -o 'jsonpath={.status.revision}') - if [ "$((revision + 1))" == "$new_revision" ]; then - break - fi - sleep 1 - done + # wait for new revision + for i in {1..10}; do + local new_revision + new_revision=$(kubectl get -n "${NAMESPACE}" certificate "$certificate" -o 'jsonpath={.status.revision}') + if [ "$((revision + 1))" == "$new_revision" ]; then + break + fi + sleep 1 + done } deploy_cmctl() { - local service_account="cmctl" + local service_account="cmctl" - sed -e "s/percona-server-mysql-operator/$service_account/g" "${DEPLOY_DIR}/rbac.yaml" \ - | yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' \ - | kubectl apply -n "${NAMESPACE}" -f - - kubectl apply -n "${NAMESPACE}" -f "${TESTS_CONFIG_DIR}/cmctl.yml" + sed -e "s/percona-server-mysql-operator/$service_account/g" "${DEPLOY_DIR}/rbac.yaml" | + yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' | + kubectl apply -n "${NAMESPACE}" -f - + kubectl apply -n "${NAMESPACE}" -f "${TESTS_CONFIG_DIR}/cmctl.yml" } diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index 11f0d1ecb..bd978939e 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -21,6 +21,7 @@ one-pod operator-self-healing recreate scaling +self-healing service-per-pod sidecars smart-update diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index fa1b86161..f964f083a 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -25,6 +25,7 @@ one-pod operator-self-healing recreate scaling +self-healing service-per-pod sidecars smart-update diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index fa1b86161..f964f083a 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -25,6 +25,7 @@ one-pod operator-self-healing recreate scaling +self-healing service-per-pod sidecars smart-update diff --git a/e2e-tests/tests/gr-self-healing/18-drop-finalizer.yaml b/e2e-tests/tests/gr-self-healing/18-drop-finalizer.yaml new file mode 100644 index 000000000..98952bc22 --- /dev/null +++ b/e2e-tests/tests/gr-self-healing/18-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-self-healing + finalizers: [] diff --git a/e2e-tests/tests/operator-self-healing/02-assert.yaml b/e2e-tests/tests/operator-self-healing/02-assert.yaml index 14b5ab5b9..d6e507c96 100644 --- a/e2e-tests/tests/operator-self-healing/02-assert.yaml +++ b/e2e-tests/tests/operator-self-healing/02-assert.yaml @@ -58,16 +58,3 @@ status: size: 3 state: ready state: ready ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: chaos-daemon -status: - currentNumberScheduled: 3 - desiredNumberScheduled: 3 - numberAvailable: 3 - numberMisscheduled: 0 - numberReady: 3 - observedGeneration: 1 - updatedNumberScheduled: 3 diff --git a/e2e-tests/tests/operator-self-healing/12-drop-finalizer.yaml b/e2e-tests/tests/operator-self-healing/12-drop-finalizer.yaml new file mode 100644 index 000000000..73ab6351b --- /dev/null +++ b/e2e-tests/tests/operator-self-healing/12-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: operator-self-healing + finalizers: [] diff --git a/e2e-tests/tests/self-healing/00-assert.yaml b/e2e-tests/tests/self-healing/00-assert.yaml new file mode 100644 index 000000000..d9146fe1b --- /dev/null +++ b/e2e-tests/tests/self-healing/00-assert.yaml @@ -0,0 +1,26 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconaservermysqls.ps.percona.com +spec: + group: ps.percona.com + names: + kind: PerconaServerMySQL + listKind: PerconaServerMySQLList + plural: perconaservermysqls + shortNames: + - ps + singular: perconaservermysql + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/self-healing/00-deploy-operator.yaml b/e2e-tests/tests/self-healing/00-deploy-operator.yaml new file mode 100644 index 000000000..67307fe5d --- /dev/null +++ b/e2e-tests/tests/self-healing/00-deploy-operator.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_operator + deploy_non_tls_cluster_secrets + deploy_tls_cluster_secrets + deploy_client diff --git a/e2e-tests/tests/self-healing/01-assert.yaml b/e2e-tests/tests/self-healing/01-assert.yaml new file mode 100644 index 000000000..9caa36184 --- /dev/null +++ b/e2e-tests/tests/self-healing/01-assert.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: chaos-controller-manager +spec: + replicas: 3 +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: chaos-daemon +status: + currentNumberScheduled: 3 + desiredNumberScheduled: 3 + numberAvailable: 3 + numberMisscheduled: 0 + numberReady: 3 + updatedNumberScheduled: 3 diff --git a/e2e-tests/tests/self-healing/01-deploy-chaos-mesh.yaml b/e2e-tests/tests/self-healing/01-deploy-chaos-mesh.yaml new file mode 100644 index 000000000..e5638625e --- /dev/null +++ b/e2e-tests/tests/self-healing/01-deploy-chaos-mesh.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_chaos_mesh + timeout: 120 diff --git a/e2e-tests/tests/self-healing/02-assert.yaml b/e2e-tests/tests/self-healing/02-assert.yaml new file mode 100644 index 000000000..f2da509cf --- /dev/null +++ b/e2e-tests/tests/self-healing/02-assert.yaml @@ -0,0 +1,60 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/self-healing/02-create-cluster.yaml b/e2e-tests/tests/self-healing/02-create-cluster.yaml new file mode 100644 index 000000000..3dd3dea43 --- /dev/null +++ b/e2e-tests/tests/self-healing/02-create-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval '.spec.mysql.size=3' - \ + | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \ + | yq eval '.spec.proxy.haproxy.enabled=true' - \ + | yq eval '.spec.proxy.haproxy.size=3' - \ + | yq eval '.spec.proxy.haproxy.affinity.antiAffinityTopologyKey="none"' - \ + | yq eval '.spec.orchestrator.enabled=true' - \ + | yq eval '.spec.orchestrator.size=3' - \ + | yq eval '.spec.orchestrator.affinity.antiAffinityTopologyKey="none"' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/self-healing/03-write-data.yaml b/e2e-tests/tests/self-healing/03-write-data.yaml new file mode 100644 index 000000000..bc82e7920 --- /dev/null +++ b/e2e-tests/tests/self-healing/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/self-healing/04-assert.yaml b/e2e-tests/tests/self-healing/04-assert.yaml new file mode 100644 index 000000000..8a8037060 --- /dev/null +++ b/e2e-tests/tests/self-healing/04-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 04-read-from-primary +data: + data: "100500" diff --git a/e2e-tests/tests/self-healing/04-read-from-primary.yaml b/e2e-tests/tests/self-healing/04-read-from-primary.yaml new file mode 100644 index 000000000..274332522 --- /dev/null +++ b/e2e-tests/tests/self-healing/04-read-from-primary.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password") + + kubectl create configmap -n "${NAMESPACE}" 04-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/self-healing/05-assert.yaml b/e2e-tests/tests/self-healing/05-assert.yaml new file mode 100644 index 000000000..7e8ec76e7 --- /dev/null +++ b/e2e-tests/tests/self-healing/05-assert.yaml @@ -0,0 +1,79 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: chaos-pod-kill-primary +spec: + action: pod-kill + mode: one +status: + experiment: + containerRecords: + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + desiredPhase: Run diff --git a/e2e-tests/tests/self-healing/05-kill-primary.yaml b/e2e-tests/tests/self-healing/05-kill-primary.yaml new file mode 100644 index 000000000..1dc699052 --- /dev/null +++ b/e2e-tests/tests/self-healing/05-kill-primary.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + init_pod="$(get_primary_from_haproxy ${test_name}-haproxy-0)" + kill_pods "${NAMESPACE}" "pod" "$init_pod" "" "primary" + sleep 10 # wait a bit for pod to be killed + + if [ "$init_pod" == "$(get_primary_from_haproxy ${test_name}-haproxy-0)" ]; then + echo "primary pod was not killed! something went wrong." + exit 1 + fi diff --git a/e2e-tests/tests/self-healing/06-write-data.yaml b/e2e-tests/tests/self-healing/06-write-data.yaml new file mode 100644 index 000000000..dada31537 --- /dev/null +++ b/e2e-tests/tests/self-healing/06-write-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100501)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/self-healing/07-assert.yaml b/e2e-tests/tests/self-healing/07-assert.yaml new file mode 100644 index 000000000..d5acf9414 --- /dev/null +++ b/e2e-tests/tests/self-healing/07-assert.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-from-replicas-0 +data: + data: |- + 100500 + 100501 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-from-replicas-1 +data: + data: |- + 100500 + 100501 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-from-replicas-2 +data: + data: |- + 100500 + 100501 diff --git a/e2e-tests/tests/self-healing/07-read-from-replicas.yaml b/e2e-tests/tests/self-healing/07-read-from-replicas.yaml new file mode 100644 index 000000000..30f4d2649 --- /dev/null +++ b/e2e-tests/tests/self-healing/07-read-from-replicas.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for i in 0 1 2; do + host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") + kubectl create configmap -n "${NAMESPACE}" 07-read-from-replicas-${i} --from-literal=data="${data}" + done diff --git a/e2e-tests/tests/self-healing/08-assert.yaml b/e2e-tests/tests/self-healing/08-assert.yaml new file mode 100644 index 000000000..41a5dd87d --- /dev/null +++ b/e2e-tests/tests/self-healing/08-assert.yaml @@ -0,0 +1,82 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 180 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: chaos-pod-failure-primary +spec: + action: pod-failure + duration: 60s + mode: one +status: + experiment: + containerRecords: + - events: + - operation: Apply + type: Succeeded + - operation: Recover + type: Succeeded + injectedCount: 1 + phase: Not Injected + recoveredCount: 1 + selectorKey: . + desiredPhase: Stop diff --git a/e2e-tests/tests/self-healing/08-failure-primary.yaml b/e2e-tests/tests/self-healing/08-failure-primary.yaml new file mode 100644 index 000000000..b159a17c4 --- /dev/null +++ b/e2e-tests/tests/self-healing/08-failure-primary.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + failure_pod "${NAMESPACE}" "$(get_primary_from_haproxy ${test_name}-haproxy-0)" "primary" + sleep 10 # wait a bit for pod to be killed diff --git a/e2e-tests/tests/self-healing/09-write-data.yaml b/e2e-tests/tests/self-healing/09-write-data.yaml new file mode 100644 index 000000000..b84c28860 --- /dev/null +++ b/e2e-tests/tests/self-healing/09-write-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100502)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/self-healing/10-assert.yaml b/e2e-tests/tests/self-healing/10-assert.yaml new file mode 100644 index 000000000..2f9ba0826 --- /dev/null +++ b/e2e-tests/tests/self-healing/10-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 10-read-from-replicas-0 +data: + data: |- + 100500 + 100501 + 100502 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 10-read-from-replicas-1 +data: + data: |- + 100500 + 100501 + 100502 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 10-read-from-replicas-2 +data: + data: |- + 100500 + 100501 + 100502 diff --git a/e2e-tests/tests/self-healing/10-read-from-replicas.yaml b/e2e-tests/tests/self-healing/10-read-from-replicas.yaml new file mode 100644 index 000000000..55a419ab7 --- /dev/null +++ b/e2e-tests/tests/self-healing/10-read-from-replicas.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for i in 0 1 2; do + host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") + kubectl create configmap -n "${NAMESPACE}" 10-read-from-replicas-${i} --from-literal=data="${data}" + done diff --git a/e2e-tests/tests/self-healing/11-assert.yaml b/e2e-tests/tests/self-healing/11-assert.yaml new file mode 100644 index 000000000..c75f54535 --- /dev/null +++ b/e2e-tests/tests/self-healing/11-assert.yaml @@ -0,0 +1,86 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: NetworkChaos +metadata: + name: chaos-pod-network-loss-primary +spec: + action: loss + direction: to + duration: 60s + loss: + correlation: "100" + loss: "100" + mode: one +status: + experiment: + containerRecords: + - events: + - operation: Apply + type: Succeeded + - operation: Recover + type: Succeeded + injectedCount: 1 + phase: Not Injected + recoveredCount: 1 + selectorKey: . + desiredPhase: Stop diff --git a/e2e-tests/tests/self-healing/11-network-loss-primary.yaml b/e2e-tests/tests/self-healing/11-network-loss-primary.yaml new file mode 100644 index 000000000..599a9d82e --- /dev/null +++ b/e2e-tests/tests/self-healing/11-network-loss-primary.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 90 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + network_loss "${NAMESPACE}" "$(get_primary_from_haproxy ${test_name}-haproxy-0)" "primary" + sleep 30 # wait for new master to get elected + timeout: 90 diff --git a/e2e-tests/tests/self-healing/12-write-data.yaml b/e2e-tests/tests/self-healing/12-write-data.yaml new file mode 100644 index 000000000..3dce72866 --- /dev/null +++ b/e2e-tests/tests/self-healing/12-write-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100503)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/self-healing/13-assert.yaml b/e2e-tests/tests/self-healing/13-assert.yaml new file mode 100644 index 000000000..b21c4894e --- /dev/null +++ b/e2e-tests/tests/self-healing/13-assert.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-from-replicas-0 +data: + data: |- + 100500 + 100501 + 100502 + 100503 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-from-replicas-1 +data: + data: |- + 100500 + 100501 + 100502 + 100503 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-from-replicas-2 +data: + data: |- + 100500 + 100501 + 100502 + 100503 diff --git a/e2e-tests/tests/self-healing/13-read-from-replicas.yaml b/e2e-tests/tests/self-healing/13-read-from-replicas.yaml new file mode 100644 index 000000000..77b99efc0 --- /dev/null +++ b/e2e-tests/tests/self-healing/13-read-from-replicas.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for i in 0 1 2; do + host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") + kubectl create configmap -n "${NAMESPACE}" 13-read-from-replicas-${i} --from-literal=data="${data}" + done diff --git a/e2e-tests/tests/self-healing/14-assert.yaml b/e2e-tests/tests/self-healing/14-assert.yaml new file mode 100644 index 000000000..aea139b02 --- /dev/null +++ b/e2e-tests/tests/self-healing/14-assert.yaml @@ -0,0 +1,135 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 480 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: self-healing-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: chaos-kill-label-cluster-crash +spec: + action: pod-kill + mode: all +status: + experiment: + containerRecords: + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + - events: + - operation: Apply + type: Succeeded + injectedCount: 1 + phase: Injected + recoveredCount: 0 + selectorKey: . + desiredPhase: Run diff --git a/e2e-tests/tests/self-healing/14-cluster-crash.yaml b/e2e-tests/tests/self-healing/14-cluster-crash.yaml new file mode 100644 index 000000000..f4890c894 --- /dev/null +++ b/e2e-tests/tests/self-healing/14-cluster-crash.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kill_pods "${NAMESPACE}" "label" "app.kubernetes.io/instance" "self-healing" "cluster-crash" + sleep 30 # wait for crash diff --git a/e2e-tests/tests/self-healing/15-write-data.yaml b/e2e-tests/tests/self-healing/15-write-data.yaml new file mode 100644 index 000000000..9c5791f6d --- /dev/null +++ b/e2e-tests/tests/self-healing/15-write-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100504)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/self-healing/16-assert.yaml b/e2e-tests/tests/self-healing/16-assert.yaml new file mode 100644 index 000000000..5f278c285 --- /dev/null +++ b/e2e-tests/tests/self-healing/16-assert.yaml @@ -0,0 +1,39 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 16-read-from-replicas-0 +data: + data: |- + 100500 + 100501 + 100502 + 100503 + 100504 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 16-read-from-replicas-1 +data: + data: |- + 100500 + 100501 + 100502 + 100503 + 100504 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 16-read-from-replicas-2 +data: + data: |- + 100500 + 100501 + 100502 + 100503 + 100504 diff --git a/e2e-tests/tests/self-healing/16-read-from-replicas.yaml b/e2e-tests/tests/self-healing/16-read-from-replicas.yaml new file mode 100644 index 000000000..ab4cfa84c --- /dev/null +++ b/e2e-tests/tests/self-healing/16-read-from-replicas.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for i in 0 1 2; do + host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") + kubectl create configmap -n "${NAMESPACE}" 16-read-from-replicas-${i} --from-literal=data="${data}" + done diff --git a/e2e-tests/tests/self-healing/17-destroy-chaos-mesh.yaml b/e2e-tests/tests/self-healing/17-destroy-chaos-mesh.yaml new file mode 100644 index 000000000..3f0cbc6b8 --- /dev/null +++ b/e2e-tests/tests/self-healing/17-destroy-chaos-mesh.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_chaos_mesh + timeout: 120 diff --git a/e2e-tests/tests/self-healing/18-drop-finalizer.yaml b/e2e-tests/tests/self-healing/18-drop-finalizer.yaml new file mode 100644 index 000000000..38885e7ef --- /dev/null +++ b/e2e-tests/tests/self-healing/18-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: self-healing + finalizers: []