diff --git a/cmd/pitr/collector/collector.go b/cmd/pitr/collector/collector.go index 34058c326e..ab2a828ed2 100644 --- a/cmd/pitr/collector/collector.go +++ b/cmd/pitr/collector/collector.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/go-sql-driver/mysql" "github.com/minio/minio-go/v7" "github.com/pkg/errors" @@ -163,31 +164,10 @@ func (c *Collector) close() error { return c.db.Close() } -func (c *Collector) CurrentSourceID(ctx context.Context, logs []pxc.Binlog) (string, error) { - var ( - gtidSet string - err error - ) - for i := len(logs) - 1; i >= 0 && gtidSet == ""; i-- { - gtidSet, err = c.db.GetGTIDSet(ctx, logs[i].Name) - if err != nil { - return gtidSet, err - } - } - return strings.Split(gtidSet, ":")[0], nil -} - func (c *Collector) removeEmptyBinlogs(ctx context.Context, logs []pxc.Binlog) ([]pxc.Binlog, error) { result := make([]pxc.Binlog, 0) for _, v := range logs { - set, err := c.db.GetGTIDSet(ctx, v.Name) - if err != nil { - return nil, errors.Wrap(err, "get GTID set") - } - // we don't upload binlog without gtid - // because it is empty and doesn't have any information - if set != "" { - v.GTIDSet = set + if v.GTIDSet != "" { result = append(result, v) } } @@ -238,15 +218,33 @@ func createGapFile(gtidSet string) error { return nil } +func (c *Collector) addGTIDSets(ctx context.Context, logs []pxc.Binlog) error { + for i, v := range logs { + set, err := c.db.GetGTIDSet(ctx, v.Name) + if err != nil { + if errors.Is(err, &mysql.MySQLError{Number: 3200}) { + log.Printf("ERROR: Binlog file %s is invalid on host %s: %s\n", v.Name, c.db.GetHost(), err.Error()) + continue + } + return errors.Wrap(err, "get GTID set") + } + logs[i].GTIDSet = set + } + return nil +} + func (c *Collector) CollectBinLogs(ctx context.Context) error { list, err := c.db.GetBinLogList(ctx) if err != nil { return errors.Wrap(err, "get binlog list") } - - sourceID, err := c.CurrentSourceID(ctx, list) + err = c.addGTIDSets(ctx, list) if err != nil { - return errors.Wrap(err, "get current source id") + return errors.Wrap(err, "get GTID sets") + } + var sourceID string + for i := len(list) - 1; i >= 0 && sourceID == ""; i-- { + sourceID = strings.Split(list[i].GTIDSet, ":")[0] } if sourceID == "" { @@ -262,10 +260,11 @@ func (c *Collector) CollectBinLogs(ctx context.Context) error { lastUploadedBinlogName := "" if c.lastSet != "" { - // get last uploaded binlog file name - lastUploadedBinlogName, err = c.db.GetBinLogName(ctx, c.lastSet) - if err != nil { - return errors.Wrap(err, "get last uploaded binlog name by gtid set") + for i := len(list) - 1; i >= 0; i-- { + if list[i].GTIDSet == c.lastSet { + lastUploadedBinlogName = list[i].Name + break + } } if lastUploadedBinlogName == "" { diff --git a/cmd/pitr/pxc/pxc.go b/cmd/pitr/pxc/pxc.go index 7066dc93c6..fb5fc941b8 100644 --- a/cmd/pitr/pxc/pxc.go +++ b/cmd/pitr/pxc/pxc.go @@ -129,34 +129,6 @@ func (p *PXC) GetBinLogNamesList(ctx context.Context) ([]string, error) { return binlogs, nil } -// GetBinLogName returns name of the binary log file by given GTID set -func (p *PXC) GetBinLogName(ctx context.Context, gtidSet string) (string, error) { - if len(gtidSet) == 0 { - return "", nil - } - var existFunc string - nameRow := p.db.QueryRowContext(ctx, "select name from mysql.func where name='get_binlog_by_gtid_set'") - err := nameRow.Scan(&existFunc) - if err != nil && err != sql.ErrNoRows { - return "", errors.Wrap(err, "get udf name") - } - if len(existFunc) == 0 { - _, err = p.db.ExecContext(ctx, "CREATE FUNCTION get_binlog_by_gtid_set RETURNS STRING SONAME 'binlog_utils_udf.so'") - if err != nil { - return "", errors.Wrap(err, "create function") - } - } - var binlog sql.NullString - row := p.db.QueryRowContext(ctx, "SELECT get_binlog_by_gtid_set(?)", gtidSet) - - err = row.Scan(&binlog) - if err != nil { - return "", errors.Wrap(err, "scan binlog") - } - - return strings.TrimPrefix(binlog.String, "./"), nil -} - // GetBinLogFirstTimestamp return binary log file first timestamp func (p *PXC) GetBinLogFirstTimestamp(ctx context.Context, binlog string) (string, error) { var existFunc string diff --git a/e2e-tests/functions b/e2e-tests/functions index 4f08786ad8..3456530861 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -17,7 +17,7 @@ SKIP_REMOTE_BACKUPS=${SKIP_REMOTE_BACKUPS:-1} PMM_SERVER_VER=${PMM_SERVER_VER:-"9.9.9"} IMAGE_PMM_SERVER_REPO=${IMAGE_PMM_SERVER_REPO:-"perconalab/pmm-server"} IMAGE_PMM_SERVER_TAG=${IMAGE_PMM_SERVER_TAG:-"dev-latest"} -CERT_MANAGER_VER="1.8.0" +CERT_MANAGER_VER="1.12.1" tmp_dir=$(mktemp -d) sed=$(which gsed || which sed) date=$(which gdate || which date) @@ -509,7 +509,7 @@ run_mysql_local() { local container_name="$4" set +o xtrace kubectl_bin exec $pod ${container_name:+-c $container_name} -- \ - bash -c "printf '$command\n' | mysql -sN $uri" 2>&1 \ + bash -c "printf \"$command\n\" | mysql -sN $uri" 2>&1 \ | sed -e 's/mysql: //' \ | (egrep -v 'Using a password on the command line interface can be insecure.|Defaulted container|Defaulting container name|see all of the containers in this pod' || :) set_debug @@ -562,11 +562,11 @@ compare_mysql_cmd_local() { get_proxy_primary() { local uri="$1" local pod="$2" - local ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql') + local ip=$(run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='ONLINE';" "$uri" "$pod" 'proxysql') while [ $(echo "$ip" | wc -l) != 1 ]; do sleep 1 - ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql') + ip=$(run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='ONLINE';" "$uri" "$pod" 'proxysql') done echo $ip | cut -d'.' -f1 diff --git a/e2e-tests/pitr-gap-errors/compare/select-1.sql b/e2e-tests/pitr-gap-errors/compare/select-1.sql new file mode 100644 index 0000000000..8e738f4cf2 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/compare/select-1.sql @@ -0,0 +1 @@ +100500 diff --git a/e2e-tests/pitr-gap-errors/compare/select-3.sql b/e2e-tests/pitr-gap-errors/compare/select-3.sql new file mode 100644 index 0000000000..769df1828a --- /dev/null +++ b/e2e-tests/pitr-gap-errors/compare/select-3.sql @@ -0,0 +1,3 @@ +100500 +100501 +100502 diff --git a/e2e-tests/pitr/compare/select-5.sql b/e2e-tests/pitr-gap-errors/compare/select-5.sql similarity index 100% rename from e2e-tests/pitr/compare/select-5.sql rename to e2e-tests/pitr-gap-errors/compare/select-5.sql diff --git a/e2e-tests/pitr-gap-errors/compare/select-6.sql b/e2e-tests/pitr-gap-errors/compare/select-6.sql new file mode 100644 index 0000000000..f28933c75e --- /dev/null +++ b/e2e-tests/pitr-gap-errors/compare/select-6.sql @@ -0,0 +1,10 @@ +100500 +100501 +100502 +100503 +100504 +100505 +100506 +100507 +100508 +100509 diff --git a/e2e-tests/pitr/compare/select-gap.sql b/e2e-tests/pitr-gap-errors/compare/select-gap.sql similarity index 100% rename from e2e-tests/pitr/compare/select-gap.sql rename to e2e-tests/pitr-gap-errors/compare/select-gap.sql diff --git a/e2e-tests/pitr-gap-errors/compare/select-invalid.sql b/e2e-tests/pitr-gap-errors/compare/select-invalid.sql new file mode 100644 index 0000000000..aff30726c6 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/compare/select-invalid.sql @@ -0,0 +1,2 @@ +ERROR 1146 (42S02) at line 1: Table 'test.invalid' doesn't exist +command terminated with exit code 1 diff --git a/e2e-tests/pitr-gap-errors/conf/cert.yml b/e2e-tests/pitr-gap-errors/conf/cert.yml new file mode 100644 index 0000000000..5faf14d524 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/cert.yml @@ -0,0 +1,9 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: tls-minio +spec: + commonName: minio-service + secretName: tls-minio + issuerRef: + name: selfsigning-issuer \ No newline at end of file diff --git a/e2e-tests/pitr-gap-errors/conf/issuer.yml b/e2e-tests/pitr-gap-errors/conf/issuer.yml new file mode 100644 index 0000000000..275093f73f --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/issuer.yml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigning-issuer +spec: + selfSigned: {} \ No newline at end of file diff --git a/e2e-tests/pitr/conf/on-pitr-minio-gap.yml b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio-gap.yml similarity index 80% rename from e2e-tests/pitr/conf/on-pitr-minio-gap.yml rename to e2e-tests/pitr-gap-errors/conf/on-pitr-minio-gap.yml index 4d16f15544..ed8a6e792a 100755 --- a/e2e-tests/pitr/conf/on-pitr-minio-gap.yml +++ b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio-gap.yml @@ -3,5 +3,5 @@ kind: PerconaXtraDBClusterBackup metadata: name: on-pitr-minio-gap spec: - pxcCluster: pitr + pxcCluster: pitr-gap-errors storageName: minio diff --git a/e2e-tests/pitr-gap-errors/conf/on-pitr-minio.yml b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio.yml new file mode 100755 index 0000000000..7ad0e8c4eb --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio.yml @@ -0,0 +1,7 @@ +apiVersion: pxc.percona.com/v1 +kind: PerconaXtraDBClusterBackup +metadata: + name: on-pitr-minio +spec: + pxcCluster: pitr-gap-errors + storageName: minio diff --git a/e2e-tests/pitr/conf/on-pitr-minio1.yml b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio1.yml similarity index 80% rename from e2e-tests/pitr/conf/on-pitr-minio1.yml rename to e2e-tests/pitr-gap-errors/conf/on-pitr-minio1.yml index 16c4937bbc..05ad195a31 100755 --- a/e2e-tests/pitr/conf/on-pitr-minio1.yml +++ b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio1.yml @@ -3,5 +3,5 @@ kind: PerconaXtraDBClusterBackup metadata: name: on-pitr-minio1 spec: - pxcCluster: pitr + pxcCluster: pitr-gap-errors storageName: minio diff --git a/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml b/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml new file mode 100755 index 0000000000..1a019e39d3 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml @@ -0,0 +1,81 @@ +apiVersion: pxc.percona.com/v1-6-0 +kind: PerconaXtraDBCluster +metadata: + name: pitr-gap-errors + finalizers: + - delete-pxc-pods-in-order + # annotations: + # percona.com/issue-vault-token: "true" +spec: + secretsName: my-cluster-secrets + vaultSecretName: some-name-vault + pause: false + pxc: + size: 3 + image: -pxc + configuration: | + [sst] + xbstream-opts=--decompress + [xtrabackup] + compress=lz4 + resources: + requests: + memory: 0.1G + cpu: 100m + limits: + memory: "1G" + cpu: "1" + volumeSpec: + persistentVolumeClaim: + resources: + requests: + storage: 2Gi + affinity: + antiAffinityTopologyKey: "kubernetes.io/hostname" + proxysql: + enabled: true + size: 2 + image: -proxysql + resources: + requests: + memory: 0.1G + cpu: 100m + limits: + memory: 1G + cpu: 700m + volumeSpec: + persistentVolumeClaim: + resources: + requests: + storage: 2Gi + affinity: + antiAffinityTopologyKey: "kubernetes.io/hostname" + pmm: + enabled: false + image: perconalab/pmm-client:1.17.1 + serverHost: monitoring-service + serverUser: pmm + backup: + image: -backup + imagePullPolicy: Always + pitr: + enabled: true + storageName: minio-binlogs + timeBetweenUploads: 55 + storages: + minio: + type: s3 + verifyTLS: false + s3: + credentialsSecret: minio-secret + region: us-east-1 + bucket: operator-testing + endpointUrl: https://minio-service:9000 + minio-binlogs: + type: s3 + verifyTLS: false + s3: + credentialsSecret: minio-secret + region: us-east-1 + bucket: operator-testing/binlogs + endpointUrl: https://minio-service:9000 diff --git a/e2e-tests/pitr/conf/restore-on-pitr-minio-gap-error.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-error.yaml similarity index 88% rename from e2e-tests/pitr/conf/restore-on-pitr-minio-gap-error.yaml rename to e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-error.yaml index af436e307a..48149abe35 100755 --- a/e2e-tests/pitr/conf/restore-on-pitr-minio-gap-error.yaml +++ b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-error.yaml @@ -3,7 +3,7 @@ kind: PerconaXtraDBClusterRestore metadata: name: on-pitr-minio-gap-error spec: - pxcCluster: pitr + pxcCluster: pitr-gap-errors backupName: on-pitr-minio-gap pitr: type: latest diff --git a/e2e-tests/pitr/conf/restore-on-pitr-minio-gap-force.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-force.yaml similarity index 90% rename from e2e-tests/pitr/conf/restore-on-pitr-minio-gap-force.yaml rename to e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-force.yaml index d3854085d0..77ef85701d 100755 --- a/e2e-tests/pitr/conf/restore-on-pitr-minio-gap-force.yaml +++ b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-gap-force.yaml @@ -5,7 +5,7 @@ metadata: percona.com/unsafe-pitr: "true" name: on-pitr-minio-gap-force spec: - pxcCluster: pitr + pxcCluster: pitr-gap-errors backupName: on-pitr-minio-gap pitr: type: latest diff --git a/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-invalid.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-invalid.yaml new file mode 100755 index 0000000000..40d66e91b0 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-invalid.yaml @@ -0,0 +1,14 @@ +apiVersion: pxc.percona.com/v1 +kind: PerconaXtraDBClusterRestore +metadata: + annotations: + percona.com/unsafe-pitr: "true" + name: on-pitr-minio-invalid +spec: + pxcCluster: pitr-gap-errors + backupName: on-pitr-minio + pitr: + type: transaction + gtid: + backupSource: + storageName: "minio-binlogs" diff --git a/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio.yaml new file mode 100755 index 0000000000..7a2ce986ac --- /dev/null +++ b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio.yaml @@ -0,0 +1,24 @@ +apiVersion: pxc.percona.com/v1 +kind: PerconaXtraDBClusterRestore +metadata: + name: on-pitr-minio +spec: + pxcCluster: pitr-gap-errors + backupSource: + verifyTLS: false + destination: + s3: + bucket: operator-testing + credentialsSecret: minio-secret + endpointUrl: https://minio-service:9000 + region: us-east-1 + pitr: + type: latest + backupSource: + verifyTLS: false + s3: + bucket: operator-testing/binlogs + credentialsSecret: minio-secret + endpointUrl: https://minio-service:9000 + region: us-east-1 + diff --git a/e2e-tests/pitr/conf/restore-on-pitr-minio1.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio1.yaml similarity index 94% rename from e2e-tests/pitr/conf/restore-on-pitr-minio1.yaml rename to e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio1.yaml index b36b090fbe..7f7e99fe1e 100755 --- a/e2e-tests/pitr/conf/restore-on-pitr-minio1.yaml +++ b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio1.yaml @@ -3,7 +3,7 @@ kind: PerconaXtraDBClusterRestore metadata: name: on-pitr-minio1 spec: - pxcCluster: pitr + pxcCluster: pitr-gap-errors backupSource: destination: verifyTLS: false diff --git a/e2e-tests/pitr-gap-errors/run b/e2e-tests/pitr-gap-errors/run new file mode 100755 index 0000000000..8caa522048 --- /dev/null +++ b/e2e-tests/pitr-gap-errors/run @@ -0,0 +1,382 @@ +#!/bin/bash + +set -o errexit + +test_dir=$(realpath $(dirname $0)) +. "${test_dir}/../functions" + +set_debug + +GTID_PATTERN='[A-F0-9a-f]{8}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{12}:[0-9]+' + +if [[ $IMAGE_PXC =~ 5\.7 ]]; then + echo "Skipping PITR test because 5.7 doesn't support it!" + exit 1 +fi + +run_recovery_check_pitr() { + local cluster=$1 + local restore=$2 + local backup=$3 + local compare=$4 + local time_now=$5 + local dest=$6 + local gtid=$7 + + desc 'recover backup' $restore + cat "$test_dir/conf/${restore}.yaml" \ + | $sed -e "s//${time_now}/g" \ + | $sed -e "s//${dest}/g" \ + | $sed -e "s//${gtid}/g" \ + | kubectl_bin apply -f - + wait_backup_restore ${backup} + kubectl_bin logs job/restore-job-${backup}-${cluster} + wait_for_running "$cluster-proxysql" 2 + wait_for_running "$cluster-pxc" 3 + wait_cluster_consistency "$cluster" 3 2 + desc 'check data after backup' $restore + compare_mysql_cmd $compare "SELECT * from test.test;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd $compare "SELECT * from test.test;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd $compare "SELECT * from test.test;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password" + kubectl_bin delete -f "$test_dir/conf/${restore}.yaml" +} + +write_test_data() { + local cluster=$1 + local config=$2 + local size="${3:-3}" + local sleep="${4:-10}" + local secretsFile="${5:-$conf_dir/secrets.yml}" + local pxcClientFile="${6:-$conf_dir/client.yml}" + + local proxy=$(get_proxy "$cluster") + + desc 'write test data' + if [[ $IMAGE_PXC =~ 5\.7 ]] && [[ "$(is_keyring_plugin_in_use "$cluster")" ]]; then + encrypt='ENCRYPTION=\"Y\"' + fi + run_mysql \ + "CREATE DATABASE IF NOT EXISTS test; use test; CREATE TABLE IF NOT EXISTS test (id int PRIMARY KEY) $encrypt;" \ + "-h $proxy -uroot -proot_password" + run_mysql \ + 'INSERT test.test (id) VALUES (100500); INSERT test.test (id) VALUES (100501); INSERT test.test (id) VALUES (100502);' \ + "-h $proxy -uroot -proot_password" + sleep 30 + for i in $(seq 0 $((size - 1))); do + compare_mysql_cmd "select-3" "SELECT * from test.test;" "-h $cluster-pxc-$i.$cluster-pxc -uroot -proot_password" + done + + if [ "$(is_keyring_plugin_in_use "$cluster")" ]; then + table_must_be_encrypted "$cluster" "test" + fi +} + +write_data_for_pitr() { + local cluster=$1 + local proxy=$(get_proxy "$cluster") + + desc "write data for pitr" + run_mysql \ + 'INSERT test.test (id) VALUES (100503); INSERT test.test (id) VALUES (100504); INSERT test.test (id) VALUES (100505);' \ + "-h $proxy -uroot -proot_password" +} + +write_more_data() { + local cluster=$1 + local proxy=$(get_proxy "$cluster") + desc "write extra data" + run_mysql \ + 'INSERT test.test (id) VALUES (100506); INSERT test.test (id) VALUES (100507); INSERT test.test (id) VALUES (100508); INSERT test.test (id) VALUES (100509); INSERT test.test (id) VALUES (100510);' \ + "-h $proxy -uroot -proot_password" +} + +create_binlog_gap() { + desc 'create binlog gap' + + kubectl patch pxc $cluster --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":false}}}}' + sleep 5 # wait for pitr pod to shutdown + # write data which will be lost + run_mysql \ + 'INSERT test.gap (id) VALUES (100800); INSERT test.gap (id) VALUES (100801); INSERT test.gap (id) VALUES (100802);' \ + "-h $proxy -uroot -proot_password" + # flush binlogs + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + # purge binlogs + run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + # re-enable pitr + kubectl patch pxc $cluster --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":true}}}}' + # flush binlogs to trigger binlog collector + # data below will be force recovered + run_mysql \ + 'INSERT test.gap (id) VALUES (100803); INSERT test.gap (id) VALUES (100804); INSERT test.gap (id) VALUES (100805);' \ + "-h $cluster-proxysql -uroot -proot_password" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + sleep 65 # wait for next PITR collect cycle and error to appear +} + +check_binlog_gap_error() { + desc 'check binlog gap error' + + # check error in pitr log + local err_text1=$(kubectl_bin logs $(get_pitr_pod) | grep -c "ERROR: Couldn't find the binlog that contains GTID set") + local err_text2=$(kubectl_bin logs $(get_pitr_pod) | grep -c "ERROR: Gap detected in the binary logs. Binary logs will be uploaded anyway, but full backup needed for consistent recovery.") + if [[ $err_text1 -eq 0 || $err_text2 -eq 0 ]]; then + echo "ERROR: Gap error text is not found in PITR pod logs." + exit 1 + fi + # check error in operator log + local err_text3=$(kubectl_bin logs ${OPERATOR_NS:+-n$OPERATOR_NS} $(get_operator_pod) | grep -c "Gap detected in binary logs") + if [[ $err_text3 -eq 0 ]]; then + echo "ERROR: Gap error text is not found in operator pod logs." + exit 1 + fi + # check backup on-pitr-minio-gap marked as unready for PITR restore + local backup_cond=$(kubectl_bin get pxc-backup on-pitr-minio-gap -ojsonpath='{.status.conditions[]}' | grep -c '"reason":"BinlogGapDetected","status":"False","type":"PITRReady"') + if [[ $backup_cond -eq 0 ]]; then + echo "ERROR: Backup is not tagged as PITR unready in the backup condition." + kubectl_bin get pxc-backup on-pitr-minio-gap -oyaml + exit 1 + fi +} + +check_binlog_gap_restore() { + local type=$1 + + desc 'check binlog gap restore: ' $type + # disable pitr + kubectl patch pxc $cluster --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":false}}}}' + # try restore, check error + if [ "$type" == "error" ]; then + kubectl_bin apply -f $test_dir/conf/restore-on-pitr-minio-gap-error.yaml + wait_backup_restore "on-pitr-minio-gap-error" "Failed" + local backup_error=$(kubectl_bin get pxc-restore on-pitr-minio-gap-error -ojsonpath='{.status.comments}' | grep -c "Backup doesn't guarantee consistent recovery with PITR. Annotate PerconaXtraDBClusterRestore with percona.com/unsafe-pitr to force it.") + if [[ $backup_error -eq 0 ]]; then + echo "ERROR: Backup is not tagged as PITR unready in the backup condition." + kubectl_bin get pxc-backup on-pitr-minio-gap -oyaml + exit 1 + fi + kubectl_bin delete -f "$test_dir/conf/restore-on-pitr-minio-gap-error.yaml" + elif [ "$type" == "force" ]; then + kubectl_bin apply -f "$test_dir/conf/restore-on-pitr-minio-gap-force.yaml" + wait_backup_restore "on-pitr-minio-gap-force" "Succeeded" + wait_for_running "$cluster-proxysql" 2 + wait_for_running "$cluster-pxc" 3 + wait_cluster_consistency "$cluster" 3 2 + kubectl_bin logs job/restore-job-on-pitr-minio-gap-force-${cluster} + compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password" + kubectl_bin delete -f "$test_dir/conf/restore-on-pitr-minio-gap-force.yaml" + else + echo "Wrong restore type!" + exit 1 + fi +} + +check_invalid_binlogs_error() { + local binlog=$1 + desc 'check invalid binlogs' + + # check error in pitr log + local err_text + err_text=$(kubectl_bin logs "$(get_pitr_pod)" | grep -c "ERROR: Binlog file $binlog is invalid") + if [[ $err_text -eq 0 ]]; then + echo "ERROR: Invalid binlog error text is not found in PITR pod logs." + exit 1 + fi + + sleep 65 # wait for next PITR collect cycle and error to appear +} + +find_invalid_binlog_name() { + local pod=$1 + local offset=$2 + + binlogs=() + while IFS= read -r line; do + binlogs+=("$line") + done < <(run_mysql_local "SHOW BINARY LOGS" "-h127.0.0.1 -P3306 -uroot -proot_password" "$pod" | awk '{print $1}') + + for ((i = 0; i < ${#binlogs[@]}; i++)); do + local events + events=$(run_mysql_local "SHOW BINLOG EVENTS IN '${binlogs[$i]}';" "-h127.0.0.1 -P3306 -uroot -proot_password" "$pod") + if echo "$events" | grep -q "CREATE TABLE IF NOT EXISTS invalid"; then + echo "${binlogs[$((i + offset))]}" + return + fi + done + + echo "No invalid binlog found" + exit 1 +} + +gtidset_to_gtid() { + local gtidset=$1 + local gtid + # Split the input string into two parts using ":" as the delimiter + IFS=':' read -ra parts <<<"$gtidset" + + # Get the number after the "-" symbol + number_part="${parts[1]#*-}" + + # Create the final transformed string + transformed_string="${parts[0]}:$number_part" + + echo "$transformed_string" +} + +invalid_binlog_test() { + desc 'start invalid binlog test' + + local proxy + proxy="$(get_proxy "$cluster")" + + kubectl patch pxc $cluster --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":false}}}}' + sleep 5 # wait for pitr pod to shutdown + + # restore to initial state + local backup + backup="on-pitr-minio" + yq "$test_dir/conf/restore-on-pitr-minio.yaml" \ + | yq eval 'del(.spec.pitr)' \ + | yq eval 'del(.spec.backupSource)' \ + | yq eval ".spec.backupName=\"$backup\"" \ + | kubectl_bin apply -f - + wait_backup_restore ${backup} + wait_for_running "$cluster-proxysql" 2 + wait_for_running "$cluster-pxc" 3 + wait_cluster_consistency "$cluster" 3 2 + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS test; use test; " \ + "-h $proxy -uroot -proot_password" + write_test_data "$cluster" + write_data_for_pitr "$cluster" + + # we need to use this function in test + run_mysql_local "CREATE FUNCTION get_binlog_by_gtid_set RETURNS STRING SONAME 'binlog_utils_udf.so';" \ + "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local "CREATE FUNCTION get_binlog_by_gtid_set RETURNS STRING SONAME 'binlog_utils_udf.so';" \ + "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local "CREATE FUNCTION get_binlog_by_gtid_set RETURNS STRING SONAME 'binlog_utils_udf.so';" \ + "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + # flush binlogs + run_mysql_local 'FLUSH BINARY LOGS;FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local 'FLUSH BINARY LOGS;FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local 'FLUSH BINARY LOGS;FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + + # this data should be in new binlog + run_mysql \ + "USE test; CREATE TABLE IF NOT EXISTS invalid (id int PRIMARY KEY);" \ + "-h $proxy -uroot -proot_password" + run_mysql \ + 'INSERT test.invalid (id) VALUES (100900); + INSERT test.invalid (id) VALUES (100901); + INSERT test.invalid (id) VALUES (100902);' \ + "-h $proxy -uroot -proot_password" + + # flush binlogs + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" + run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" + + write_more_data "$cluster" # this data should be in new binlog + + invalid_binlog=$(find_invalid_binlog_name "$cluster-pxc-0") + # get gtidset of invalid binlog + gtidset=$(run_mysql_local "SELECT get_gtid_set_by_binlog('$invalid_binlog');" "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0") + + next_binlog=$(find_invalid_binlog_name "$cluster-pxc-0" 1) + next_gtidset=$(run_mysql_local "SELECT get_gtid_set_by_binlog('$next_binlog');" "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0") + next_gtid=$(gtidset_to_gtid "$next_gtidset") + + # we should make binlogs with this gtidset empty + local binlog + binlog="$invalid_binlog" + kubectl exec $cluster-pxc-0 -- bash -c "echo \"\" > /var/lib/mysql/$binlog" + binlog=$(run_mysql_local "SELECT get_binlog_by_gtid_set('$gtidset');" "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1") + kubectl exec $cluster-pxc-1 -- bash -c "echo \"\" > /var/lib/mysql/$binlog" + binlog=$(run_mysql_local "SELECT get_binlog_by_gtid_set('$gtidset');" "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2") + kubectl exec $cluster-pxc-2 -- bash -c "echo \"\" > /var/lib/mysql/$binlog" + + sleep 20 + + kubectl patch pxc $cluster --type=merge -p '{"spec": {"backup": {"storages": {"minio-binlogs": {"s3": {"bucket": "operator-testing/binlogs-invalid"}}}}}}' + # re-enable pitr + kubectl patch pxc $cluster --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":true}}}}' + sleep 180 # wait for next PITR collect cycle and error to appear + check_invalid_binlogs_error "$invalid_binlog" + + gtid="$next_gtid" + + if [[ ! ${gtid} =~ ${GTID_PATTERN} ]]; then + printf "Some garbage --> %s <-- instead of legit GTID. Exiting ${gtid}" + exit 1 + fi + + run_recovery_check_pitr "$cluster" "restore-on-pitr-minio-invalid" "on-pitr-minio" "select-6" "" "" "$gtid" + + # invalid table should not exist + compare_mysql_cmd "select-invalid" "SELECT * from test.invalid;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd "select-invalid" "SELECT * from test.invalid;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password" + compare_mysql_cmd "select-invalid" "SELECT * from test.invalid;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password" + + desc 'done test invalid binlogs' +} + +main() { + create_infra $namespace + deploy_cert_manager + kubectl_bin apply -f "$test_dir/conf/issuer.yml" + kubectl_bin apply -f "$test_dir/conf/cert.yml" + sleep 25 + # We are using minio with tls enabled to check if `verifyTLS: false` works fine + start_minio "tls-minio" + + cluster="pitr-gap-errors" + spinup_pxc "$cluster" "$test_dir/conf/$cluster.yml" + + run_backup "$cluster" "on-pitr-minio" + + write_test_data "$cluster" + + desc 'show binlog events' + proxy=$(get_proxy "$cluster") + run_mysql "SHOW BINLOG EVENTS IN 'binlog.000005';" "-h ${proxy} -uroot -proot_password" + run_mysql "SHOW BINLOG EVENTS IN 'binlog.000006';" "-h ${proxy} -uroot -proot_password" + + write_data_for_pitr "$cluster" + sleep 120 # need to wait while collector catch new data + + desc 'check second backup/restore data from binlogs' + run_backup "$cluster" "on-pitr-minio1" + write_more_data "$cluster" + dest=$(sed 's,/,\\/,g' <<<$(kubectl get pxc-backup on-pitr-minio1 -o jsonpath='{.status.destination}')) + sleep 80 # need to wait while collector catch new data + run_recovery_check_pitr "$cluster" "restore-on-pitr-minio1" "on-pitr-minio1" "select-5" "" "$dest" "" + + desc 'binlog gap test' + desc 'create binlog gap backup (will be marked as PITR unready)' + run_mysql \ + "CREATE DATABASE IF NOT EXISTS test; use test; CREATE TABLE IF NOT EXISTS gap (id int PRIMARY KEY);" \ + "-h $proxy -uroot -proot_password" + run_backup "$cluster" "on-pitr-minio-gap" + create_binlog_gap + check_binlog_gap_error + check_binlog_gap_restore "error" + check_binlog_gap_restore "force" + desc "done binlog gap test" + + invalid_binlog_test + + destroy $namespace + desc "test passed" +} + +main diff --git a/e2e-tests/pitr/run b/e2e-tests/pitr/run index 8de8aaa610..093ca99e68 100755 --- a/e2e-tests/pitr/run +++ b/e2e-tests/pitr/run @@ -81,104 +81,6 @@ write_data_for_pitr() { "-h $proxy -uroot -proot_password" } -write_more_data() { - local cluster=$1 - local proxy=$(get_proxy "$cluster") - desc "write extra data" - run_mysql \ - 'INSERT test.test (id) VALUES (100506); INSERT test.test (id) VALUES (100507); INSERT test.test (id) VALUES (100508); INSERT test.test (id) VALUES (100509); INSERT test.test (id) VALUES (100510);' \ - "-h $proxy -uroot -proot_password" -} - -create_binlog_gap() { - desc 'create binlog gap' - - kubectl patch pxc pitr --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":false}}}}' - sleep 5 # wait for pitr pod to shutdown - # write data which will be lost - run_mysql \ - 'INSERT test.gap (id) VALUES (100800); INSERT test.gap (id) VALUES (100801); INSERT test.gap (id) VALUES (100802);' \ - "-h $proxy -uroot -proot_password" - # flush binlogs - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" - # purge binlogs - run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" - run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" - run_mysql_local 'PURGE BINARY LOGS BEFORE now();' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" - # re-enable pitr - kubectl patch pxc pitr --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":true}}}}' - # flush binlogs to trigger binlog collector - # data below will be force recovered - run_mysql \ - 'INSERT test.gap (id) VALUES (100803); INSERT test.gap (id) VALUES (100804); INSERT test.gap (id) VALUES (100805);' \ - "-h pitr-proxysql -uroot -proot_password" - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-0" - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-1" - run_mysql_local 'FLUSH BINARY LOGS;' "-h127.0.0.1 -P3306 -uroot -proot_password" "$cluster-pxc-2" - sleep 65 # wait for next PITR collect cycle and error to appear -} - -check_binlog_gap_error() { - desc 'check binlog gap error' - - # check error in pitr log - local err_text1=$(kubectl_bin logs $(get_pitr_pod) | grep -c "ERROR: Couldn't find the binlog that contains GTID set") - local err_text2=$(kubectl_bin logs $(get_pitr_pod) | grep -c "ERROR: Gap detected in the binary logs. Binary logs will be uploaded anyway, but full backup needed for consistent recovery.") - if [[ $err_text1 -eq 0 || $err_text2 -eq 0 ]]; then - echo "ERROR: Gap error text is not found in PITR pod logs." - exit 1 - fi - # check error in operator log - local err_text3=$(kubectl_bin logs ${OPERATOR_NS:+-n$OPERATOR_NS} $(get_operator_pod) | grep -c "Gap detected in binary logs") - if [[ $err_text3 -eq 0 ]]; then - echo "ERROR: Gap error text is not found in operator pod logs." - exit 1 - fi - # check backup on-pitr-minio-gap marked as unready for PITR restore - local backup_cond=$(kubectl_bin get pxc-backup on-pitr-minio-gap -ojsonpath='{.status.conditions[]}' | grep -c '"reason":"BinlogGapDetected","status":"False","type":"PITRReady"') - if [[ $backup_cond -eq 0 ]]; then - echo "ERROR: Backup is not tagged as PITR unready in the backup condition." - kubectl_bin get pxc-backup on-pitr-minio-gap -oyaml - exit 1 - fi -} - -check_binlog_gap_restore() { - local type=$1 - - desc 'check binlog gap restore: ' $type - # disable pitr - kubectl patch pxc pitr --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":false}}}}' - # try restore, check error - if [ "$type" == "error" ]; then - kubectl_bin apply -f $test_dir/conf/restore-on-pitr-minio-gap-error.yaml - wait_backup_restore "on-pitr-minio-gap-error" "Failed" - local backup_error=$(kubectl_bin get pxc-restore on-pitr-minio-gap-error -ojsonpath='{.status.comments}' | grep -c "Backup doesn't guarantee consistent recovery with PITR. Annotate PerconaXtraDBClusterRestore with percona.com/unsafe-pitr to force it.") - if [[ $backup_error -eq 0 ]]; then - echo "ERROR: Backup is not tagged as PITR unready in the backup condition." - kubectl_bin get pxc-backup on-pitr-minio-gap -oyaml - exit 1 - fi - kubectl_bin delete -f "$test_dir/conf/restore-on-pitr-minio-gap-error.yaml" - elif [ "$type" == "force" ]; then - kubectl_bin apply -f "$test_dir/conf/restore-on-pitr-minio-gap-force.yaml" - wait_backup_restore "on-pitr-minio-gap-force" "Succeeded" - wait_for_running "$cluster-proxysql" 2 - wait_for_running "$cluster-pxc" 3 - wait_cluster_consistency "$cluster" 3 2 - kubectl_bin logs job/restore-job-on-pitr-minio-gap-force-${cluster} - compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password" - compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password" - compare_mysql_cmd "select-gap" "SELECT * from test.gap;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password" - kubectl_bin delete -f "$test_dir/conf/restore-on-pitr-minio-gap-force.yaml" - else - echo "Wrong restore type!" - exit 1 - fi -} - main() { create_infra $namespace deploy_cert_manager @@ -240,27 +142,6 @@ main() { run_recovery_check_pitr "$cluster" "restore-on-pitr-minio" "on-pitr-minio" "select-4" "" "$dest" "" desc "done latest type" - desc 'check second backup/restore data from binlogs' - kubectl patch pxc pitr --type=merge -p '{"spec":{"backup":{"pitr":{"enabled":true}}}}' - sleep 20 - run_backup "$cluster" "on-pitr-minio1" - write_more_data "$cluster" - dest=$(sed 's,/,\\/,g' <<<$(kubectl get pxc-backup on-pitr-minio1 -o jsonpath='{.status.destination}')) - sleep 80 # need to wait while collector catch new data - run_recovery_check_pitr "$cluster" "restore-on-pitr-minio1" "on-pitr-minio1" "select-5" "" "$dest" "" - - desc 'binlog gap test' - desc 'create binlog gap backup (will be marked as PITR unready)' - run_mysql \ - "CREATE DATABASE IF NOT EXISTS test; use test; CREATE TABLE IF NOT EXISTS gap (id int PRIMARY KEY);" \ - "-h $proxy -uroot -proot_password" - run_backup "$cluster" "on-pitr-minio-gap" - create_binlog_gap - check_binlog_gap_error - check_binlog_gap_restore "error" - check_binlog_gap_restore "force" - desc "done binlog gap test" - destroy $namespace desc "test passed" } diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index 003ecb650a..39c3129607 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -6,6 +6,7 @@ init-deploy monitoring-2-0 one-pod pitr +pitr-gap-errors proxy-protocol recreate restore-to-encrypted-cluster diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 1253c4595e..fd90f6a65a 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -13,6 +13,7 @@ monitoring-2-0,8.0 one-pod,5.7 one-pod,8.0 pitr,8.0 +pitr-gap-errors,8.0 proxy-protocol,8.0 proxysql-sidecar-res-limits,8.0 recreate,8.0 diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 6343e3c75d..aaa1f5de83 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -13,6 +13,7 @@ monitoring-2-0 one-pod operator-self-healing-chaos pitr +pitr-gap-errors proxy-protocol proxysql-sidecar-res-limits recreate diff --git a/e2e-tests/scaling/run b/e2e-tests/scaling/run index 40cc7b934e..b1608a1af1 100755 --- a/e2e-tests/scaling/run +++ b/e2e-tests/scaling/run @@ -30,10 +30,10 @@ compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-4.$cl sleep 15 desc 'check new Pods exists in ProxySQL' -run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status="ONLINE";' "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ - | grep "$cluster-pxc-3" -run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status="ONLINE";' "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ - | grep "$cluster-pxc-4" +run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE status='ONLINE';" "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ + | grep "$cluster-pxc-3" +run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE status='ONLINE';" "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ + | grep "$cluster-pxc-4" desc 'scale down from 5 to 3' apply_config "$test_dir/conf/$cluster.yml" @@ -48,9 +48,9 @@ compare_kubectl pvc/datadir-$cluster-pxc-3 compare_kubectl pvc/datadir-$cluster-pxc-4 desc 'check if Pod deleted from ProxySQL' -run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status="ONLINE" AND hostgroup_id!=13;' "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ +run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE status='ONLINE' AND hostgroup_id!=13;" "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ | (grep "$cluster-pxc-3" && exit 1 || :) -run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status="ONLINE" AND hostgroup_id!=13;' "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ +run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE status='ONLINE' AND hostgroup_id!=13;" "-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0" \ | (grep "$cluster-pxc-4" && exit 1 || :) destroy $namespace