K8SPXC-1106: PITR not uploading files if corrupted binlog exists (per…

…cona#1423) * K8SPXC-1106: PITR not uploading files if corrupted binlog exists https://jira.percona.com/browse/K8SPXC-1106 * add test * improve test * optimize `pitr` test * fix tests * split `pitr` test * fix pitr2 test * rename test from pitr2 to pitr-gap-errors * Update e2e-tests/scaling/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/scaling/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-gap-errors/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * remove BACKUP_IMAGE --------- Co-authored-by: Viacheslav Sarzhan <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
dcaputo-harmoni · Jun 13, 2023 · 338f4cf · 338f4cf
1 parent 545b2e0
commit 338f4cf
Show file tree

Hide file tree

Showing 26 changed files with 585 additions and 191 deletions.
diff --git a/cmd/pitr/collector/collector.go b/cmd/pitr/collector/collector.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/go-sql-driver/mysql"
 	"github.com/minio/minio-go/v7"
 	"github.com/pkg/errors"
 
@@ -163,31 +164,10 @@ func (c *Collector) close() error {
 	return c.db.Close()
 }
 
-func (c *Collector) CurrentSourceID(ctx context.Context, logs []pxc.Binlog) (string, error) {
-	var (
-		gtidSet string
-		err     error
-	)
-	for i := len(logs) - 1; i >= 0 && gtidSet == ""; i-- {
-		gtidSet, err = c.db.GetGTIDSet(ctx, logs[i].Name)
-		if err != nil {
-			return gtidSet, err
-		}
-	}
-	return strings.Split(gtidSet, ":")[0], nil
-}
-
 func (c *Collector) removeEmptyBinlogs(ctx context.Context, logs []pxc.Binlog) ([]pxc.Binlog, error) {
 	result := make([]pxc.Binlog, 0)
 	for _, v := range logs {
-		set, err := c.db.GetGTIDSet(ctx, v.Name)
-		if err != nil {
-			return nil, errors.Wrap(err, "get GTID set")
-		}
-		// we don't upload binlog without gtid
-		// because it is empty and doesn't have any information
-		if set != "" {
-			v.GTIDSet = set
+		if v.GTIDSet != "" {
 			result = append(result, v)
 		}
 	}
@@ -238,15 +218,33 @@ func createGapFile(gtidSet string) error {
 	return nil
 }
 
+func (c *Collector) addGTIDSets(ctx context.Context, logs []pxc.Binlog) error {
+	for i, v := range logs {
+		set, err := c.db.GetGTIDSet(ctx, v.Name)
+		if err != nil {
+			if errors.Is(err, &mysql.MySQLError{Number: 3200}) {
+				log.Printf("ERROR: Binlog file %s is invalid on host %s: %s\n", v.Name, c.db.GetHost(), err.Error())
+				continue
+			}
+			return errors.Wrap(err, "get GTID set")
+		}
+		logs[i].GTIDSet = set
+	}
+	return nil
+}
+
 func (c *Collector) CollectBinLogs(ctx context.Context) error {
 	list, err := c.db.GetBinLogList(ctx)
 	if err != nil {
 		return errors.Wrap(err, "get binlog list")
 	}
-
-	sourceID, err := c.CurrentSourceID(ctx, list)
+	err = c.addGTIDSets(ctx, list)
 	if err != nil {
-		return errors.Wrap(err, "get current source id")
+		return errors.Wrap(err, "get GTID sets")
+	}
+	var sourceID string
+	for i := len(list) - 1; i >= 0 && sourceID == ""; i-- {
+		sourceID = strings.Split(list[i].GTIDSet, ":")[0]
 	}
 
 	if sourceID == "" {
@@ -262,10 +260,11 @@ func (c *Collector) CollectBinLogs(ctx context.Context) error {
 	lastUploadedBinlogName := ""
 
 	if c.lastSet != "" {
-		// get last uploaded binlog file name
-		lastUploadedBinlogName, err = c.db.GetBinLogName(ctx, c.lastSet)
-		if err != nil {
-			return errors.Wrap(err, "get last uploaded binlog name by gtid set")
+		for i := len(list) - 1; i >= 0; i-- {
+			if list[i].GTIDSet == c.lastSet {
+				lastUploadedBinlogName = list[i].Name
+				break
+			}
 		}
 
 		if lastUploadedBinlogName == "" {

diff --git a/cmd/pitr/pxc/pxc.go b/cmd/pitr/pxc/pxc.go
@@ -129,34 +129,6 @@ func (p *PXC) GetBinLogNamesList(ctx context.Context) ([]string, error) {
 	return binlogs, nil
 }
 
-// GetBinLogName returns name of the binary log file by given GTID set
-func (p *PXC) GetBinLogName(ctx context.Context, gtidSet string) (string, error) {
-	if len(gtidSet) == 0 {
-		return "", nil
-	}
-	var existFunc string
-	nameRow := p.db.QueryRowContext(ctx, "select name from mysql.func where name='get_binlog_by_gtid_set'")
-	err := nameRow.Scan(&existFunc)
-	if err != nil && err != sql.ErrNoRows {
-		return "", errors.Wrap(err, "get udf name")
-	}
-	if len(existFunc) == 0 {
-		_, err = p.db.ExecContext(ctx, "CREATE FUNCTION get_binlog_by_gtid_set RETURNS STRING SONAME 'binlog_utils_udf.so'")
-		if err != nil {
-			return "", errors.Wrap(err, "create function")
-		}
-	}
-	var binlog sql.NullString
-	row := p.db.QueryRowContext(ctx, "SELECT get_binlog_by_gtid_set(?)", gtidSet)
-
-	err = row.Scan(&binlog)
-	if err != nil {
-		return "", errors.Wrap(err, "scan binlog")
-	}
-
-	return strings.TrimPrefix(binlog.String, "./"), nil
-}
-
 // GetBinLogFirstTimestamp return binary log file first timestamp
 func (p *PXC) GetBinLogFirstTimestamp(ctx context.Context, binlog string) (string, error) {
 	var existFunc string

diff --git a/e2e-tests/functions b/e2e-tests/functions
@@ -17,7 +17,7 @@ SKIP_REMOTE_BACKUPS=${SKIP_REMOTE_BACKUPS:-1}
 PMM_SERVER_VER=${PMM_SERVER_VER:-"9.9.9"}
 IMAGE_PMM_SERVER_REPO=${IMAGE_PMM_SERVER_REPO:-"perconalab/pmm-server"}
 IMAGE_PMM_SERVER_TAG=${IMAGE_PMM_SERVER_TAG:-"dev-latest"}
-CERT_MANAGER_VER="1.8.0"
+CERT_MANAGER_VER="1.12.1"
 tmp_dir=$(mktemp -d)
 sed=$(which gsed || which sed)
 date=$(which gdate || which date)
@@ -509,7 +509,7 @@ run_mysql_local() {
 	local container_name="$4"
 	set +o xtrace
 	kubectl_bin exec $pod ${container_name:+-c $container_name} -- \
-		bash -c "printf '$command\n' | mysql -sN $uri" 2>&1 \
+		bash -c "printf \"$command\n\" | mysql -sN $uri" 2>&1 \
 		| sed -e 's/mysql: //' \
 		| (egrep -v 'Using a password on the command line interface can be insecure.|Defaulted container|Defaulting container name|see all of the containers in this pod' || :)
 	set_debug
@@ -562,11 +562,11 @@ compare_mysql_cmd_local() {
 get_proxy_primary() {
 	local uri="$1"
 	local pod="$2"
-	local ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql')
+	local ip=$(run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='ONLINE';" "$uri" "$pod" 'proxysql')
 
 	while [ $(echo "$ip" | wc -l) != 1 ]; do
 		sleep 1
-		ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql')
+		ip=$(run_mysql_local "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='ONLINE';" "$uri" "$pod" 'proxysql')
 	done
 
 	echo $ip | cut -d'.' -f1

diff --git a/e2e-tests/pitr-gap-errors/compare/select-1.sql b/e2e-tests/pitr-gap-errors/compare/select-1.sql
@@ -0,0 +1 @@
+100500
diff --git a/e2e-tests/pitr-gap-errors/compare/select-3.sql b/e2e-tests/pitr-gap-errors/compare/select-3.sql
@@ -0,0 +1,3 @@
+100500
+100501
+100502
diff --git a/e2e-tests/pitr/compare/select-5.sql → ...ests/pitr-gap-errors/compare/select-5.sql b/e2e-tests/pitr/compare/select-5.sql → ...ests/pitr-gap-errors/compare/select-5.sql
diff --git a/e2e-tests/pitr-gap-errors/compare/select-6.sql b/e2e-tests/pitr-gap-errors/compare/select-6.sql
@@ -0,0 +1,10 @@
+100500
+100501
+100502
+100503
+100504
+100505
+100506
+100507
+100508
+100509
diff --git a/e2e-tests/pitr/compare/select-gap.sql → ...ts/pitr-gap-errors/compare/select-gap.sql b/e2e-tests/pitr/compare/select-gap.sql → ...ts/pitr-gap-errors/compare/select-gap.sql
diff --git a/e2e-tests/pitr-gap-errors/compare/select-invalid.sql b/e2e-tests/pitr-gap-errors/compare/select-invalid.sql
@@ -0,0 +1,2 @@
+ERROR 1146 (42S02) at line 1: Table 'test.invalid' doesn't exist
+command terminated with exit code 1
diff --git a/e2e-tests/pitr-gap-errors/conf/cert.yml b/e2e-tests/pitr-gap-errors/conf/cert.yml
@@ -0,0 +1,9 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: tls-minio
+spec:
+  commonName: minio-service
+  secretName: tls-minio
+  issuerRef:
+    name: selfsigning-issuer
diff --git a/e2e-tests/pitr-gap-errors/conf/issuer.yml b/e2e-tests/pitr-gap-errors/conf/issuer.yml
@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigning-issuer
+spec:
+  selfSigned: {}
diff --git a/e2e-tests/pitr/conf/on-pitr-minio-gap.yml → ...itr-gap-errors/conf/on-pitr-minio-gap.yml b/e2e-tests/pitr/conf/on-pitr-minio-gap.yml → ...itr-gap-errors/conf/on-pitr-minio-gap.yml
@@ -3,5 +3,5 @@ kind: PerconaXtraDBClusterBackup
 metadata:
   name: on-pitr-minio-gap
 spec:
-  pxcCluster: pitr
+  pxcCluster: pitr-gap-errors
   storageName: minio
diff --git a/e2e-tests/pitr-gap-errors/conf/on-pitr-minio.yml b/e2e-tests/pitr-gap-errors/conf/on-pitr-minio.yml
@@ -0,0 +1,7 @@
+apiVersion: pxc.percona.com/v1
+kind: PerconaXtraDBClusterBackup
+metadata:
+  name: on-pitr-minio
+spec:
+  pxcCluster: pitr-gap-errors
+  storageName: minio
diff --git a/e2e-tests/pitr/conf/on-pitr-minio1.yml → ...s/pitr-gap-errors/conf/on-pitr-minio1.yml b/e2e-tests/pitr/conf/on-pitr-minio1.yml → ...s/pitr-gap-errors/conf/on-pitr-minio1.yml
@@ -3,5 +3,5 @@ kind: PerconaXtraDBClusterBackup
 metadata:
   name: on-pitr-minio1
 spec:
-  pxcCluster: pitr
+  pxcCluster: pitr-gap-errors
   storageName: minio
diff --git a/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml b/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
@@ -0,0 +1,81 @@
+apiVersion: pxc.percona.com/v1-6-0
+kind: PerconaXtraDBCluster
+metadata:
+  name: pitr-gap-errors
+  finalizers:
+    - delete-pxc-pods-in-order
+  # annotations:
+  #   percona.com/issue-vault-token: "true"
+spec:
+  secretsName: my-cluster-secrets
+  vaultSecretName: some-name-vault
+  pause: false
+  pxc:
+    size: 3
+    image: -pxc
+    configuration: |
+      [sst]
+      xbstream-opts=--decompress
+      [xtrabackup]
+      compress=lz4
+    resources:
+      requests:
+        memory: 0.1G
+        cpu: 100m
+      limits:
+        memory: "1G"
+        cpu: "1"
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 2Gi
+    affinity:
+      antiAffinityTopologyKey: "kubernetes.io/hostname"
+  proxysql:
+    enabled: true
+    size: 2
+    image: -proxysql
+    resources:
+      requests:
+        memory: 0.1G
+        cpu: 100m
+      limits:
+        memory: 1G
+        cpu: 700m
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 2Gi
+    affinity:
+      antiAffinityTopologyKey: "kubernetes.io/hostname"
+  pmm:
+    enabled: false
+    image: perconalab/pmm-client:1.17.1
+    serverHost: monitoring-service
+    serverUser: pmm
+  backup:
+    image: -backup
+    imagePullPolicy: Always
+    pitr:
+      enabled: true
+      storageName: minio-binlogs
+      timeBetweenUploads: 55
+    storages:
+      minio:
+        type: s3
+        verifyTLS: false
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          endpointUrl: https://minio-service:9000
+      minio-binlogs:
+        type: s3
+        verifyTLS: false
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing/binlogs
+          endpointUrl: https://minio-service:9000
diff --git a/...conf/restore-on-pitr-minio-gap-error.yaml → ...conf/restore-on-pitr-minio-gap-error.yaml b/...conf/restore-on-pitr-minio-gap-error.yaml → ...conf/restore-on-pitr-minio-gap-error.yaml
@@ -3,7 +3,7 @@ kind: PerconaXtraDBClusterRestore
 metadata:
   name: on-pitr-minio-gap-error
 spec:
-  pxcCluster: pitr
+  pxcCluster: pitr-gap-errors
   backupName: on-pitr-minio-gap
   pitr:
     type: latest

diff --git a/...conf/restore-on-pitr-minio-gap-force.yaml → ...conf/restore-on-pitr-minio-gap-force.yaml b/...conf/restore-on-pitr-minio-gap-force.yaml → ...conf/restore-on-pitr-minio-gap-force.yaml
@@ -5,7 +5,7 @@ metadata:
     percona.com/unsafe-pitr: "true"
   name: on-pitr-minio-gap-force
 spec:
-  pxcCluster: pitr
+  pxcCluster: pitr-gap-errors
   backupName: on-pitr-minio-gap
   pitr:
     type: latest

diff --git a/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-invalid.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio-invalid.yaml
@@ -0,0 +1,14 @@
+apiVersion: pxc.percona.com/v1
+kind: PerconaXtraDBClusterRestore
+metadata:
+  annotations:
+    percona.com/unsafe-pitr: "true"
+  name: on-pitr-minio-invalid
+spec:
+  pxcCluster: pitr-gap-errors
+  backupName: on-pitr-minio 
+  pitr:
+    type: transaction
+    gtid: <gtid>
+    backupSource:
+      storageName: "minio-binlogs"
diff --git a/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio.yaml b/e2e-tests/pitr-gap-errors/conf/restore-on-pitr-minio.yaml
@@ -0,0 +1,24 @@
+apiVersion: pxc.percona.com/v1
+kind: PerconaXtraDBClusterRestore
+metadata:
+  name: on-pitr-minio
+spec:
+  pxcCluster: pitr-gap-errors
+  backupSource:
+    verifyTLS: false
+    destination: <destination>
+    s3:
+      bucket: operator-testing
+      credentialsSecret: minio-secret
+      endpointUrl: https://minio-service:9000
+      region: us-east-1
+  pitr:
+    type: latest
+    backupSource:
+      verifyTLS: false
+      s3:
+        bucket: operator-testing/binlogs
+        credentialsSecret: minio-secret
+        endpointUrl: https://minio-service:9000
+        region: us-east-1
+
diff --git a/...sts/pitr/conf/restore-on-pitr-minio1.yaml → ...p-errors/conf/restore-on-pitr-minio1.yaml b/...sts/pitr/conf/restore-on-pitr-minio1.yaml → ...p-errors/conf/restore-on-pitr-minio1.yaml
@@ -3,7 +3,7 @@ kind: PerconaXtraDBClusterRestore
 metadata:
   name: on-pitr-minio1
 spec:
-  pxcCluster: pitr
+  pxcCluster: pitr-gap-errors
   backupSource:
     destination: <destination>
     verifyTLS: false
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,10 @@
		100500
		100501
		100502
		100503
		100504
		100505
		100506
		100507
		100508
		100509
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		ERROR 1146 (42S02) at line 1: Table 'test.invalid' doesn't exist
		command terminated with exit code 1