From 1127c6132aa79177ffb86b710f02dc5e128d846b Mon Sep 17 00:00:00 2001 From: Inel Pandzic Date: Thu, 21 Sep 2023 08:36:25 +0200 Subject: [PATCH] K8SPSMDB-903: Add bucket/container to destination (#1307) * Add s3 bucket to destination in status. * Update tests. * Add Azure container to destination. * Prefix .state.destination with protocol schema. * Fix incomplete destination. * Fix destination * Fix destination. * Fix tests. * Fix upgrade and upgrade-sharded tests. --- e2e-tests/data-at-rest-encryption/run | 4 +-- e2e-tests/demand-backup-eks-credentials/run | 2 +- e2e-tests/demand-backup-sharded/run | 26 +++++++++---------- .../conf/restore-backupsource.yml | 2 +- e2e-tests/demand-backup/run | 20 +++++++------- e2e-tests/functions | 6 ++--- e2e-tests/pitr/conf/restore.yml | 2 +- e2e-tests/pitr/run | 2 +- e2e-tests/scheduled-backup/run | 8 +++--- .../perconaservermongodbbackup/backup.go | 19 +++++++++++--- 10 files changed, 52 insertions(+), 39 deletions(-) diff --git a/e2e-tests/data-at-rest-encryption/run b/e2e-tests/data-at-rest-encryption/run index 37a0e4bb3b..ff2c08b65d 100755 --- a/e2e-tests/data-at-rest-encryption/run +++ b/e2e-tests/data-at-rest-encryption/run @@ -59,7 +59,7 @@ desc "check backup and restore -- minio" backup_dest_minio=$(get_backup_dest "$backup_name_minio") kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/${backup_dest_minio}/rs0/ \ + /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://${backup_dest_minio}/rs0/ \ | grep myApp.test.gz run_mongos 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-mongos.$namespace" compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "-2nd" @@ -70,7 +70,7 @@ compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- gcp-cs' backup_dest_gcp=$(get_backup_dest "$backup_name_gcp") - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null run_mongos 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-mongos.$namespace" compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "-2nd" run_restore "$backup_name_gcp" diff --git a/e2e-tests/demand-backup-eks-credentials/run b/e2e-tests/demand-backup-eks-credentials/run index cc791adb16..ce77dfc3a8 100755 --- a/e2e-tests/demand-backup-eks-credentials/run +++ b/e2e-tests/demand-backup-eks-credentials/run @@ -57,7 +57,7 @@ sleep 5 desc 'check backup and restore -- aws-s3' backup_dest_aws=$(get_backup_dest "$backup_name_aws") -curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null +curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace" compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd" compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd" diff --git a/e2e-tests/demand-backup-sharded/run b/e2e-tests/demand-backup-sharded/run index 45f4fe2ecb..9e5e5b433d 100755 --- a/e2e-tests/demand-backup-sharded/run +++ b/e2e-tests/demand-backup-sharded/run @@ -114,9 +114,9 @@ sleep 5 if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- aws-s3' backup_dest_aws=$(get_backup_dest "$backup_name_aws") - curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null - curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs1/myApp1.test.gz" | gunzip >/dev/null - curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs2/myApp2.test.gz" | gunzip >/dev/null + curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs1/myApp1.test.gz" | gunzip >/dev/null + curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs2/myApp2.test.gz" | gunzip >/dev/null insert_data "100501" check_data "-2nd" run_restore "$backup_name_aws" @@ -125,9 +125,9 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- gcp-cs' backup_dest_gcp=$(get_backup_dest "$backup_name_gcp") - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs1/myApp1.test.gz" | gunzip >/dev/null - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs2/myApp2.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs1/myApp1.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs2/myApp2.test.gz" | gunzip >/dev/null insert_data "100501" check_data "-2nd" run_restore "$backup_name_gcp" @@ -136,9 +136,9 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- azure-blob' backup_dest_azure=$(get_backup_dest "$backup_name_azure") - curl -s "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null - curl -s "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}/rs1/myApp1.test.gz" | gunzip >/dev/null - curl -s "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}/rs2/myApp2.test.gz" | gunzip >/dev/null + curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs1/myApp1.test.gz" | gunzip >/dev/null + curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs2/myApp2.test.gz" | gunzip >/dev/null insert_data "100501" check_data "-2nd" run_restore "$backup_name_azure" @@ -150,7 +150,7 @@ desc 'check backup and restore -- minio' backup_dest_minio=$(get_backup_dest "$backup_name_minio") kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls "s3://operator-testing/${backup_dest_minio}/rs0/" \ + /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls "s3://${backup_dest_minio}/rs0/" \ | grep "myApp.test.gz" insert_data "100501" check_data "-2nd" @@ -171,9 +171,9 @@ if [[ $backup_exists -eq 1 ]]; then fi if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then - check_backup_deletion "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}" "aws-s3" - check_backup_deletion "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}" "gcp-cs" - check_backup_deletion "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}" "azure-blob" + check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3" + check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}" "gcp-cs" + check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}" "azure-blob" fi desc 'check for passwords leak' diff --git a/e2e-tests/demand-backup/conf/restore-backupsource.yml b/e2e-tests/demand-backup/conf/restore-backupsource.yml index b3613428bb..9a0a58dedb 100644 --- a/e2e-tests/demand-backup/conf/restore-backupsource.yml +++ b/e2e-tests/demand-backup/conf/restore-backupsource.yml @@ -6,7 +6,7 @@ spec: clusterName: some-name storageName: backupSource: - destination: s3://operator-testing/BACKUP-NAME + destination: s3://BACKUP-NAME s3: credentialsSecret: minio-secret region: us-east-1 diff --git a/e2e-tests/demand-backup/run b/e2e-tests/demand-backup/run index 330ea6780d..1a065a5e46 100755 --- a/e2e-tests/demand-backup/run +++ b/e2e-tests/demand-backup/run @@ -98,17 +98,17 @@ sleep 5 if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- aws-s3' backup_dest_aws=$(get_backup_dest "$backup_name_aws") - curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null run_recovery_check "$backup_name_aws" "$cluster" desc 'check backup and restore -- gcp-cs' backup_dest_gcp=$(get_backup_dest "$backup_name_gcp") - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null run_recovery_check "$backup_name_gcp" "$cluster" desc 'check backup and restore -- azure-blob' backup_dest_azure=$(get_backup_dest "$backup_name_azure") - curl -s "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null run_recovery_check "$backup_name_azure" "$cluster" fi @@ -116,7 +116,7 @@ desc 'check backup and restore -- minio' backup_dest_minio=$(get_backup_dest "$backup_name_minio") kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/${backup_dest_minio}/rs0/ \ + /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://${backup_dest_minio}/rs0/ \ | grep myApp.test.gz run_recovery_check "$backup_name_minio" "$cluster" @@ -140,9 +140,9 @@ if [[ $backup_exists -eq 1 ]]; then fi if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then - check_backup_deletion "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}" "aws-s3" - check_backup_deletion "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}" "gcp-cs" - check_backup_deletion "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}" "azure-blob" + check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3" + check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}" "gcp-cs" + check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}" "azure-blob" fi desc 'checking backup deletion without cr' @@ -175,9 +175,9 @@ if [[ $backup_exists -eq 1 ]]; then fi if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then - check_backup_deletion "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}" "aws-s3" - check_backup_deletion "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}" "gcp-cs" - check_backup_deletion "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}" "azure-blob" + check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3" + check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}" "gcp-cs" + check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}" "azure-blob" fi desc 'check for passwords leak' diff --git a/e2e-tests/functions b/e2e-tests/functions index 4e9a7ee514..baf53273f0 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -257,7 +257,7 @@ run_restore_backupsource() { if [ -z "$storageName" ]; then cat $test_dir/conf/restore-backupsource.yml \ | $sed -e "s/name:/name: restore-$backupName/" \ - | $sed -e "s/BACKUP-NAME/$backupDest/" \ + | $sed -e "s|BACKUP-NAME|$backupDest|" \ | $sed -e "/storageName/d" \ | kubectl_bin apply -f - @@ -266,7 +266,7 @@ run_restore_backupsource() { cat $test_dir/conf/restore-backupsource.yml \ | $sed -e "s/name:/name: restore-$backupName/" \ - | $sed -e "s/BACKUP-NAME/$backupDest/" \ + | $sed -e "s|BACKUP-NAME|$backupDest|" \ | $sed -e "s/storageName:/storageName: $storageName/" \ | kubectl_bin apply -f - } @@ -935,7 +935,7 @@ get_backup_dest() { local backup_name=$1 kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.destination}' \ - | sed -e 's/.json$//' + | sed -e 's/.json$//' | sed "s|s3://||" | sed "s|azure://||" } get_service_endpoint() { diff --git a/e2e-tests/pitr/conf/restore.yml b/e2e-tests/pitr/conf/restore.yml index 0066c3a5f4..ab0435c570 100644 --- a/e2e-tests/pitr/conf/restore.yml +++ b/e2e-tests/pitr/conf/restore.yml @@ -10,4 +10,4 @@ spec: type: date: backupSource: - destination: s3://operator-testing/BACKUP-NAME + destination: s3://BACKUP-NAME diff --git a/e2e-tests/pitr/run b/e2e-tests/pitr/run index ec04fbe61e..d298919e10 100755 --- a/e2e-tests/pitr/run +++ b/e2e-tests/pitr/run @@ -81,7 +81,7 @@ check_recovery() { | $sed -e "/backupName/d" \ | $sed -e "s/type:/type: $restore_type/" \ | if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $restore_date/"; fi \ - | $sed -e "s/BACKUP-NAME/$backup_dest/" \ + | $sed -e "s|BACKUP-NAME|$backup_dest|" \ | kubectl_bin apply -f - fi wait_restore "$backup_name" "$cluster_name" diff --git a/e2e-tests/scheduled-backup/run b/e2e-tests/scheduled-backup/run index 32595c9957..eee4225707 100755 --- a/e2e-tests/scheduled-backup/run +++ b/e2e-tests/scheduled-backup/run @@ -92,7 +92,7 @@ desc 'check backup and restore -- minio' backup_dest_minio=$(get_backup_dest "$backup_name_minio") kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/${backup_dest_minio}/rs0/ \ + /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://${backup_dest_minio}/rs0/ \ | grep "myApp.test.gz" run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace" compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd" @@ -107,7 +107,7 @@ compare_mongo_cmd "find" "myApp:myPass@$cluster-2.$cluster.$namespace" if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- aws-s3' backup_dest_aws=$(get_backup_dest "$backup_name_aws") - curl -s "https://s3.amazonaws.com/operator-testing/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace" compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd" compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd" @@ -120,7 +120,7 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- gcp-cs' backup_dest_gcp=$(get_backup_dest "$backup_name_gcp") - curl -s "https://storage.googleapis.com/operator-testing/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace" compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd" compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd" @@ -133,7 +133,7 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then desc 'check backup and restore -- azure-blob' backup_dest_azure=$(get_backup_dest "$backup_name_azure") - curl -s "https://engk8soperators.blob.core.windows.net/operator-testing/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null + curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace" compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd" compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd" diff --git a/pkg/controller/perconaservermongodbbackup/backup.go b/pkg/controller/perconaservermongodbbackup/backup.go index 1ece92dd61..b70b064dc3 100644 --- a/pkg/controller/perconaservermongodbbackup/backup.go +++ b/pkg/controller/perconaservermongodbbackup/backup.go @@ -3,6 +3,7 @@ package perconaservermongodbbackup import ( "context" "fmt" + "strings" "time" "github.com/percona/percona-backup-mongodb/pbm" @@ -90,16 +91,28 @@ func (b *Backup) Start(ctx context.Context, k8sclient client.Client, cluster *ap switch stg.Type { case api.BackupStorageS3: status.S3 = &stg.S3 + + status.Destination = stg.S3.Bucket + if stg.S3.Prefix != "" { - status.Destination = stg.S3.Prefix + "/" + status.Destination = stg.S3.Bucket + "/" + stg.S3.Prefix + } + if !strings.HasPrefix(stg.S3.Bucket, "s3://") { + status.Destination = "s3://" + status.Destination } case api.BackupStorageAzure: status.Azure = &stg.Azure + + status.Destination = stg.Azure.Container + if stg.Azure.Prefix != "" { - status.Destination = stg.Azure.Prefix + "/" + status.Destination = stg.Azure.Container + "/" + stg.Azure.Prefix + } + if !strings.HasPrefix(stg.Azure.Container, "azure://") { + status.Destination = "azure://" + status.Destination } } - status.Destination += status.PBMname + status.Destination += "/" + status.PBMname return status, nil }