From 8866d803a97b2d8c3a3456be6e1f59480ce1afcc Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 5 Aug 2022 15:33:12 -0400 Subject: [PATCH] Add data mover to e2e Signed-off-by: Tiger Kaovilai --- Makefile | 34 +++++++++++-- go.mod | 3 +- tests/e2e/backup_restore_suite_test.go | 67 +++++++++++++++++++++----- tests/e2e/lib/apps.go | 22 ++++++++- tests/e2e/lib/dpa_helpers.go | 44 +++++++++++++---- tests/e2e/volsync/volsync-sub.yaml | 16 ++++++ 6 files changed, 160 insertions(+), 26 deletions(-) create mode 100644 tests/e2e/volsync/volsync-sub.yaml diff --git a/Makefile b/Makefile index 7b896d37248..94cfb14e75c 100644 --- a/Makefile +++ b/Makefile @@ -330,7 +330,7 @@ deploy-olm: IMG=$(THIS_OPERATOR_IMAGE) BUNDLE_IMG=$(THIS_BUNDLE_IMAGE) \ make docker-build docker-push bundle bundle-build bundle-push; \ rm -rf $(DEPLOY_TMP) - operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE) + operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE) --index-image=quay.io/operator-framework/opm:v1.23.0 .PHONY: opm OPM = ./bin/opm @@ -384,6 +384,7 @@ sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//")) || $(CLUSTER_TY #TEST_FILTER := $(shell echo '! aws && ! gcp && ! azure' | sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//") SETTINGS_TMP=/tmp/test-settings +.PHONY: test-e2e-setup test-e2e-setup: mkdir -p $(SETTINGS_TMP) TARGET_CI_CRED_FILE="$(CI_CRED_FILE)" AZURE_RESOURCE_FILE="$(AZURE_RESOURCE_FILE)" CI_JSON_CRED_FILE="$(AZURE_CI_JSON_CRED_FILE)" \ @@ -391,7 +392,8 @@ test-e2e-setup: PROVIDER="$(VELERO_PLUGIN)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \ VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh" -test-e2e: test-e2e-setup +.PHONY: test-e2e-ginkgo +test-e2e-ginkgo: test-e2e-setup ginkgo run -mod=mod tests/e2e/ -- -credentials=$(OADP_CRED_FILE) \ -velero_namespace=$(OADP_TEST_NAMESPACE) \ -settings=$(SETTINGS_TMP)/oadpcreds \ @@ -404,5 +406,31 @@ test-e2e: test-e2e-setup -artifact_dir=$(ARTIFACT_DIR) \ -oc_cli=$(OC_CLI) -test-e2e-cleanup: +.PHONY: test-e2e +test-e2e: volsync-install test-e2e-ginkgo + +.PHONY: test-e2e-cleanup +test-e2e-cleanup: volsync-uninstall rm -rf $(SETTINGS_TMP) + +.PHONY: volsync-install +volsync-install: + $(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}')) + # OperatorGroup not required, volsync is global operator which has operatorgroup already. + # Create subscription for operator if not installed. + @if [ "$(VS_CURRENT_CSV)" == "" ]; then \ + $(OC_CLI) replace --force -f tests/e2e/volsync/volsync-sub.yaml; \ + else \ + echo $(VS_CURRENT_CSV) already installed; \ + fi + +.PHONY: volsync-uninstall +volsync-uninstall: + $(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}')) + @if [ "$(VS_CURRENT_CSV)" != "" ]; then \ + echo "Uninstalling $(VS_CURRENT_CSV)"; \ + $(OC_CLI) delete subscription volsync-product -n openshift-operators && \ + $(OC_CLI) delete csv $(VS_CURRENT_CSV) -n openshift-operators; \ + else \ + echo No subscription found, skipping uninstall; \ + fi diff --git a/go.mod b/go.mod index 12d092cdaec..47b322add21 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,8 @@ require ( sigs.k8s.io/controller-runtime v0.11.0 ) +require github.com/google/go-cmp v0.5.6 + require ( cloud.google.com/go v0.93.3 // indirect github.com/Azure/azure-sdk-for-go v61.4.0+incompatible // indirect @@ -52,7 +54,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/go-hclog v0.14.1 // indirect diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index ab15b745327..b2c30248a29 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" + "github.com/openshift/oadp-operator/controllers" + "github.com/openshift/oadp-operator/pkg/common" . "github.com/openshift/oadp-operator/tests/e2e/lib" corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" @@ -20,6 +22,19 @@ import ( type VerificationFunction func(client.Client, string) error +type appVerificationFunction func(bool, BackupRestoreType) VerificationFunction + +func dataMoverReady(preBackupState bool, appVerificationFunction appVerificationFunction) VerificationFunction { + return VerificationFunction(func(ocClient client.Client, namespace string) error { + // check volsync subscription exists + Eventually(InstalledSubscriptionCSV(ocClient, "openshift-operators", "volsync-product"), timeoutMultiplier*time.Minute*10, time.Second*10).ShouldNot(Equal("")) + // check volsync controller is ready + fmt.Printf("waiting for volsync controller readiness") + Eventually(IsDeploymentReady(ocClient, common.VolSyncDeploymentNamespace, common.VolSyncDeploymentName), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + return appVerificationFunction(preBackupState, CSIDataMover)(ocClient, namespace) + }) +} + func mongoready(preBackupState bool, backupRestoreType BackupRestoreType) VerificationFunction { return VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(IsDCReady(ocClient, namespace, "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) @@ -79,15 +94,17 @@ var _ = Describe("AWS backup restore tests", func() { GinkgoWriter.Println("End of velero deployment pod logs") } // remove app namespace if leftover (likely previously failed before reaching uninstall applications) to clear items such as PVCs which are immutable so that next test can create new ones - err := dpaCR.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: v1.ObjectMeta{ - Name: lastInstallingApplicationNamespace, - Namespace: lastInstallingApplicationNamespace, - }}, &client.DeleteOptions{}) - if k8serror.IsNotFound(err) { - err = nil + if lastInstallingApplicationNamespace != dpaCR.Namespace { + err := dpaCR.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: v1.ObjectMeta{ + Name: lastInstallingApplicationNamespace, + Namespace: lastInstallingApplicationNamespace, + }}, &client.DeleteOptions{}) + if k8serror.IsNotFound(err) { + err = nil + } + Expect(err).ToNot(HaveOccurred()) } - Expect(err).ToNot(HaveOccurred()) - err = dpaCR.Delete() + err := dpaCR.Delete() Expect(err).ToNot(HaveOccurred()) }) @@ -139,14 +156,24 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for restic pods to be running") Eventually(AreResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) } - if brCase.BackupRestoreType == CSI { + if brCase.BackupRestoreType == CSI || brCase.BackupRestoreType == CSIDataMover { if provider == "aws" || provider == "ibmcloud" || provider == "gcp" || provider == "azure" { log.Printf("Creating VolumeSnapshotClass for CSI backuprestore of %s", brCase.Name) snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider) err = InstallApplication(dpaCR.Client, snapshotClassPath) Expect(err).ToNot(HaveOccurred()) } - + if brCase.BackupRestoreType == CSIDataMover { + dpaCR.Client.Create(context.Background(), &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: controllers.ResticsecretName, + Namespace: dpaCR.Namespace, + }, + StringData: map[string]string{ + controllers.ResticPassword: "e2e-restic-password", + }, + }, &client.CreateOptions{}) + } } // TODO: check registry deployments are deleted @@ -256,7 +283,7 @@ var _ = Describe("AWS backup restore tests", func() { }, Entry("MySQL application CSI", Label("ibmcloud", "aws", "gcp", "azure"), BackupRestoreCase{ - ApplicationTemplate: fmt.Sprintf("./sample-applications/mysql-persistent/mysql-persistent-csi.yaml"), + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml", ApplicationNamespace: "mysql-persistent", Name: "mysql-csi-e2e", BackupRestoreType: CSI, @@ -264,7 +291,7 @@ var _ = Describe("AWS backup restore tests", func() { PostRestoreVerify: mysqlReady(false, CSI), }, nil), Entry("Mongo application CSI", Label("ibmcloud", "aws", "gcp", "azure"), BackupRestoreCase{ - ApplicationTemplate: fmt.Sprintf("./sample-applications/mongo-persistent/mongo-persistent-csi.yaml"), + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml", ApplicationNamespace: "mongo-persistent", Name: "mongo-csi-e2e", BackupRestoreType: CSI, @@ -287,5 +314,21 @@ var _ = Describe("AWS backup restore tests", func() { PreBackupVerify: mysqlReady(true, RESTIC), PostRestoreVerify: mysqlReady(false, RESTIC), }, nil), + Entry("Mongo application DATAMOVER", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + ApplicationNamespace: "mongo-persistent", + Name: "mongo-restic-e2e", + BackupRestoreType: CSIDataMover, + PreBackupVerify: dataMoverReady(true, mongoready), + PostRestoreVerify: dataMoverReady(false, mongoready), + }, nil), + Entry("MySQL application DATAMOVER", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + ApplicationNamespace: "mysql-persistent", + Name: "mysql-restic-e2e", + BackupRestoreType: CSIDataMover, + PreBackupVerify: dataMoverReady(true, mysqlReady), + PostRestoreVerify: dataMoverReady(false, mysqlReady), + }, nil), ) }) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 27620845c26..ee578407366 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -24,6 +24,7 @@ import ( routev1 "github.com/openshift/api/route/v1" security "github.com/openshift/api/security/v1" templatev1 "github.com/openshift/api/template/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" "github.com/vmware-tanzu/velero/pkg/label" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -347,6 +348,25 @@ func AreApplicationPodsRunning(namespace string) wait.ConditionFunc { } } +func InstalledSubscriptionCSV(ocClient client.Client, namespace, subscriptionName string) func() (string, error) { + return func() (string, error) { + // get operator-sdk subscription + subscription := &operatorsv1alpha1.Subscription{} + err := ocClient.Get(context.Background(), client.ObjectKey{ + Namespace: namespace, + Name: subscriptionName, + }, subscription) + if err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Error getting subscription: %v\n", err))) + return "", err + } + return subscription.Status.InstalledCSV, nil + } +} + func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) { log.Println("Printing events for namespace: ", namespace) clientset, err := setUpClient() @@ -454,7 +474,7 @@ func VerifyBackupRestoreData(artifact_dir string, namespace string, routeName st return err } //Verifying backup-restore data only for CSI as of now. - if backupRestoretype == CSI || backupRestoretype == RESTIC { + if backupRestoretype == CSI || backupRestoretype == CSIDataMover || backupRestoretype == RESTIC { //check if backupfile exists. If true { compare data response with data from file} (post restore step) //else write data to backup-data.txt (prebackup step) if _, err := os.Stat(backupFile); err == nil { diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index a3ac939e5e8..ebba6ebfeb1 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -24,10 +24,12 @@ import ( oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" utils "github.com/openshift/oadp-operator/tests/e2e/utils" operators "github.com/operator-framework/api/pkg/operators/v1alpha1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/pointer" @@ -39,6 +41,7 @@ type BackupRestoreType string const ( CSI BackupRestoreType = "csi" + CSIDataMover BackupRestoreType = "csi-datamover" RESTIC BackupRestoreType = "restic" ) @@ -57,6 +60,10 @@ var Dpa *oadpv1alpha1.DataProtectionApplication func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. dpaInstance := oadpv1alpha1.DataProtectionApplication{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataProtectionApplication", + APIVersion: "oadp.openshift.io/v1alpha1", + }, ObjectMeta: metav1.ObjectMeta{ Name: v.Name, Namespace: v.Namespace, @@ -102,26 +109,32 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { for _, plugin := range dpaInstance.Spec.Configuration.Velero.DefaultPlugins { defaultPlugins[plugin] = emptyStruct{} } - featureFlags := make(map[string]emptyStruct) + veleroFeatureFlags := make(map[string]emptyStruct) for _, flag := range dpaInstance.Spec.Configuration.Velero.FeatureFlags { - featureFlags[flag] = emptyStruct{} + veleroFeatureFlags[flag] = emptyStruct{} } + dpaInstance.Spec.Features = &oadpv1alpha1.Features{DataMover: &oadpv1alpha1.DataMover{Enable: false}} switch backupRestoreType { case RESTIC: dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(true) delete(defaultPlugins, oadpv1alpha1.DefaultPluginCSI) - delete(featureFlags, "EnableCSI") + delete(veleroFeatureFlags, "EnableCSI") case CSI: dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(false) defaultPlugins[oadpv1alpha1.DefaultPluginCSI] = emptyStruct{} - featureFlags["EnableCSI"] = emptyStruct{} + veleroFeatureFlags["EnableCSI"] = emptyStruct{} + case CSIDataMover: + dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(false) + defaultPlugins[oadpv1alpha1.DefaultPluginCSI] = emptyStruct{} + veleroFeatureFlags["EnableCSI"] = emptyStruct{} + dpaInstance.Spec.Features.DataMover.Enable = true } dpaInstance.Spec.Configuration.Velero.DefaultPlugins = make([]oadpv1alpha1.DefaultPlugin, 0) for k := range defaultPlugins { dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, k) } dpaInstance.Spec.Configuration.Velero.FeatureFlags = make([]string, 0) - for k := range featureFlags { + for k := range veleroFeatureFlags { dpaInstance.Spec.Configuration.Velero.FeatureFlags = append(dpaInstance.Spec.Configuration.Velero.FeatureFlags, k) } v.CustomResource = &dpaInstance @@ -173,14 +186,26 @@ func (v *DpaCustomResource) CreateOrUpdateWithRetries(spec *oadpv1alpha1.DataPro ) for i := 0; i < retries; i++ { if cr, err = v.Get(); apierrors.IsNotFound(err) { - v.Build(v.backupRestoreType) - v.CustomResource.Spec = *spec + v.CustomResource = &oadpv1alpha1.DataProtectionApplication{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataProtectionApplication", + APIVersion: "oadp.openshift.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: v.Name, + Namespace: v.Namespace, + }, + Spec: *spec.DeepCopy(), + } return v.Create() } else if err != nil { return err } - cr.Spec = *spec - if err = v.Client.Update(context.Background(), cr); err != nil { + spec.DeepCopyInto(&cr.Spec) + cr.GetObjectKind().SetGroupVersionKind(schema.FromAPIVersionAndKind("oadp.openshift.io/v1alpha1", "DataProtectionApplication")) + cr.ObjectMeta.ManagedFields = nil + if err = v.Client.Patch(context.Background(), cr, client.Merge, &client.PatchOptions{}); err != nil { + log.Println("error patching velero cr", err) if apierrors.IsConflict(err) && i < retries-1 { log.Println("conflict detected during DPA CreateOrUpdate, retrying for ", retries-i-1, " more times") time.Sleep(time.Second * 2) @@ -219,6 +244,7 @@ func (v *DpaCustomResource) SetClient() error { operators.AddToScheme(client.Scheme()) volumesnapshotv1.AddToScheme(client.Scheme()) buildv1.AddToScheme(client.Scheme()) + operatorsv1alpha1.AddToScheme(client.Scheme()) v.Client = client return nil diff --git a/tests/e2e/volsync/volsync-sub.yaml b/tests/e2e/volsync/volsync-sub.yaml new file mode 100644 index 00000000000..3f5f0eb8469 --- /dev/null +++ b/tests/e2e/volsync/volsync-sub.yaml @@ -0,0 +1,16 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + labels: + operators.coreos.com/volsync-product.openshift-operators: "" + name: volsync-product + namespace: openshift-operators +spec: + channel: stable + installPlanApproval: Automatic + name: volsync-product + source: redhat-operators + sourceNamespace: openshift-marketplace + # v0.4.1 was latest version of the operator. We comment this out so it always install the latest version. + # If it breaks in the future we can re-specify startingCSV to install a specific version. + # startingCSV: volsync-product.v0.4.1