diff --git a/internal/common/function/function.go b/internal/common/function/function.go index 6db8508..c96064f 100644 --- a/internal/common/function/function.go +++ b/internal/common/function/function.go @@ -602,3 +602,22 @@ func checkLabelAnnotationValueIsValid(labelsOrAnnotations map[string]string, key func GetLogger(ctx context.Context, obj client.Object, key string) logr.Logger { return log.FromContext(ctx).WithValues(key, types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}) } + +// TODO remove +func GetVeleroBackupStorageLocationByLabel(ctx context.Context, clientInstance client.Client, namespace string, labelValue string) (*velerov1.BackupStorageLocation, error) { + bslList := &velerov1.BackupStorageLocationList{} + + // Call the generic ListLabeledObjectsInNamespace function + if err := ListObjectsByLabel(ctx, clientInstance, namespace, "openshift.io/oadp-nabsl-origin-nacuuid", labelValue, bslList); err != nil { + return nil, err + } + + switch len(bslList.Items) { + case 0: + return nil, nil // No matching VeleroBackupStorageLocation found + case 1: + return &bslList.Items[0], nil // Found 1 matching VeleroBackupStorageLocation + default: + return nil, fmt.Errorf("multiple VeleroBackupStorageLocation objects found with label %s=%s in namespace '%s'", velerov1.StorageLocationLabel, labelValue, namespace) + } +} diff --git a/internal/controller/nonadminbackup_controller.go b/internal/controller/nonadminbackup_controller.go index f75270d..df34cd2 100644 --- a/internal/controller/nonadminbackup_controller.go +++ b/internal/controller/nonadminbackup_controller.go @@ -89,6 +89,8 @@ func (r *NonAdminBackupReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } + _, syncBackup := nab.Labels["openshift.io/oadp-nab-synced-from-nacuuid"] + // Determine which path to take var reconcileSteps []nonAdminBackupReconcileStepFunction @@ -117,6 +119,14 @@ func (r *NonAdminBackupReconciler) Reconcile(ctx context.Context, req ctrl.Reque r.removeNabFinalizerUponVeleroBackupDeletion, } + case syncBackup: + logger.V(1).Info("Executing nab sync path") + reconcileSteps = []nonAdminBackupReconcileStepFunction{ + r.setBackupUUIDInStatus, + r.setFinalizerOnNonAdminBackup, + r.createVeleroBackupAndSyncWithNonAdminBackup, + } + default: // Standard creation/update path logger.V(1).Info("Executing nab creation/update path") @@ -555,7 +565,12 @@ func (r *NonAdminBackupReconciler) setBackupUUIDInStatus(ctx context.Context, lo } if nab.Status.VeleroBackup == nil || nab.Status.VeleroBackup.NACUUID == constant.EmptyString { - veleroBackupNACUUID := function.GenerateNacObjectUUID(nab.Namespace, nab.Name) + var veleroBackupNACUUID string + if value, ok := nab.Labels["openshift.io/oadp-nab-synced-from-nacuuid"]; ok { + veleroBackupNACUUID = value + } else { + veleroBackupNACUUID = function.GenerateNacObjectUUID(nab.Namespace, nab.Name) + } nab.Status.VeleroBackup = &nacv1alpha1.VeleroBackup{ NACUUID: veleroBackupNACUUID, Namespace: r.OADPNamespace, diff --git a/internal/controller/sync_controller.go b/internal/controller/sync_controller.go new file mode 100644 index 0000000..4bbf5bc --- /dev/null +++ b/internal/controller/sync_controller.go @@ -0,0 +1,237 @@ +/* +TODO example implementation +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + // "github.com/sirupsen/logrus" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + // "github.com/vmware-tanzu/velero/pkg/persistence" + // "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + // "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + // "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + nacv1alpha1 "github.com/migtools/oadp-non-admin/api/v1alpha1" + "github.com/migtools/oadp-non-admin/internal/common/constant" + "github.com/migtools/oadp-non-admin/internal/common/function" +) + +// NonAdminBackupSyncReconciler reconciles a NonAdminBackupStorageLocation object +type NonAdminBackupSyncReconciler struct { + client.Client + Scheme *runtime.Scheme + OADPNamespace string + SyncPeriod time.Duration +} + +type nacCredentialsFileStore struct{} + +func (f *nacCredentialsFileStore) Path(*corev1.SecretKeySelector) (string, error) { + return "/tmp/credentials/secret-file", nil +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state, +// defined in NonAdminBackupStorageLocation object Spec. +func (r *NonAdminBackupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + // on NonAdminBackupStorageLocation create/update and every configured timestamp + logger.V(1).Info("NonAdminBackup Synchronization start") + + nonAdminBackupStorageLocation := &nacv1alpha1.NonAdminBackupStorageLocation{} + err := r.Get(ctx, req.NamespacedName, nonAdminBackupStorageLocation) + if err != nil { + if apierrors.IsNotFound(err) { + logger.V(1).Info(err.Error()) + return ctrl.Result{}, nil + } + logger.Error(err, "Unable to fetch NonAdminBackupStorageLocation") + return ctrl.Result{}, err + } + + // Get related Velero BSL + if nonAdminBackupStorageLocation.Status.VeleroBackupStorageLocation == nil { + err = fmt.Errorf("VeleroBackupStorageLocation not yet processed") + logger.Error(err, "VeleroBackupStorageLocation not yet processed") + return ctrl.Result{}, err + } + backupStorageLocation, err := function.GetVeleroBackupStorageLocationByLabel( + ctx, r.Client, r.OADPNamespace, + nonAdminBackupStorageLocation.Status.VeleroBackupStorageLocation.NACUUID, + ) + if err != nil { + logger.Error(err, "Failed to get VeleroBackupStorageLocation") + return ctrl.Result{}, err + } + if backupStorageLocation == nil { + err = fmt.Errorf("VeleroBackupStorageLocation not yet created") + logger.Error(err, "VeleroBackupStorageLocation not yet created") + return ctrl.Result{}, err + } + + // // OPTION 1: copy how Velero does + // // PROBLEM: NAC Pod will not have Velero Pod files + // pluginRegistry := process.NewRegistry("/plugins", logger, logger.Level) + // if err := pluginRegistry.DiscoverPlugins(); err != nil { + // return ctrl.Result{}, err + // } + // newPluginManager := func(logger logrus.FieldLogger) clientmgmt.Manager { + // return clientmgmt.NewManager(logger, logLevel, pluginRegistry) + // } + // objectStorageClient := persistence.NewObjectBackupStoreGetter(&nacCredentialsFileStore{}) + // objectStorage, err := objectStorageClient.Get(backupStorageLocation, newPluginManager(logger), logger) + // if err != nil { + // return ctrl.Result{}, err + // } + // objectStorageBackupList, err := objectStorage.ListBackups() + // if err != nil { + // return ctrl.Result{}, nil + // } + + // // get all Velero backups that + // // NAC owns + // // are finished + // // were created from same namespace as NABSL + // var veleroBackupList velerov1.BackupList + // labelSelector := client.MatchingLabels(function.GetNonAdminLabels()) + + // if err := r.List(ctx, &veleroBackupList, client.InNamespace(r.OADPNamespace), labelSelector); err != nil { + // return ctrl.Result{}, err + // } + // logger.V(1).Info(fmt.Sprintf("%v Backups owned by NAC controller in OADP namespace", len(veleroBackupList.Items))) + + // var backupsToSync []velerov1.Backup + // // var possibleBackupsToSync []velerov1.Backup + // var possibleBackupsToSync []string + // for _, backup := range veleroBackupList.Items { + // if backup.Status.CompletionTimestamp != nil && + // backup.Annotations[constant.NabOriginNamespaceAnnotation] == nonAdminBackupStorageLocation.Namespace { + // possibleBackupsToSync = append(possibleBackupsToSync, backup.Name) + // } + // } + + // objectStorageBackupSet := sets.New(objectStorageBackupList...) + // inClusterBackupSet := sets.New(possibleBackupsToSync...) + // backupsToSyncSet := objectStorageBackupSet.Difference(inClusterBackupSet) + + // for backupName := range backupsToSyncSet { + // veleroBackup := velerov1.Backup{} + // err := r.Get(ctx, types.NamespacedName{ + // Namespace: r.OADPNamespace, + // Name: backupName, + // }, &veleroBackup) + // if err != nil { + // if apierrors.IsNotFound(err) { + // // not yet synced by velero + // continue + // } + // logger.Error(err, "Unable to fetch Velero Backup") + // return ctrl.Result{}, err + // } + // backupsToSync = append(backupsToSync, veleroBackup) + // } + + // logger.V(1).Info(fmt.Sprintf("%v Backup(s) to sync to NonAdminBackupStorageLocation namespace", len(backupsToSync))) + // for _, backup := range backupsToSync { + // nab := &nacv1alpha1.NonAdminBackup{ + // ObjectMeta: v1.ObjectMeta{ + // Name: backup.Annotations[constant.NabOriginNameAnnotation], + // Namespace: backup.Annotations[constant.NabOriginNamespaceAnnotation], + // // TODO sync operation does not preserve labels + // Labels: map[string]string{ + // "openshift.io/oadp-nab-synced-from-nacuuid": backup.Labels[constant.NabOriginNACUUIDLabel], + // }, + // }, + // Spec: nacv1alpha1.NonAdminBackupSpec{ + // BackupSpec: &backup.Spec, + // }, + // } + // nab.Spec.BackupSpec.StorageLocation = nonAdminBackupStorageLocation.Name + // r.Create(ctx, nab) + // } + + // ------------------------------------------------------------------------ + + // OPTION 2: compare BSLs specs + // PROBLEM: if user deletes NABSL, and them recreates it, velero backup will point to a BSL that does not exist + + // get all Velero backups that + // NAC owns + // are finished + // were created from same namespace as NABSL + // were created from this NABSL + var veleroBackupList velerov1.BackupList + labelSelector := client.MatchingLabels(function.GetNonAdminLabels()) + + if err := r.List(ctx, &veleroBackupList, client.InNamespace(r.OADPNamespace), labelSelector); err != nil { + return ctrl.Result{}, err + } + + var backupsToSync []velerov1.Backup + var possibleBackupsToSync []velerov1.Backup + for _, backup := range veleroBackupList.Items { + if backup.Status.CompletionTimestamp != nil && + backup.Annotations[constant.NabOriginNamespaceAnnotation] == nonAdminBackupStorageLocation.Namespace && + backup.Spec.StorageLocation == backupStorageLocation.Name { + possibleBackupsToSync = append(possibleBackupsToSync, backup) + } + } + logger.V(1).Info(fmt.Sprintf("%v possible Backup(s) to be synced to NonAdminBackupStorageLocation namespace", len(possibleBackupsToSync))) + + for _, backup := range possibleBackupsToSync { + nab := &nacv1alpha1.NonAdminBackup{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: backup.Annotations[constant.NabOriginNamespaceAnnotation], + Name: backup.Annotations[constant.NabOriginNameAnnotation], + }, nab) + if err != nil { + if apierrors.IsNotFound(err) { + backupsToSync = append(backupsToSync, backup) + continue + } + logger.Error(err, "Unable to fetch NonAdminBackup") + return ctrl.Result{}, err + } + } + logger.V(1).Info(fmt.Sprintf("%v Backup(s) to sync to NonAdminBackupStorageLocation namespace", len(backupsToSync))) + for _, backup := range backupsToSync { + nab := &nacv1alpha1.NonAdminBackup{ + ObjectMeta: v1.ObjectMeta{ + Name: backup.Annotations[constant.NabOriginNameAnnotation], + Namespace: backup.Annotations[constant.NabOriginNamespaceAnnotation], + // TODO sync operation does not preserve labels + Labels: map[string]string{ + "openshift.io/oadp-nab-synced-from-nacuuid": backup.Labels[constant.NabOriginNACUUIDLabel], + }, + }, + Spec: nacv1alpha1.NonAdminBackupSpec{ + BackupSpec: &backup.Spec, + }, + } + nab.Spec.BackupSpec.StorageLocation = nonAdminBackupStorageLocation.Name + r.Create(ctx, nab) + } + + logger.V(1).Info("NonAdminBackup Synchronization exit") + return ctrl.Result{RequeueAfter: r.SyncPeriod}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NonAdminBackupSyncReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nacv1alpha1.NonAdminBackupStorageLocation{}). + Complete(r) +} diff --git a/internal/controller/sync_controller_test.go b/internal/controller/sync_controller_test.go new file mode 100644 index 0000000..f08cea2 --- /dev/null +++ b/internal/controller/sync_controller_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" + + nacv1alpha1 "github.com/migtools/oadp-non-admin/api/v1alpha1" +) + +type nonAdminBackupStorageLocationFullReconcileScenario struct { + spec nacv1alpha1.NonAdminBackupStorageLocationSpec +} + +func buildTestNonAdminBackupStorageLocation(nonAdminNamespace string, nonAdminName string, spec nacv1alpha1.NonAdminBackupStorageLocationSpec) *nacv1alpha1.NonAdminBackupStorageLocation { + return &nacv1alpha1.NonAdminBackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: nonAdminName, + Namespace: nonAdminNamespace, + }, + Spec: spec, + } +} + +var _ = ginkgo.Describe("Test full reconcile loop of NonAdminBackupSyncReconciler Controller", func() { + var ( + ctx context.Context + cancel context.CancelFunc + nonAdminBackupStorageLocationName string + nonAdminBackupStorageLocationNamespace string + oadpNamespace string + counter int + ) + + ginkgo.BeforeEach(func() { + counter++ + nonAdminBackupStorageLocationName = fmt.Sprintf("non-admin-bsl-object-%v", counter) + nonAdminBackupStorageLocationNamespace = fmt.Sprintf("test-non-admin-bsl-reconcile-full-%v", counter) + oadpNamespace = nonAdminBackupStorageLocationNamespace + "-oadp" + }) + + ginkgo.AfterEach(func() { + gomega.Expect(deleteTestNamespaces(ctx, nonAdminBackupStorageLocationNamespace, oadpNamespace)).To(gomega.Succeed()) + + cancel() + // wait cancel + time.Sleep(1 * time.Second) + }) + + ginkgo.FDescribeTable("Reconcile triggered by NonAdminBackupStorageLocation Create event", + func(scenario nonAdminBackupStorageLocationFullReconcileScenario) { + ctx, cancel = context.WithCancel(context.Background()) + + gomega.Expect(createTestNamespaces(ctx, nonAdminBackupStorageLocationNamespace, oadpNamespace)).To(gomega.Succeed()) + + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: k8sClient.Scheme(), + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + err = (&NonAdminBackupSyncReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + OADPNamespace: oadpNamespace, + SyncPeriod: 3 * time.Second, + }).SetupWithManager(k8sManager) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + go func() { + defer ginkgo.GinkgoRecover() + err = k8sManager.Start(ctx) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run manager") + }() + // wait manager start + managerStartTimeout := 30 * time.Second + pollInterval := 100 * time.Millisecond + ctxTimeout, cancel := context.WithTimeout(ctx, managerStartTimeout) + defer cancel() + + // TODO read logs + err = wait.PollUntilContextTimeout(ctxTimeout, pollInterval, managerStartTimeout, true, func(ctx context.Context) (done bool, err error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + // Check if the manager has started by verifying if the client is initialized + return k8sManager.GetClient() != nil, nil + } + }) + // Check if the context timeout or another error occurred + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "Manager failed to start within the timeout period") + + ginkgo.By("Waiting Reconcile of create event") + nonAdminBackupStorageLocation := buildTestNonAdminBackupStorageLocation(nonAdminBackupStorageLocationNamespace, nonAdminBackupStorageLocationName, scenario.spec) + gomega.Expect(k8sClient.Create(ctxTimeout, nonAdminBackupStorageLocation)).To(gomega.Succeed()) + // wait NonAdminRestore reconcile + time.Sleep(13 * time.Second) + + // TODO check synced NonAdminBackups specs are expected + + ginkgo.By("Waiting NonAdminBackupStorageLocation deletion") + gomega.Expect(k8sClient.Delete(ctxTimeout, nonAdminBackupStorageLocation)).To(gomega.Succeed()) + gomega.Eventually(func() (bool, error) { + err := k8sClient.Get( + ctxTimeout, + types.NamespacedName{ + Name: nonAdminBackupStorageLocationName, + Namespace: nonAdminBackupStorageLocationNamespace, + }, + nonAdminBackupStorageLocation, + ) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + }, 10*time.Second, 1*time.Second).Should(gomega.BeTrue()) + }, + ginkgo.Entry("Should sync NonAdminBackup 5 times, then delete NonAdminBackupStorageLocation", nonAdminBackupStorageLocationFullReconcileScenario{ + spec: nacv1alpha1.NonAdminBackupStorageLocationSpec{ + BackupStorageLocationSpec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + StorageType: velerov1.StorageType{ + ObjectStorage: &velerov1.ObjectStorageLocation{ + Bucket: "my-bucket-name", + Prefix: "velero", + }, + }, + Config: map[string]string{ + "bucket": "my-bucket-name", + "prefix": "velero", + }, + Credential: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: "cloud", + }, + }, + }, + }), + // ginkgo.Entry("Should check that NonAdminBackupStorageLocation update changes next sync time, then delete NonAdminBackupStorageLocation", nonAdminBackupStorageLocationFullReconcileScenario{}), + // ginkgo.Entry("Should sync only finished NonAdminBackups, then delete NonAdminBackupStorageLocation", nonAdminBackupStorageLocationFullReconcileScenario{}), + // ginkgo.Entry("Should sync only related NonAdminBackupStorageLocation NonAdminBackups, then delete NonAdminBackupStorageLocation", nonAdminBackupStorageLocationFullReconcileScenario{}), + ) +})