enhance tests (part2)

.github/workflows/e2e-multiple-k8s-clusters.yaml

@@ -19,6 +19,14 @@ concurrency:
 
 jobs:
   build:
+    strategy:
+      fail-fast: false
+      matrix:
+        package:
+          - replication
+          - changetostandalone
+          - changetoprimary
+          - changetosecondary
     #runs-on: "ubuntu-22.04"
     runs-on: mantle_large_runner_16core
     steps:
@@ -29,4 +37,4 @@ jobs:
       - run: sudo apt-get update
       - uses: ./.github/actions/set-up-kvm-for-e2e-tests
       - run: make -C test/e2e setup
-      - run: make -C test/e2e test-multiple-k8s-clusters
+      - run: make -C test/e2e test-multiple-k8s-clusters TEST_MULTIK8S_PACKAGES=${{ matrix.package }}

charts/mantle/templates/deployment.yaml

@@ -108,6 +108,9 @@ spec:
                   fieldPath: metadata.namespace
             - name: POD_IMAGE
               value: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+            {{- range .Values.controller.env }}
+            - {{ toYaml . | nindent 14 }}
+            {{- end }}
           ports:
             {{- toYaml .Values.controller.ports | nindent 12 }}
         - command:

charts/mantle/values.yaml

@@ -25,6 +25,7 @@ affinity: {}
 controller:
   role: standalone
   ports: []
+  env: []
 
 secondaryService:
   # type:

internal/controller/mantlebackup_controller.go

@@ -262,7 +262,7 @@ func (r *MantleBackupReconciler) getSnapshotTarget(ctx context.Context, backup *
 	}
 	if !ok {
 		logger.Info("waiting for PVC bound.")
-		return nil, ctrl.Result{Requeue: true}, nil
+		return nil, requeueReconciliation(), nil
 	}
 
 	pvName := pvc.Spec.VolumeName
@@ -392,7 +392,7 @@ func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, back
 	default:
 		return ctrl.Result{}, getSnapshotTargetErr
 	}
-	if result.Requeue {
+	if !result.IsZero() {
 		return result, nil
 	}
 
@@ -415,7 +415,7 @@ func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, back
 		if err != nil {
 			return ctrl.Result{}, err
 		}
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if err := r.expire(ctx, backup); err != nil {
@@ -461,7 +461,7 @@ func (r *MantleBackupReconciler) reconcileAsSecondary(ctx context.Context, backu
 	default:
 		return ctrl.Result{}, getSnapshotTargetErr
 	}
-	if result.Requeue {
+	if !result.IsZero() {
 		return result, nil
 	}
 
@@ -564,7 +564,7 @@ func (r *MantleBackupReconciler) replicateManifests(
 			*backup1.Status.SnapID < *backup.Status.SnapID &&
 				backup1.ObjectMeta.DeletionTimestamp.IsZero() &&
 				!meta.IsStatusConditionTrue(backup1.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
-			return ctrl.Result{Requeue: true}, nil
+			return requeueReconciliation(), nil
 		}
 	}
 
@@ -703,7 +703,7 @@ func (r *MantleBackupReconciler) finalizeStandalone(
 ) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 	if _, ok := backup.GetAnnotations()[annotDiffTo]; ok {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
@@ -741,7 +741,7 @@ func (r *MantleBackupReconciler) finalizeSecondary(
 ) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 	if _, ok := backup.GetAnnotations()[annotDiffTo]; ok {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
@@ -965,7 +965,7 @@ func (r *MantleBackupReconciler) export(
 		return ctrl.Result{}, err
 	}
 
-	return ctrl.Result{Requeue: true}, nil
+	return requeueReconciliation(), nil
 }
 
 func (r *MantleBackupReconciler) annotateExportTargetMantleBackup(
@@ -1025,7 +1025,7 @@ func (r *MantleBackupReconciler) checkIfNewJobCanBeCreated(ctx context.Context)
 	}
 
 	if len(jobs.Items) >= r.primarySettings.MaxExportJobs {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	return ctrl.Result{}, nil
@@ -1271,7 +1271,7 @@ func (r *MantleBackupReconciler) checkIfExportJobIsCompleted(
 		return ctrl.Result{}, nil
 	}
 
-	return ctrl.Result{Requeue: true}, nil
+	return requeueReconciliation(), nil
 }
 
 func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.Context, target *mantlev1.MantleBackup) error {
@@ -1422,7 +1422,7 @@ func (r *MantleBackupReconciler) startImport(
 ) (ctrl.Result, error) {
 	if !r.doesMantleBackupHaveSyncModeAnnot(backup) {
 		// SetSynchronizingg is not called yet or the cache is stale.
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if result, err := r.isExportDataAlreadyUploaded(ctx, backup); err != nil || !result.IsZero() {
@@ -1431,7 +1431,7 @@ func (r *MantleBackupReconciler) startImport(
 
 	// Requeue if the PV is smaller than the PVC. (This may be the case if pvc-autoresizer is used.)
 	if isPVSmallerThanPVC(target.pv, target.pvc) {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if err := r.updateStatusManifests(ctx, backup, target.pv, target.pvc); err != nil {
@@ -1528,7 +1528,7 @@ func (r *MantleBackupReconciler) isExportDataAlreadyUploaded(
 	if uploaded {
 		return ctrl.Result{}, nil
 	}
-	return ctrl.Result{Requeue: true}, nil
+	return requeueReconciliation(), nil
 }
 
 func isPVSmallerThanPVC(
@@ -1592,7 +1592,7 @@ func (r *MantleBackupReconciler) reconcileDiscardJob(
 	if completed {
 		return ctrl.Result{}, nil
 	}
-	return ctrl.Result{Requeue: true}, nil
+	return requeueReconciliation(), nil
 }
 
 func (r *MantleBackupReconciler) createOrUpdateDiscardPV(
@@ -1777,11 +1777,11 @@ func (r *MantleBackupReconciler) reconcileImportJob(
 		if err := r.createOrImportJob(ctx, backup, snapshotTarget); err != nil {
 			return ctrl.Result{}, err
 		}
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	if !IsJobConditionTrue(job.Status.Conditions, batchv1.JobComplete) {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	snapshot, err := ceph.FindRBDSnapshot(

internal/controller/mantlerestore_controller.go

@@ -137,7 +137,7 @@ func (r *MantleRestoreReconciler) restore(ctx context.Context, restore *mantlev1
 	// check if the backup is ReadyToUse
 	if !meta.IsStatusConditionTrue(backup.Status.Conditions, mantlev1.BackupConditionReadyToUse) {
 		logger.Info("backup is not ready to use", "backup", backup.Name, "namespace", backup.Namespace)
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	// store the pool name in the status

internal/controller/persistentvolume_controller.go

@@ -89,7 +89,7 @@ func (r *PersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 	// Wait until the PV's status becomes Released.
 	if pv.Status.Phase != corev1.VolumeReleased {
-		return ctrl.Result{Requeue: true}, nil
+		return requeueReconciliation(), nil
 	}
 
 	// Delete the RBD clone image.

internal/controller/util.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
+	"time"
 
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -81,3 +84,15 @@ func IsJobConditionTrue(conditions []batchv1.JobCondition, conditionType batchv1
 	}
 	return false
 }
+
+func requeueReconciliation() ctrl.Result {
+	requeueAfter := os.Getenv("REQUEUE_RECONCILIATION_AFTER")
+	if requeueAfter != "" {
+		duration, err := time.ParseDuration(requeueAfter)
+		if err != nil {
+			panic(fmt.Sprintf("set REQUEUE_RECONCILIATION_AFTER properly: %v", err))
+		}
+		return ctrl.Result{RequeueAfter: duration}
+	}
+	return ctrl.Result{Requeue: true}
+}

test/e2e/Makefile

@@ -25,6 +25,7 @@ LOOP_DEV2 := /dev/loop1
 MINIKUBE_PROFILE_PRIMARY=profile1
 MINIKUBE_PROFILE_SECONDARY=profile2
 TMPDIR := tmp
+TEST_MULTIK8S_PACKAGES := replication changetostandalone changetoprimary changetosecondary
 
 export MINIKUBE_HOME
 
@@ -240,4 +241,4 @@ do-test-multik8s: $(GINKGO)
 	E2ETEST=1 \
 	KUBECTL_PRIMARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_PRIMARY) kubectl -- " \
 	KUBECTL_SECONDARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_SECONDARY) kubectl -- " \
-	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) multik8s
+	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) $(addprefix multik8s/, $(TEST_MULTIK8S_PACKAGES))

test/e2e/multik8s/changetoprimary/suite_test.go

@@ -0,0 +1,122 @@
+package changetoprimary
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/cybozu-go/mantle/internal/controller"
+	. "github.com/cybozu-go/mantle/test/e2e/multik8s/testutil"
+	"github.com/cybozu-go/mantle/test/util"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMtest(t *testing.T) {
+	if os.Getenv("E2ETEST") == "" {
+		t.Skip("Run under e2e/")
+	}
+
+	RegisterFailHandler(Fail)
+
+	SetDefaultEventuallyPollingInterval(time.Second)
+	SetDefaultEventuallyTimeout(3 * time.Minute)
+	EnforceDefaultTimeoutsWhenUsingContexts()
+
+	RunSpecs(t, "replication test with multiple k8s clusters")
+}
+
+var _ = Describe("Mantle", func() {
+	Context("wait controller to be ready", WaitControllerToBeReady)
+	Context("change to primary", changePrimaryToStandaloneTemporarily)
+})
+
+func changePrimaryToStandaloneTemporarily() {
+	Describe("change to primary", func() {
+		var (
+			namespace                                                                  string
+			pvcName0, backupName00, backupName01, writtenDataHash00, writtenDataHash01 string
+			pvcName1, backupName10, writtenDataHash10                                  string
+		)
+
+		/*
+			Overview of the test:
+
+			 primary k8s cluster        | secondary k8s cluster
+			============================|==========================
+			  role=primary              | role=secondary
+			  PVC0, MB00 (created)      |
+			                            | PVC0, MB00 (synced)
+			  role=standalone (changed) |
+			  MB01, PVC1, MB10 (created)|
+			  role=primary (changed)    |
+			                            | MB01, PVC1, MB10 (synced)
+		*/
+
+		It("should replicate a MantleBackup resource", func(ctx context.Context) {
+			namespace = util.GetUniqueName("ns-")
+			pvcName0 = util.GetUniqueName("pvc-")
+			backupName00 = util.GetUniqueName("mb-")
+
+			SetupEnvironment(namespace)
+			CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName0)
+			writtenDataHash00 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
+			CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName00)
+			WaitMantleBackupSynced(namespace, backupName00)
+		})
+
+		It("should change the role from primary to standalone", func() {
+			By("changing the primary mantle to standalone")
+			err := ChangeClusterRole(PrimaryK8sCluster, controller.RoleStandalone)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should restore the synced MantleBackup in the both clusters", func(ctx context.Context) {
+			restoreName00 := util.GetUniqueName("mr-")
+			EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+			EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		})
+
+		It("should create a MantleBackup resource", func(ctx SpecContext) {
+			backupName01 = util.GetUniqueName("mb-")
+			writtenDataHash01 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
+
+			CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName01)
+
+			pvcName1 = util.GetUniqueName("pvc-")
+			backupName10 = util.GetUniqueName("mb-")
+
+			Eventually(func() error {
+				return ApplyPVCTemplate(PrimaryK8sCluster, namespace, pvcName1)
+			}).Should(Succeed())
+			writtenDataHash10 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName1)
+			CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName1, backupName10)
+		})
+
+		It("should change the role from standalone to primary", func() {
+			By("changing the standalone mantle to primary")
+			err := ChangeClusterRole(PrimaryK8sCluster, controller.RolePrimary)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should synchronize MantleBackups correctly", func() {
+			WaitMantleBackupSynced(namespace, backupName01)
+			WaitMantleBackupSynced(namespace, backupName10)
+		})
+
+		It("should restore MantleBackups correctly", func(ctx SpecContext) {
+			restoreName00 := util.GetUniqueName("mr-")
+			EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+			EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+
+			restoreName01 := util.GetUniqueName("mr-")
+			EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)
+			EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)
+
+			restoreName10 := util.GetUniqueName("mr-")
+			EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
+			EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
+		})
+	})
+}