From 10196b1f7f0da579d23fe65b07e354f49d77f949 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Mon, 23 Dec 2024 02:30:06 +0000
Subject: [PATCH 01/11] test/e2e: singlek8s: make sure no resources necessary
 only for primary/secondary mantle-controller are created

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/singlek8s/backup_test.go | 39 +++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/test/e2e/singlek8s/backup_test.go b/test/e2e/singlek8s/backup_test.go
index cae2d2e1..b2d4c3bf 100644
--- a/test/e2e/singlek8s/backup_test.go
+++ b/test/e2e/singlek8s/backup_test.go
@@ -1,6 +1,7 @@
 package singlek8s
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -9,6 +10,8 @@ import (
 	"github.com/cybozu-go/mantle/test/util"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -165,6 +168,42 @@ func (test *backupTest) testCase1() {
 		}).Should(Succeed())
 	})
 
+	It("should not create any resources necessary only for primary or secondary mantle controller", func() {
+		// Check that export and upload Jobs are not created.
+		stdout, _, err := kubectl("-n", cephCluster1Namespace, "get", "job", "-o", "json")
+		Expect(err).NotTo(HaveOccurred())
+		var jobs batchv1.JobList
+		err = json.Unmarshal(stdout, &jobs)
+		Expect(err).NotTo(HaveOccurred())
+		for _, job := range jobs.Items {
+			Expect(strings.HasPrefix(job.GetName(), "mantle-export-")).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), "mantle-upload-")).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), "mantle-import-")).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), "mantle-discard-")).To(BeFalse())
+		}
+
+		// Check that export and discard PVC are not created.
+		stdout, _, err = kubectl("-n", cephCluster1Namespace, "get", "pvc", "-o", "json")
+		Expect(err).NotTo(HaveOccurred())
+		var pvcs corev1.PersistentVolumeClaimList
+		err = json.Unmarshal(stdout, &pvcs)
+		Expect(err).NotTo(HaveOccurred())
+		for _, pvc := range pvcs.Items {
+			Expect(strings.HasPrefix(pvc.GetName(), "mantle-export-")).To(BeFalse())
+			Expect(strings.HasPrefix(pvc.GetName(), "mantle-discard-")).To(BeFalse())
+		}
+
+		// Check that discard PV is not created.
+		stdout, _, err = kubectl("-n", cephCluster1Namespace, "get", "pv", "-o", "json")
+		Expect(err).NotTo(HaveOccurred())
+		var pvs corev1.PersistentVolumeList
+		err = json.Unmarshal(stdout, &pvs)
+		Expect(err).NotTo(HaveOccurred())
+		for _, pv := range pvs.Items {
+			Expect(strings.HasPrefix(pv.GetName(), "mantle-discard-")).To(BeFalse())
+		}
+	})
+
 	It("should not delete MantleBackup resource when delete backup target PVC", func() {
 		By("Deleting backup target PVC")
 		_, _, err := kubectl("-n", test.tenantNamespace, "delete", "pvc", test.pvcName2)

From 7c81a1e0db4cecd904f57494e1d6c7ccffc74b71 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Mon, 23 Dec 2024 04:55:16 +0000
Subject: [PATCH 02/11] test/e2e: multik8s: add tests when changing
 mantle-controller roles to standalone

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 102 ++++++++++++++++++++++++++++++++
 test/e2e/multik8s/util.go       |  68 ++++++++++++++++++++-
 2 files changed, 169 insertions(+), 1 deletion(-)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 4796eea4..e8019796 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -2,12 +2,14 @@ package multik8s
 
 import (
 	_ "embed"
+	"encoding/json"
 	"errors"
 	"os"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/cybozu-go/mantle/internal/controller"
 	"github.com/cybozu-go/mantle/test/util"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -35,6 +37,7 @@ func TestMtest(t *testing.T) {
 var _ = Describe("Mantle", func() {
 	Context("wait controller to be ready", waitControllerToBeReady)
 	Context("replication test", replicationTestSuite)
+	Context("change to standalone", changeToStandalone)
 })
 
 func waitControllerToBeReady() {
@@ -181,3 +184,102 @@ func replicationTestSuite() {
 		})
 	})
 }
+
+func changeToStandalone() {
+	Describe("change to standalone", func() {
+		var namespace, pvcName, backupName string
+
+		It("should replicate a MantleBackup resource", func() {
+			namespace = util.GetUniqueName("ns-")
+			pvcName = util.GetUniqueName("pvc-")
+			backupName = util.GetUniqueName("mb-")
+
+			By("setting up the environment")
+			Eventually(func() error {
+				return createNamespace(primaryK8sCluster, namespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return createNamespace(secondaryK8sCluster, namespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
+			}).Should(Succeed())
+
+			By("creating a MantleBackup resource")
+			Eventually(func() error {
+				return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
+			}).Should(Succeed())
+
+			By("checking MantleBackup's SyncedToRemote status")
+			Eventually(func() error {
+				mb, err := getMB(primaryK8sCluster, namespace, backupName)
+				if err != nil {
+					return err
+				}
+				if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
+					return errors.New("status of SyncedToRemote condition is not True")
+				}
+				return nil
+			}, "10m", "1s").Should(Succeed())
+		})
+
+		It("should change the roles to standalone", func() {
+			By("changing the primary mantle to standalone")
+			err := changeClusterRole(primaryK8sCluster, controller.RoleStandalone)
+			Expect(err).NotTo(HaveOccurred())
+			By("changing the secondary mantle to standalone")
+			err = changeClusterRole(secondaryK8sCluster, controller.RoleStandalone)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should delete MantleBackup created by primary mantle from standalone mantle", func(ctx SpecContext) {
+			By("deleting the MantleBackup in the primary cluster")
+			_, _, err := kubectl(primaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("checking that the MantleBackup is actually deleted")
+			Eventually(ctx, func(g Gomega) {
+				stdout, _, err := kubectl(primaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json")
+				g.Expect(err).NotTo(HaveOccurred())
+				var mbs mantlev1.MantleBackupList
+				err = json.Unmarshal(stdout, &mbs)
+				g.Expect(err).NotTo(HaveOccurred())
+				found := false
+				for _, mb := range mbs.Items {
+					if mb.GetName() == backupName {
+						found = true
+					}
+				}
+				g.Expect(found).To(BeFalse())
+			}).Should(Succeed())
+		})
+
+		It("should NOT delete MantleBackup created by secondary mantle from standalone mantle", func(ctx SpecContext) {
+			By("deleting the MantleBackup in the secondary cluster")
+			_, _, err := kubectl(secondaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("checking that the MantleBackup is NOT deleted")
+			Consistently(ctx, func(g Gomega) {
+				stdout, _, err := kubectl(secondaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json")
+				g.Expect(err).NotTo(HaveOccurred())
+				var mbs mantlev1.MantleBackupList
+				err = json.Unmarshal(stdout, &mbs)
+				g.Expect(err).NotTo(HaveOccurred())
+				found := false
+				for _, mb := range mbs.Items {
+					if mb.GetName() == backupName {
+						found = true
+					}
+				}
+				g.Expect(found).To(BeTrue())
+			}, "10s", "1s").Should(Succeed())
+		})
+	})
+}
diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go
index ca08b530..0affeb69 100644
--- a/test/e2e/multik8s/util.go
+++ b/test/e2e/multik8s/util.go
@@ -4,12 +4,16 @@ import (
 	"bytes"
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
+	"time"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -112,7 +116,7 @@ func createNamespace(clusterNo int, name string) error {
 	return nil
 }
 
-func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error {
+func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error { //nolint:unparam
 	manifest := fmt.Sprintf(
 		testRBDPoolSCTemplate, namespace,
 		namespace, namespace, namespace, namespace)
@@ -148,3 +152,65 @@ func getPVC(clusterNo int, namespace, name string) (*corev1.PersistentVolumeClai
 func getMR(clusterNo int, namespace, name string) (*mantlev1.MantleRestore, error) {
 	return getObject[mantlev1.MantleRestore](clusterNo, "mantlerestore", namespace, name)
 }
+
+func getDeploy(clusterNo int, namespace, name string) (*appsv1.Deployment, error) {
+	return getObject[appsv1.Deployment](clusterNo, "deploy", namespace, name)
+}
+
+func changeClusterRole(clusterNo int, newRole string) error {
+	deployName := "mantle-controller"
+	deploy, err := getDeploy(clusterNo, cephClusterNamespace, deployName)
+	if err != nil {
+		return fmt.Errorf("failed to get mantle-controller deploy: %w", err)
+	}
+
+	roleIndex := slices.IndexFunc(
+		deploy.Spec.Template.Spec.Containers[0].Args,
+		func(arg string) bool { return strings.HasPrefix(arg, "--role=") },
+	)
+	if roleIndex == -1 {
+		return errors.New("failed to find --role= argument")
+	}
+
+	_, _, err = kubectl(
+		clusterNo, nil, "patch", "deploy", "-n", cephClusterNamespace, deployName, "--type=json",
+		fmt.Sprintf(
+			`-p=[{"op": "replace", "path": "/spec/template/spec/containers/0/args/%d", "value":"--role=%s"}]`,
+			roleIndex,
+			newRole,
+		),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to patch mantle-controller deploy: %w", err)
+	}
+
+	// Wait for the new controller to start
+	numRetries := 10
+	for i := 0; i < numRetries; i++ {
+		stdout, _, err := kubectl(clusterNo, nil, "get", "pod", "-n", cephClusterNamespace, "-o", "json")
+		if err != nil {
+			return fmt.Errorf("failed to get pod: %w", err)
+		}
+		var pods corev1.PodList
+		err = json.Unmarshal(stdout, &pods)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal pod list: %w", err)
+		}
+		ready := true
+		for _, pod := range pods.Items {
+			if strings.HasPrefix(pod.GetName(), deployName) {
+				for _, container := range pod.Spec.Containers {
+					if !slices.Contains(container.Args, fmt.Sprintf("--role=%s", newRole)) {
+						ready = false
+					}
+				}
+			}
+		}
+		if ready {
+			break
+		}
+		time.Sleep(10 * time.Second)
+	}
+
+	return nil
+}

From 671c45ed437531de43c8e407b0574a5dfd49eeed Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:37:09 +0000
Subject: [PATCH 03/11] test/e2e: multik8s: check MantleRestore correctly
 restores replicated backups in both clusters

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go               | 190 ++++++++++--------
 .../testdata/mount-deploy-template.yaml       |  33 +++
 .../multik8s/testdata/write-job-template.yaml |  25 +++
 test/e2e/multik8s/util.go                     |  39 ++++
 4 files changed, 204 insertions(+), 83 deletions(-)
 create mode 100644 test/e2e/multik8s/testdata/mount-deploy-template.yaml
 create mode 100644 test/e2e/multik8s/testdata/write-job-template.yaml

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index e8019796..b3e2f559 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -1,9 +1,11 @@
 package multik8s
 
 import (
+	"context"
 	_ "embed"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"reflect"
 	"testing"
@@ -17,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -52,47 +55,112 @@ func waitControllerToBeReady() {
 	})
 }
 
+func setupEnvironment(namespace, pvcName string) {
+	GinkgoHelper()
+	By("setting up the environment")
+	Eventually(func() error {
+		return createNamespace(primaryK8sCluster, namespace)
+	}).Should(Succeed())
+	Eventually(func() error {
+		return createNamespace(secondaryK8sCluster, namespace)
+	}).Should(Succeed())
+	Eventually(func() error {
+		return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace)
+	}).Should(Succeed())
+	Eventually(func() error {
+		return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace)
+	}).Should(Succeed())
+	Eventually(func() error {
+		return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
+	}).Should(Succeed())
+}
+
+func writeRandomDataToPV(ctx context.Context, namespace, pvcName string) string {
+	GinkgoHelper()
+	By("writing some random data to PV(C)")
+	writeJobName := util.GetUniqueName("job-")
+	Eventually(ctx, func() error {
+		return applyWriteJobTemplate(primaryK8sCluster, namespace, writeJobName, pvcName)
+	}).Should(Succeed())
+	Eventually(ctx, func(g Gomega) {
+		job, err := getJob(primaryK8sCluster, namespace, writeJobName)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(IsJobConditionTrue(job.Status.Conditions, batchv1.JobComplete)).To(BeTrue())
+	}).Should(Succeed())
+	stdout, _, err := kubectl(primaryK8sCluster, nil, "logs", "-n", namespace, "job/"+writeJobName)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(stdout)).NotTo(Equal(0))
+	return string(stdout)
+}
+
+func createMantleBackup(namespace, pvcName, backupName string) {
+	GinkgoHelper()
+	By("creating a MantleBackup object")
+	Eventually(func() error {
+		return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
+	}).Should(Succeed())
+}
+
+func waitMantleBackupSynced(namespace, backupName string) {
+	GinkgoHelper()
+	By("checking MantleBackup's SyncedToRemote status")
+	Eventually(func() error {
+		mb, err := getMB(primaryK8sCluster, namespace, backupName)
+		if err != nil {
+			return err
+		}
+		if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
+			return errors.New("status of SyncedToRemote condition is not True")
+		}
+		return nil
+	}, "10m", "1s").Should(Succeed())
+}
+
+func ensureCorrectRestoration(
+	clusterNo int,
+	ctx context.Context,
+	namespace, backupName, restoreName, writtenDataHash string,
+) {
+	GinkgoHelper()
+	mountDeployName := util.GetUniqueName("deploy-")
+	clusterName := "primary"
+	if clusterNo == secondaryK8sCluster {
+		clusterName = "secondary"
+	}
+	By(fmt.Sprintf("%s: %s: creating MantleRestore by using the MantleBackup replicated above",
+		clusterName, backupName))
+	Eventually(ctx, func() error {
+		return applyMantleRestoreTemplate(clusterNo, namespace, restoreName, backupName)
+	}).Should(Succeed())
+	By(fmt.Sprintf("%s: %s: checking the MantleRestore can be ready to use", clusterName, backupName))
+	Eventually(ctx, func(g Gomega) {
+		mr, err := getMR(clusterNo, namespace, restoreName)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse")).To(BeTrue())
+	}).Should(Succeed())
+	By(fmt.Sprintf("%s: %s: checking the MantleRestore has the correct contents", clusterName, backupName))
+	Eventually(ctx, func(g Gomega) {
+		err := applyMountDeployTemplate(clusterNo, namespace, mountDeployName, restoreName)
+		g.Expect(err).NotTo(HaveOccurred())
+		stdout, _, err := kubectl(clusterNo, nil, "exec", "-n", namespace, "deploy/"+mountDeployName, "--",
+			"bash", "-c", "sha256sum /volume/data | awk '{print $1}'")
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(string(stdout)).To(Equal(writtenDataHash))
+	}).Should(Succeed())
+}
+
 func replicationTestSuite() {
 	Describe("replication test", func() {
-		It("should correctly replicate PVC and MantleBackup resources", func() {
+		It("should correctly replicate PVC and MantleBackup resources", func(ctx SpecContext) {
 			namespace := util.GetUniqueName("ns-")
 			pvcName := util.GetUniqueName("pvc-")
 			backupName := util.GetUniqueName("mb-")
 			restoreName := util.GetUniqueName("mr-")
 
-			By("setting up the environment")
-			Eventually(func() error {
-				return createNamespace(primaryK8sCluster, namespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return createNamespace(secondaryK8sCluster, namespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
-			}).Should(Succeed())
-
-			By("creating a MantleBackup object")
-			Eventually(func() error {
-				return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
-			}).Should(Succeed())
-
-			By("checking MantleBackup's SyncedToRemote status")
-			Eventually(func() error {
-				mb, err := getMB(primaryK8sCluster, namespace, backupName)
-				if err != nil {
-					return err
-				}
-				if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
-					return errors.New("status of SyncedToRemote condition is not True")
-				}
-				return nil
-			}, "10m", "1s").Should(Succeed())
+			setupEnvironment(namespace, pvcName)
+			writtenDataHash := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName)
+			waitMantleBackupSynced(namespace, backupName)
 
 			By("checking PVC is replicated")
 			Eventually(func() error {
@@ -165,22 +233,8 @@ func replicationTestSuite() {
 				return nil
 			}).Should(Succeed())
 
-			By("creating MantleRestore on the secondary k8s cluster by using the MantleBackup replicated above")
-			Eventually(func() error {
-				return applyMantleRestoreTemplate(secondaryK8sCluster, namespace, restoreName, backupName)
-			}).Should(Succeed())
-
-			By("checking MantleRestore can be ready to use")
-			Eventually(func() error {
-				mr, err := getMR(secondaryK8sCluster, namespace, restoreName)
-				if err != nil {
-					return err
-				}
-				if !meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse") {
-					return errors.New("ReadyToUse of .Status.Conditions is not True")
-				}
-				return nil
-			}).Should(Succeed())
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 		})
 	})
 }
@@ -194,39 +248,9 @@ func changeToStandalone() {
 			pvcName = util.GetUniqueName("pvc-")
 			backupName = util.GetUniqueName("mb-")
 
-			By("setting up the environment")
-			Eventually(func() error {
-				return createNamespace(primaryK8sCluster, namespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return createNamespace(secondaryK8sCluster, namespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace)
-			}).Should(Succeed())
-			Eventually(func() error {
-				return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
-			}).Should(Succeed())
-
-			By("creating a MantleBackup resource")
-			Eventually(func() error {
-				return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
-			}).Should(Succeed())
-
-			By("checking MantleBackup's SyncedToRemote status")
-			Eventually(func() error {
-				mb, err := getMB(primaryK8sCluster, namespace, backupName)
-				if err != nil {
-					return err
-				}
-				if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
-					return errors.New("status of SyncedToRemote condition is not True")
-				}
-				return nil
-			}, "10m", "1s").Should(Succeed())
+			setupEnvironment(namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName)
+			waitMantleBackupSynced(namespace, backupName)
 		})
 
 		It("should change the roles to standalone", func() {
diff --git a/test/e2e/multik8s/testdata/mount-deploy-template.yaml b/test/e2e/multik8s/testdata/mount-deploy-template.yaml
new file mode 100644
index 00000000..a2d2fb7e
--- /dev/null
+++ b/test/e2e/multik8s/testdata/mount-deploy-template.yaml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  selector:
+    matchLabels:
+      app: %s
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: %s
+    spec:
+      securityContext:
+        runAsUser: 10000
+        runAsGroup: 10000
+      containers:
+      - name: ubuntu
+        image: ubuntu:22.04
+        volumeMounts:
+          - name: volume
+            mountPath: /volume
+        command:
+          - bash
+          - -c
+          - |
+            sleep infinity
+      volumes:
+        - name: volume
+          persistentVolumeClaim:
+            claimName: %s
diff --git a/test/e2e/multik8s/testdata/write-job-template.yaml b/test/e2e/multik8s/testdata/write-job-template.yaml
new file mode 100644
index 00000000..88d0bcde
--- /dev/null
+++ b/test/e2e/multik8s/testdata/write-job-template.yaml
@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  template:
+    spec:
+      containers:
+      - name: ubuntu
+        image: ubuntu:22.04
+        command:
+        - bash
+        - -c
+        - |
+          dd if=/dev/urandom of=/volume/data bs=1K count=1 >&/dev/null
+          sha256sum /volume/data | awk '{print $1}'
+        volumeMounts:
+        - name: volume
+          mountPath: /volume
+      restartPolicy: Never
+      volumes:
+      - name: volume
+        persistentVolumeClaim:
+          claimName: %s
diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go
index 0affeb69..9511cff3 100644
--- a/test/e2e/multik8s/util.go
+++ b/test/e2e/multik8s/util.go
@@ -14,6 +14,7 @@ import (
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -32,6 +33,10 @@ var (
 	testMantleBackupTemplate string
 	//go:embed testdata/mantlerestore-template.yaml
 	testMantleRestoreTemplate string
+	//go:embed testdata/mount-deploy-template.yaml
+	mountDeployTemplate string
+	//go:embed testdata/write-job-template.yaml
+	writeJobTemplate string
 
 	kubectlPrefixPrimary   = os.Getenv("KUBECTL_PRIMARY")
 	kubectlPrefixSecondary = os.Getenv("KUBECTL_SECONDARY")
@@ -108,6 +113,24 @@ func applyPVCTemplate(clusterNo int, namespace, name string) error {
 	return nil
 }
 
+func applyMountDeployTemplate(clusterNo int, namespace, name, pvcName string) error {
+	manifest := fmt.Sprintf(mountDeployTemplate, name, namespace, name, name, pvcName)
+	_, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-")
+	if err != nil {
+		return fmt.Errorf("kubectl apply mount deploy failed. err: %w", err)
+	}
+	return nil
+}
+
+func applyWriteJobTemplate(clusterNo int, namespace, name, pvcName string) error {
+	manifest := fmt.Sprintf(writeJobTemplate, name, namespace, pvcName)
+	_, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-")
+	if err != nil {
+		return fmt.Errorf("kubectl apply write job failed. err: %w", err)
+	}
+	return nil
+}
+
 func createNamespace(clusterNo int, name string) error {
 	_, _, err := kubectl(clusterNo, nil, "create", "ns", name)
 	if err != nil {
@@ -157,6 +180,10 @@ func getDeploy(clusterNo int, namespace, name string) (*appsv1.Deployment, error
 	return getObject[appsv1.Deployment](clusterNo, "deploy", namespace, name)
 }
 
+func getJob(clusterNo int, namespace, name string) (*batchv1.Job, error) {
+	return getObject[batchv1.Job](clusterNo, "job", namespace, name)
+}
+
 func changeClusterRole(clusterNo int, newRole string) error {
 	deployName := "mantle-controller"
 	deploy, err := getDeploy(clusterNo, cephClusterNamespace, deployName)
@@ -214,3 +241,15 @@ func changeClusterRole(clusterNo int, newRole string) error {
 
 	return nil
 }
+
+// IsJobConditionTrue returns true when the conditionType is present and set to
+// `metav1.ConditionTrue`.  Otherwise, it returns false.  Note that we can't use
+// meta.IsStatusConditionTrue because it doesn't accept []JobCondition.
+func IsJobConditionTrue(conditions []batchv1.JobCondition, conditionType batchv1.JobConditionType) bool {
+	for _, cond := range conditions {
+		if cond.Type == conditionType && cond.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}

From 2c41f455aee8f843bf842189695d55762415224f Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:38:16 +0000
Subject: [PATCH 04/11] test/e2e: multik8s: make sure all temporary resources
 are deleted after sync succeeded

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go |  53 ++++++++++++++++
 test/e2e/multik8s/util.go       | 107 ++++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index b3e2f559..6d49ec3d 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -116,6 +118,56 @@ func waitMantleBackupSynced(namespace, backupName string) {
 	}, "10m", "1s").Should(Succeed())
 }
 
+func ensureTemporaryResourcesRemoved(ctx context.Context) {
+	GinkgoHelper()
+	By("checking all temporary Jobs related to export and import of RBD images are removed")
+	primaryJobList, err := getObjectList[batchv1.JobList](primaryK8sCluster, "job", cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(slices.ContainsFunc(primaryJobList.Items, func(job batchv1.Job) bool {
+		n := job.GetName()
+		return strings.HasPrefix(n, "mantle-export-") ||
+			strings.HasPrefix(n, "mantle-upload-")
+	})).To(BeFalse())
+	secondaryJobList, err := getObjectList[batchv1.JobList](secondaryK8sCluster, "job", cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(slices.ContainsFunc(secondaryJobList.Items, func(job batchv1.Job) bool {
+		n := job.GetName()
+		return strings.HasPrefix(n, "mantle-import-") ||
+			strings.HasPrefix(n, "mantle-discard-")
+	})).To(BeFalse())
+
+	By("checking all temporary PVCs related to export and import of RBD images are removed")
+	primaryPVCList, err := getObjectList[corev1.PersistentVolumeClaimList](
+		primaryK8sCluster, "pvc", cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(slices.ContainsFunc(primaryPVCList.Items, func(pvc corev1.PersistentVolumeClaim) bool {
+		n := pvc.GetName()
+		return strings.HasPrefix(n, "mantle-export-")
+	})).To(BeFalse())
+	secondaryPVCList, err := getObjectList[corev1.PersistentVolumeClaimList](
+		secondaryK8sCluster, "pvc", cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(slices.ContainsFunc(secondaryPVCList.Items, func(pvc corev1.PersistentVolumeClaim) bool {
+		n := pvc.GetName()
+		return strings.HasPrefix(n, "mantle-discard-")
+	})).To(BeFalse())
+
+	By("checking all temporary PVs related to export and import of RBD images are removed")
+	secondaryPVList, err := getObjectList[corev1.PersistentVolumeList](secondaryK8sCluster, "pv", cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(slices.ContainsFunc(secondaryPVList.Items, func(pv corev1.PersistentVolume) bool {
+		n := pv.GetName()
+		return strings.HasPrefix(n, "mantle-discard-")
+	})).To(BeFalse())
+
+	By("checking all temporary objects in the object storage related to export and import of RBD images are removed")
+	objectStorageClient, err := createObjectStorageClient(ctx)
+	Expect(err).NotTo(HaveOccurred())
+	listOutput, err := objectStorageClient.listObjects(ctx)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(listOutput.Contents)).To(Equal(0))
+}
+
 func ensureCorrectRestoration(
 	clusterNo int,
 	ctx context.Context,
@@ -233,6 +285,7 @@ func replicationTestSuite() {
 				return nil
 			}).Should(Succeed())
 
+			ensureTemporaryResourcesRemoved(ctx)
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 		})
diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go
index 9511cff3..2a28c2d9 100644
--- a/test/e2e/multik8s/util.go
+++ b/test/e2e/multik8s/util.go
@@ -2,6 +2,7 @@ package multik8s
 
 import (
 	"bytes"
+	"context"
 	_ "embed"
 	"encoding/json"
 	"errors"
@@ -12,6 +13,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -184,6 +188,26 @@ func getJob(clusterNo int, namespace, name string) (*batchv1.Job, error) {
 	return getObject[batchv1.Job](clusterNo, "job", namespace, name)
 }
 
+func getObjectList[T any](clusterNo int, kind, namespace string) (*T, error) {
+	var stdout []byte
+	var err error
+	if namespace == "" {
+		stdout, _, err = kubectl(clusterNo, nil, "get", kind, "-o", "json")
+	} else {
+		stdout, _, err = kubectl(clusterNo, nil, "get", kind, "-n", namespace, "-o", "json")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var objList T
+	if err := json.Unmarshal(stdout, &objList); err != nil {
+		return nil, err
+	}
+
+	return &objList, nil
+}
+
 func changeClusterRole(clusterNo int, newRole string) error {
 	deployName := "mantle-controller"
 	deploy, err := getDeploy(clusterNo, cephClusterNamespace, deployName)
@@ -242,6 +266,89 @@ func changeClusterRole(clusterNo int, newRole string) error {
 	return nil
 }
 
+type objectStorageClient struct {
+	cli        *s3.Client
+	bucketName string
+}
+
+func (c *objectStorageClient) listObjects(ctx context.Context) (*s3.ListObjectsV2Output, error) {
+	return c.cli.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+		Bucket: &c.bucketName,
+	})
+}
+
+func createObjectStorageClient(ctx context.Context) (*objectStorageClient, error) {
+	// Find the endpoint of the object storage from the command-line arguments for mantle-controller.
+	stdout, _, err := kubectl(primaryK8sCluster, nil,
+		"get", "deploy", "-n", cephClusterNamespace, "mantle-controller", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get deploy: %w", err)
+	}
+	var deploy appsv1.Deployment
+	if err := json.Unmarshal(stdout, &deploy); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal deploy: %w", err)
+	}
+	args := deploy.Spec.Template.Spec.Containers[0].Args
+	endpointIndex := slices.IndexFunc(args, func(s string) bool {
+		return strings.HasPrefix(s, "--object-storage-endpoint=")
+	})
+	if endpointIndex == -1 {
+		return nil, errors.New("failed to find object storage endpoint")
+	}
+	objectStorageEndpoint, _ := strings.CutPrefix(args[endpointIndex], "--object-storage-endpoint=")
+
+	// Get the bucket name from the OBC.
+	stdout, _, err = kubectl(secondaryK8sCluster, nil,
+		"get", "obc", "-n", cephClusterNamespace, "export-data", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get obc: %w", err)
+	}
+	var obc struct {
+		Spec struct {
+			BucketName string `json:"bucketName"`
+		} `json:"spec"`
+	}
+	if err := json.Unmarshal(stdout, &obc); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal obc: %w", err)
+	}
+
+	// Get the credentials from the Secret.
+	stdout, _, err = kubectl(secondaryK8sCluster, nil,
+		"get", "secret", "-n", cephClusterNamespace, "export-data", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get export-data secret: %w", err)
+	}
+	var secret corev1.Secret
+	if err := json.Unmarshal(stdout, &secret); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal secret: %w", err)
+	}
+	awsAccessKeyID := secret.Data["AWS_ACCESS_KEY_ID"]
+	awsSecretAccessKey := secret.Data["AWS_SECRET_ACCESS_KEY"]
+
+	// Construct a S3 client.
+	sdkConfig, err := config.LoadDefaultConfig(
+		ctx,
+		config.WithRegion("ceph"),
+		config.WithCredentialsProvider(
+			aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
+				return aws.Credentials{
+					AccessKeyID:     string(awsAccessKeyID),
+					SecretAccessKey: string(awsSecretAccessKey),
+				}, nil
+			}),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load default config: %w", err)
+	}
+	s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) {
+		o.BaseEndpoint = &objectStorageEndpoint
+		o.UsePathStyle = true
+	})
+
+	return &objectStorageClient{cli: s3Client, bucketName: obc.Spec.BucketName}, nil
+}
+
 // IsJobConditionTrue returns true when the conditionType is present and set to
 // `metav1.ConditionTrue`.  Otherwise, it returns false.  Note that we can't use
 // meta.IsStatusConditionTrue because it doesn't accept []JobCondition.

From deff624db3db343a7b568c0824eb021b1e7a4563 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:39:01 +0000
Subject: [PATCH 05/11] test/e2e: multik8s: correct backup if previous full MB
 is removed in secondary cluster

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 6d49ec3d..23ae1a2a 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -289,6 +289,39 @@ func replicationTestSuite() {
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 		})
+
+		It("should back up correctly if previous MB is deleted in the secondary cluster", func(ctx SpecContext) {
+			namespace := util.GetUniqueName("ns-")
+			pvcName := util.GetUniqueName("pvc-")
+			backupName0 := util.GetUniqueName("mb-")
+			backupName1 := util.GetUniqueName("mb-")
+			restoreName0 := util.GetUniqueName("mr-")
+			restoreName1 := util.GetUniqueName("mr-")
+
+			setupEnvironment(namespace, pvcName)
+			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
+
+			// create M0.
+			createMantleBackup(namespace, pvcName, backupName0)
+			waitMantleBackupSynced(namespace, backupName0)
+
+			// remove M0'.
+			_, _, err := kubectl(secondaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName0)
+			Expect(err).NotTo(HaveOccurred())
+
+			// create M1.
+			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName1)
+			waitMantleBackupSynced(namespace, backupName1)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// Make sure M1 and M1' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+
+			// Make sure M0 can be used for restoration.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+		})
 	})
 }
 

From c1a80edeccbaba0fad5d1bed3525d1910afa4439 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:40:37 +0000
Subject: [PATCH 06/11] test/e2e: multik8s: correct backup if previous full MB
 is removed in primary cluster

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 23ae1a2a..8d27b8cc 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -322,6 +322,39 @@ func replicationTestSuite() {
 			// Make sure M0 can be used for restoration.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 		})
+
+		It("should back up correctly if previous MB is deleted in the primary cluster", func(ctx SpecContext) {
+			namespace := util.GetUniqueName("ns-")
+			pvcName := util.GetUniqueName("pvc-")
+			backupName0 := util.GetUniqueName("mb-")
+			backupName1 := util.GetUniqueName("mb-")
+			restoreName0 := util.GetUniqueName("mr-")
+			restoreName1 := util.GetUniqueName("mr-")
+
+			setupEnvironment(namespace, pvcName)
+			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
+
+			// create M0.
+			createMantleBackup(namespace, pvcName, backupName0)
+			waitMantleBackupSynced(namespace, backupName0)
+
+			// remove M0.
+			_, _, err := kubectl(primaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName0)
+			Expect(err).NotTo(HaveOccurred())
+
+			// create M1.
+			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName1)
+			waitMantleBackupSynced(namespace, backupName1)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// Make sure M1 and M1' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+
+			// Make sure M0' can be used for restoration.
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+		})
 	})
 }
 

From 73f9968d1a916ca21ec7ea6b6892387c22751fcc Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:42:20 +0000
Subject: [PATCH 07/11] test/e2e: multik8s: correct incremental backup

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 8d27b8cc..410780c5 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -355,6 +355,37 @@ func replicationTestSuite() {
 			// Make sure M0' can be used for restoration.
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 		})
+
+		It("should perform a correct incremental backup", func(ctx SpecContext) {
+			namespace := util.GetUniqueName("ns-")
+			pvcName := util.GetUniqueName("pvc-")
+			backupName0 := util.GetUniqueName("mb-")
+			backupName1 := util.GetUniqueName("mb-")
+			restoreName0 := util.GetUniqueName("mr-")
+			restoreName1 := util.GetUniqueName("mr-")
+
+			setupEnvironment(namespace, pvcName)
+
+			// create M0.
+			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName0)
+			waitMantleBackupSynced(namespace, backupName0)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// create M1.
+			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName1)
+			waitMantleBackupSynced(namespace, backupName1)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// Make sure M1 and M1' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+
+			// Make sure M0 and M0' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+		})
 	})
 }
 

From d33266e9566fdacb8ebac4dcabd57ec3006092c0 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:43:34 +0000
Subject: [PATCH 08/11] test/e2e: multik8s: correct backup if previous
 incremental MB is removed in secondary cluster

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 51 +++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 410780c5..021f1406 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -386,6 +386,57 @@ func replicationTestSuite() {
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 		})
+
+		It("should back up correctly if previous incremental MB is removed in the secondary cluster", func(ctx SpecContext) {
+			namespace := util.GetUniqueName("ns-")
+			pvcName := util.GetUniqueName("pvc-")
+			backupName0 := util.GetUniqueName("mb-")
+			backupName1 := util.GetUniqueName("mb-")
+			backupName2 := util.GetUniqueName("mb-")
+			restoreName0 := util.GetUniqueName("mr-")
+			restoreName1 := util.GetUniqueName("mr-")
+			restoreName2 := util.GetUniqueName("mr-")
+
+			setupEnvironment(namespace, pvcName)
+
+			// create M0.
+			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName0)
+			waitMantleBackupSynced(namespace, backupName0)
+
+			// create M1.
+			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName1)
+			waitMantleBackupSynced(namespace, backupName1)
+
+			// remove M1'.
+			_, _, err := kubectl(secondaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName1)
+			Expect(err).NotTo(HaveOccurred())
+
+			// create M2.
+			writtenDataHash2 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName2)
+			waitMantleBackupSynced(namespace, backupName2)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// Make sure M2 and M2' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
+
+			// Make sure M1 has the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+
+			// Make sure M0 and M0' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+
+			// Make sure M1' isn't re-created.
+			mbList, err := getObjectList[mantlev1.MantleBackupList](secondaryK8sCluster, "mb", namespace)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(slices.ContainsFunc(mbList.Items, func(mb mantlev1.MantleBackup) bool {
+				return mb.GetName() == backupName1
+			})).To(BeFalse())
+		})
 	})
 }
 

From ab74763b3ee10ad79e5e0565258589cbb17a68e9 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Wed, 25 Dec 2024 02:44:04 +0000
Subject: [PATCH 09/11] test/e2e: multik8s: correct backup if previous
 incremental MB is removed only in the primary cluster

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 44 +++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 021f1406..dec6f94e 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -437,6 +437,50 @@ func replicationTestSuite() {
 				return mb.GetName() == backupName1
 			})).To(BeFalse())
 		})
+
+		It("should back up correctly if previous incremental MB is removed in the primary cluster", func(ctx SpecContext) {
+			namespace := util.GetUniqueName("ns-")
+			pvcName := util.GetUniqueName("pvc-")
+			backupName0 := util.GetUniqueName("mb-")
+			backupName1 := util.GetUniqueName("mb-")
+			backupName2 := util.GetUniqueName("mb-")
+			restoreName0 := util.GetUniqueName("mr-")
+			restoreName1 := util.GetUniqueName("mr-")
+			restoreName2 := util.GetUniqueName("mr-")
+
+			setupEnvironment(namespace, pvcName)
+
+			// create M0.
+			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName0)
+			waitMantleBackupSynced(namespace, backupName0)
+
+			// create M1.
+			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName1)
+			waitMantleBackupSynced(namespace, backupName1)
+
+			// remove M1.
+			_, _, err := kubectl(primaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName1)
+			Expect(err).NotTo(HaveOccurred())
+
+			// create M2.
+			writtenDataHash2 := writeRandomDataToPV(ctx, namespace, pvcName)
+			createMantleBackup(namespace, pvcName, backupName2)
+			waitMantleBackupSynced(namespace, backupName2)
+			ensureTemporaryResourcesRemoved(ctx)
+
+			// Make sure M2 and M2' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
+
+			// Make sure M1' has the same contents.
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
+
+			// Make sure M0 and M0' have the same contents.
+			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
+		})
 	})
 }
 

From 7ad40e7269a5669bc983fac866d40c743fadfcb3 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Tue, 7 Jan 2025 02:01:08 +0000
Subject: [PATCH 10/11] test/e2e: multik8s: rename to
 ensureTemporaryResourcesDeleted

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index dec6f94e..c7dacd83 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -118,7 +118,7 @@ func waitMantleBackupSynced(namespace, backupName string) {
 	}, "10m", "1s").Should(Succeed())
 }
 
-func ensureTemporaryResourcesRemoved(ctx context.Context) {
+func ensureTemporaryResourcesDeleted(ctx context.Context) {
 	GinkgoHelper()
 	By("checking all temporary Jobs related to export and import of RBD images are removed")
 	primaryJobList, err := getObjectList[batchv1.JobList](primaryK8sCluster, "job", cephClusterNamespace)
@@ -285,7 +285,7 @@ func replicationTestSuite() {
 				return nil
 			}).Should(Succeed())
 
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
 		})
@@ -313,7 +313,7 @@ func replicationTestSuite() {
 			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName1)
 			waitMantleBackupSynced(namespace, backupName1)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// Make sure M1 and M1' have the same contents.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
@@ -346,7 +346,7 @@ func replicationTestSuite() {
 			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName1)
 			waitMantleBackupSynced(namespace, backupName1)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// Make sure M1 and M1' have the same contents.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
@@ -370,13 +370,13 @@ func replicationTestSuite() {
 			writtenDataHash0 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName0)
 			waitMantleBackupSynced(namespace, backupName0)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// create M1.
 			writtenDataHash1 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName1)
 			waitMantleBackupSynced(namespace, backupName1)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// Make sure M1 and M1' have the same contents.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
@@ -417,7 +417,7 @@ func replicationTestSuite() {
 			writtenDataHash2 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName2)
 			waitMantleBackupSynced(namespace, backupName2)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// Make sure M2 and M2' have the same contents.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
@@ -468,7 +468,7 @@ func replicationTestSuite() {
 			writtenDataHash2 := writeRandomDataToPV(ctx, namespace, pvcName)
 			createMantleBackup(namespace, pvcName, backupName2)
 			waitMantleBackupSynced(namespace, backupName2)
-			ensureTemporaryResourcesRemoved(ctx)
+			ensureTemporaryResourcesDeleted(ctx)
 
 			// Make sure M2 and M2' have the same contents.
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)

From 2f520442cd43ff2b682302596bce6c65fe83ef84 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Tue, 7 Jan 2025 02:27:49 +0000
Subject: [PATCH 11/11] add new constants for resource prefixes

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 .../controller/mantlebackup_controller.go     | 22 +++++++++++++------
 .../mantlebackup_controller_test.go           |  6 ++---
 test/e2e/multik8s/suite_test.go               | 14 ++++++------
 test/e2e/singlek8s/backup_test.go             | 15 +++++++------
 4 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go
index 8080e1be..c1dde3b8 100644
--- a/internal/controller/mantlebackup_controller.go
+++ b/internal/controller/mantlebackup_controller.go
@@ -52,6 +52,14 @@ const (
 	annotRetainIfExpired          = "mantle.cybozu.io/retain-if-expired"
 	annotSyncMode                 = "mantle.cybozu.io/sync-mode"
 
+	MantleExportJobPrefix     = "mantle-export-"
+	MantleUploadJobPrefix     = "mantle-upload-"
+	MantleExportDataPVCPrefix = "mantle-export-"
+	MantleImportJobPrefix     = "mantle-import-"
+	MantleDiscardJobPrefix    = "mantle-discard-"
+	MantleDiscardPVCPrefix    = "mantle-discard-"
+	MantleDiscardPVPrefix     = "mantle-discard-"
+
 	syncModeFull        = "full"
 	syncModeIncremental = "incremental"
 )
@@ -1060,15 +1068,15 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataPVC(ctx context.Context
 }
 
 func makeExportJobName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-export-%s", target.GetUID())
+	return MantleExportJobPrefix + string(target.GetUID())
 }
 
 func makeUploadJobName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-upload-%s", target.GetUID())
+	return MantleUploadJobPrefix + string(target.GetUID())
 }
 
 func makeExportDataPVCName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-export-%s", target.GetUID())
+	return MantleExportDataPVCPrefix + string(target.GetUID())
 }
 
 func makeObjectNameOfExportedData(name, uid string) string {
@@ -1076,19 +1084,19 @@ func makeObjectNameOfExportedData(name, uid string) string {
 }
 
 func makeImportJobName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-import-%s", target.GetUID())
+	return MantleImportJobPrefix + string(target.GetUID())
 }
 
 func makeDiscardJobName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-discard-%s", target.GetUID())
+	return MantleDiscardJobPrefix + string(target.GetUID())
 }
 
 func makeDiscardPVCName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-discard-%s", target.GetUID())
+	return MantleDiscardPVCPrefix + string(target.GetUID())
 }
 
 func makeDiscardPVName(target *mantlev1.MantleBackup) string {
-	return fmt.Sprintf("mantle-discard-%s", target.GetUID())
+	return MantleDiscardPVPrefix + string(target.GetUID())
 }
 
 func (r *MantleBackupReconciler) createOrUpdateExportJob(ctx context.Context, target *mantlev1.MantleBackup, sourceBackupNamePtr *string) error {
diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go
index 3a235c13..624c2280 100644
--- a/internal/controller/mantlebackup_controller_test.go
+++ b/internal/controller/mantlebackup_controller_test.go
@@ -532,7 +532,7 @@ var _ = Describe("MantleBackup controller", func() {
 				err = k8sClient.Get(
 					ctx,
 					types.NamespacedName{
-						Name:      fmt.Sprintf("mantle-upload-%s", backup.GetUID()),
+						Name:      makeUploadJobName(backup),
 						Namespace: resMgr.ClusterID,
 					},
 					&jobUpload,
@@ -551,7 +551,7 @@ var _ = Describe("MantleBackup controller", func() {
 				err = k8sClient.Get(
 					ctx,
 					types.NamespacedName{
-						Name:      fmt.Sprintf("mantle-upload-%s", backup.GetUID()),
+						Name:      makeUploadJobName(backup),
 						Namespace: resMgr.ClusterID,
 					},
 					&jobUpload,
@@ -622,7 +622,7 @@ var _ = Describe("MantleBackup controller", func() {
 				err = k8sClient.Get(
 					ctx,
 					types.NamespacedName{
-						Name:      fmt.Sprintf("mantle-export-%s", backup2.GetUID()),
+						Name:      makeExportJobName(backup2),
 						Namespace: resMgr.ClusterID,
 					},
 					&jobExport2,
diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index c7dacd83..da6f350c 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -125,15 +125,15 @@ func ensureTemporaryResourcesDeleted(ctx context.Context) {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(slices.ContainsFunc(primaryJobList.Items, func(job batchv1.Job) bool {
 		n := job.GetName()
-		return strings.HasPrefix(n, "mantle-export-") ||
-			strings.HasPrefix(n, "mantle-upload-")
+		return strings.HasPrefix(n, controller.MantleExportJobPrefix) ||
+			strings.HasPrefix(n, controller.MantleUploadJobPrefix)
 	})).To(BeFalse())
 	secondaryJobList, err := getObjectList[batchv1.JobList](secondaryK8sCluster, "job", cephClusterNamespace)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(slices.ContainsFunc(secondaryJobList.Items, func(job batchv1.Job) bool {
 		n := job.GetName()
-		return strings.HasPrefix(n, "mantle-import-") ||
-			strings.HasPrefix(n, "mantle-discard-")
+		return strings.HasPrefix(n, controller.MantleImportJobPrefix) ||
+			strings.HasPrefix(n, controller.MantleDiscardJobPrefix)
 	})).To(BeFalse())
 
 	By("checking all temporary PVCs related to export and import of RBD images are removed")
@@ -142,14 +142,14 @@ func ensureTemporaryResourcesDeleted(ctx context.Context) {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(slices.ContainsFunc(primaryPVCList.Items, func(pvc corev1.PersistentVolumeClaim) bool {
 		n := pvc.GetName()
-		return strings.HasPrefix(n, "mantle-export-")
+		return strings.HasPrefix(n, controller.MantleExportDataPVCPrefix)
 	})).To(BeFalse())
 	secondaryPVCList, err := getObjectList[corev1.PersistentVolumeClaimList](
 		secondaryK8sCluster, "pvc", cephClusterNamespace)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(slices.ContainsFunc(secondaryPVCList.Items, func(pvc corev1.PersistentVolumeClaim) bool {
 		n := pvc.GetName()
-		return strings.HasPrefix(n, "mantle-discard-")
+		return strings.HasPrefix(n, controller.MantleDiscardPVCPrefix)
 	})).To(BeFalse())
 
 	By("checking all temporary PVs related to export and import of RBD images are removed")
@@ -157,7 +157,7 @@ func ensureTemporaryResourcesDeleted(ctx context.Context) {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(slices.ContainsFunc(secondaryPVList.Items, func(pv corev1.PersistentVolume) bool {
 		n := pv.GetName()
-		return strings.HasPrefix(n, "mantle-discard-")
+		return strings.HasPrefix(n, controller.MantleDiscardPVPrefix)
 	})).To(BeFalse())
 
 	By("checking all temporary objects in the object storage related to export and import of RBD images are removed")
diff --git a/test/e2e/singlek8s/backup_test.go b/test/e2e/singlek8s/backup_test.go
index b2d4c3bf..18eccf95 100644
--- a/test/e2e/singlek8s/backup_test.go
+++ b/test/e2e/singlek8s/backup_test.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	"github.com/cybozu-go/mantle/internal/controller"
 	"github.com/cybozu-go/mantle/test/util"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -176,10 +177,10 @@ func (test *backupTest) testCase1() {
 		err = json.Unmarshal(stdout, &jobs)
 		Expect(err).NotTo(HaveOccurred())
 		for _, job := range jobs.Items {
-			Expect(strings.HasPrefix(job.GetName(), "mantle-export-")).To(BeFalse())
-			Expect(strings.HasPrefix(job.GetName(), "mantle-upload-")).To(BeFalse())
-			Expect(strings.HasPrefix(job.GetName(), "mantle-import-")).To(BeFalse())
-			Expect(strings.HasPrefix(job.GetName(), "mantle-discard-")).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), controller.MantleExportJobPrefix)).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), controller.MantleUploadJobPrefix)).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), controller.MantleImportJobPrefix)).To(BeFalse())
+			Expect(strings.HasPrefix(job.GetName(), controller.MantleDiscardJobPrefix)).To(BeFalse())
 		}
 
 		// Check that export and discard PVC are not created.
@@ -189,8 +190,8 @@ func (test *backupTest) testCase1() {
 		err = json.Unmarshal(stdout, &pvcs)
 		Expect(err).NotTo(HaveOccurred())
 		for _, pvc := range pvcs.Items {
-			Expect(strings.HasPrefix(pvc.GetName(), "mantle-export-")).To(BeFalse())
-			Expect(strings.HasPrefix(pvc.GetName(), "mantle-discard-")).To(BeFalse())
+			Expect(strings.HasPrefix(pvc.GetName(), controller.MantleExportDataPVCPrefix)).To(BeFalse())
+			Expect(strings.HasPrefix(pvc.GetName(), controller.MantleDiscardPVCPrefix)).To(BeFalse())
 		}
 
 		// Check that discard PV is not created.
@@ -200,7 +201,7 @@ func (test *backupTest) testCase1() {
 		err = json.Unmarshal(stdout, &pvs)
 		Expect(err).NotTo(HaveOccurred())
 		for _, pv := range pvs.Items {
-			Expect(strings.HasPrefix(pv.GetName(), "mantle-discard-")).To(BeFalse())
+			Expect(strings.HasPrefix(pv.GetName(), controller.MantleDiscardPVPrefix)).To(BeFalse())
 		}
 	})