diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 06d00a40..ed02bbb0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -42,5 +42,5 @@ jobs: key: go-${{ hashFiles('go.sum', 'Makefile', 'versions.mk') }} restore-keys: | go- - - run: make -C e2e setup - - run: make -C e2e test + - run: make -C test/e2e setup + - run: make -C test/e2e test diff --git a/e2e/suite_test.go b/e2e/suite_test.go deleted file mode 100644 index 17b54457..00000000 --- a/e2e/suite_test.go +++ /dev/null @@ -1,459 +0,0 @@ -package e2e - -import ( - "bytes" - _ "embed" - "encoding/json" - "fmt" - "os" - "os/exec" - "strings" - "testing" - "time" - - backupv1 "github.com/cybozu-go/mantle/api/v1" - "github.com/cybozu-go/mantle/internal/controller" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sigs.k8s.io/yaml" -) - -var ( - //go:embed testdata/pvc-template.yaml - testPVCTemplate string - - //go:embed testdata/rbd-pool-sc-template.yaml - testRBDPoolSCTemplate string - - //go:embed testdata/mantlebackup-template.yaml - testMantleBackupTemplate string - - kubectlIsNotFoundMessage = "Error from server (NotFound):" -) - -const ( - pvcName = "rbd-pvc" - pvcName2 = "rbd-pvc-2" - poolName = "replicapool" - mantleBackupName = "mantlebackup-test" - mantleBackupName2 = "mantlebackup-test-2" - mantleBackupName3 = "mantlebackup-test-3" - namespace = "rook-ceph" - namespace2 = "rook-ceph2" - storageClassName = "rook-ceph-block" - storageClassName2 = "rook-ceph-block2" -) - -func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { - var stdout, stderr bytes.Buffer - command := exec.Command(cmd, args...) - command.Stdout = &stdout - command.Stderr = &stderr - - if len(input) != 0 { - command.Stdin = bytes.NewReader(input) - } - - err := command.Run() - return stdout.Bytes(), stderr.Bytes(), err -} - -func kubectl(args ...string) ([]byte, []byte, error) { - return execAtLocal("kubectl", nil, args...) -} - -func kubectlWithInput(input []byte, args ...string) ([]byte, []byte, error) { - return execAtLocal("kubectl", input, args...) -} - -func getImageNameFromPVName(pvName string) (string, error) { - stdout, stderr, err := kubectl("get", "pv", pvName, "-o", "json") - if err != nil { - return "", fmt.Errorf("kubectl get pv failed. stderr: %s, err: %w", string(stderr), err) - } - var pv corev1.PersistentVolume - err = json.Unmarshal(stdout, &pv) - if err != nil { - return "", err - } - imageName := pv.Spec.CSI.VolumeAttributes["imageName"] - return imageName, nil -} - -func TestMtest(t *testing.T) { - if os.Getenv("E2ETEST") == "" { - t.Skip("Run under e2e/") - } - - RegisterFailHandler(Fail) - - SetDefaultEventuallyPollingInterval(time.Second) - SetDefaultEventuallyTimeout(3 * time.Minute) - - RunSpecs(t, "rbd backup system test") -} - -var _ = BeforeSuite(func() { - By("[BeforeSuite] Creating common resources") - Eventually(func() error { - manifest := fmt.Sprintf(testRBDPoolSCTemplate, poolName, namespace, - storageClassName, poolName, namespace, namespace, namespace) - _, _, err := kubectlWithInput([]byte(manifest), "apply", "-n", namespace, "-f", "-") - if err != nil { - return err - } - manifest = fmt.Sprintf(testRBDPoolSCTemplate, poolName, namespace2, - storageClassName2, poolName, namespace2, namespace2, namespace2) - _, _, err = kubectlWithInput([]byte(manifest), "apply", "-n", namespace2, "-f", "-") - if err != nil { - return err - } - - return nil - }).Should(Succeed()) - - for _, name := range []string{pvcName, pvcName2} { - By(fmt.Sprintf("[BeforeSuite] Creating PVC (%s)", name)) - Eventually(func() error { - manifest := fmt.Sprintf(testPVCTemplate, name) - _, _, err := kubectlWithInput([]byte(manifest), "apply", "-n", namespace, "-f", "-") - if err != nil { - return err - } - - return nil - }).Should(Succeed()) - - By(fmt.Sprintf("[BeforeSuite] Waiting for PVC(%s) to get bound", name)) - pvName := "" - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "pvc", name, "-o", "json") - if err != nil { - return fmt.Errorf("kubectl get pvc failed. stderr: %s, err: %w", string(stderr), err) - } - - var pvc corev1.PersistentVolumeClaim - err = yaml.Unmarshal(stdout, &pvc) - if err != nil { - return err - } - - if pvc.Status.Phase != "Bound" { - return fmt.Errorf("PVC is not bound yet") - } - pvName = pvc.Spec.VolumeName - - return nil - }).Should(Succeed()) - - By(fmt.Sprintf("[BeforeSuite] Create a new RBD image in %s ns with the same name as that of the PVC(%s) in %s ns", - namespace2, name, namespace)) - Eventually(func() error { - // Get the image name - imageName, err := getImageNameFromPVName(pvName) - if err != nil { - return err - } - - // Create a new RBD image in namespace2 with the same name as imageName. - _, stderr, err := kubectl( - "-n", namespace2, "exec", "deploy/rook-ceph-tools", "--", - "rbd", "create", "--size", "100", poolName+"/"+imageName) - if err != nil { - return fmt.Errorf("rbd create failed. stderr: %s, err: %w", string(stderr), err) - } - - return nil - }).Should(Succeed()) - } - - By("[BeforeSuite] Waiting for mantle-controller to get ready") - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "deploy", "mantle-controller", "-o", "json") - if err != nil { - return fmt.Errorf("kubectl get deploy failed. stderr: %s, err: %w", string(stderr), err) - } - - var deploy appsv1.Deployment - err = yaml.Unmarshal(stdout, &deploy) - if err != nil { - return err - } - - if deploy.Status.AvailableReplicas != 1 { - return fmt.Errorf("mantle-controller is not available yet") - } - - return nil - }).Should(Succeed()) -}) - -var _ = AfterSuite(func() { - By("[AfterSuite] Deleting MantleBackups") - for _, mantleBackup := range []string{mantleBackupName, mantleBackupName2, mantleBackupName3} { - _, _, _ = kubectl("delete", "-n", namespace, "mantlebackup", mantleBackup) - } - - By("[AfterSuite] Deleting PVCs") - for _, pvc := range []string{pvcName, pvcName2} { - _, _, _ = kubectl("delete", "-n", namespace, "pvc", pvc) - } - - By("[AfterSuite] Deleting RBD images in " + namespace2) - stdout, _, err := kubectl("exec", "-n", namespace2, "deploy/rook-ceph-tools", "--", - "rbd", "ls", poolName, "--format=json") - if err == nil { - imageNames := []string{} - if err := json.Unmarshal(stdout, &imageNames); err == nil { - for _, imageName := range imageNames { - _, _, _ = kubectl("exec", "-n", namespace2, "deploy/rook-ceph-tools", "--", - "rbd", "rm", poolName+"/"+imageName) - } - } - } - - By("[AfterSuite] Deleting common resources") - _, _, _ = kubectl("delete", "sc", storageClassName, "--wait=false") - _, _, _ = kubectl("delete", "sc", storageClassName2, "--wait=false") - _, _, _ = kubectl("delete", "-n", namespace, "cephblockpool", poolName, "--wait=false") - _, _, _ = kubectl("delete", "-n", namespace2, "cephblockpool", poolName, "--wait=false") -}) - -var _ = Describe("rbd backup system", func() { - var firstImageName string - - testMantleBackupResourceCreation := func(mantleBackupName string, saveImageName bool) { - By("Creating MantleBackup") - manifest := fmt.Sprintf(testMantleBackupTemplate, mantleBackupName, mantleBackupName, namespace, pvcName) - _, _, err := kubectlWithInput([]byte(manifest), "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for RBD snapshot to be created") - imageName := "" - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "pvc", pvcName, "-o", "json") - if err != nil { - return fmt.Errorf("kubectl get pvc failed. stderr: %s, err: %w", string(stderr), err) - } - var pvc corev1.PersistentVolumeClaim - err = yaml.Unmarshal(stdout, &pvc) - if err != nil { - return err - } - pvName := pvc.Spec.VolumeName - - imageName, err = getImageNameFromPVName(pvName) - if err != nil { - return err - } - if saveImageName { - firstImageName = imageName - } - - stdout, stderr, err = kubectl( - "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", - "rbd", "snap", "ls", poolName+"/"+imageName, "--format=json") - if err != nil { - return fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) - } - var snapshots []controller.Snapshot - err = yaml.Unmarshal(stdout, &snapshots) - if err != nil { - return err - } - existSnapshot := false - for _, s := range snapshots { - if s.Name == mantleBackupName { - existSnapshot = true - break - } - } - if !existSnapshot { - return fmt.Errorf("snapshot not exists. snapshotName: %s", mantleBackupName) - } - - return nil - }).Should(Succeed()) - - By("Checking that the mantle-controller deployed for a certain Rook/Ceph cluster (i.e., " + - namespace2 + ") doesn't create a snapshot for a MantleBackup for a different Rook/Ceph cluster (i.e., " + - namespace + ")") - Consistently(func() error { - stdout, stderr, err := kubectl( - "-n", namespace2, "exec", "deploy/rook-ceph-tools", "--", - "rbd", "snap", "ls", poolName+"/"+imageName, "--format=json") - if err != nil { - return fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) - } - var snapshots []controller.Snapshot - err = yaml.Unmarshal(stdout, &snapshots) - if err != nil { - return err - } - existSnapshot := false - for _, s := range snapshots { - if s.Name == mantleBackupName { - existSnapshot = true - break - } - } - if existSnapshot { - return fmt.Errorf("a wrong snapshot exists. snapshotName: %s", mantleBackupName) - } - return nil - }).Should(Succeed()) - } - - It("should create MantleBackup resource", func() { - testMantleBackupResourceCreation(mantleBackupName, true) - }) - - It("should create multiple MantleBackup resources for the same PVC", func() { - testMantleBackupResourceCreation(mantleBackupName2, false) - }) - - It("should create MantleBackups resources for different PVCs", func() { - By("Creating a third MantleBackup for the other PVC") - manifest := fmt.Sprintf(testMantleBackupTemplate, mantleBackupName3, mantleBackupName3, namespace, pvcName2) - _, _, err := kubectlWithInput([]byte(manifest), "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for RBD snapshot to be created") - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "pvc", pvcName2, "-o", "json") - if err != nil { - return fmt.Errorf("kubectl get pvc failed. stderr: %s, err: %w", string(stderr), err) - } - var pvc corev1.PersistentVolumeClaim - err = yaml.Unmarshal(stdout, &pvc) - if err != nil { - return err - } - pvName := pvc.Spec.VolumeName - - imageName, err := getImageNameFromPVName(pvName) - if err != nil { - return err - } - - stdout, stderr, err = kubectl( - "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", - "rbd", "snap", "ls", poolName+"/"+imageName, "--format=json") - if err != nil { - return fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) - } - var snapshots []controller.Snapshot - err = yaml.Unmarshal(stdout, &snapshots) - if err != nil { - return err - } - existSnapshot := false - for _, s := range snapshots { - if s.Name == mantleBackupName3 { - existSnapshot = true - break - } - } - if !existSnapshot { - return fmt.Errorf("snapshot not exists. snapshotName: %s", mantleBackupName3) - } - - return nil - }).Should(Succeed()) - }) - - It("should not delete MantleBackup resource when delete backup target PVC", func() { - By("Deleting backup target PVC") - _, _, err := kubectl("-n", namespace, "delete", "pvc", pvcName2) - Expect(err).NotTo(HaveOccurred()) - - By("Checking backup target PVC deletion") - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "pvc", pvcName2) - if err != nil { - if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { - return nil - } - return fmt.Errorf("get pvc %s failed. stderr: %s, err: %w", pvcName2, string(stderr), err) - } - return fmt.Errorf("PVC %s still exists. stdout: %s", pvcName2, stdout) - }).Should(Succeed()) - - By("Checking that the status.conditions of the MantleBackup resource remain \"Bound\"") - stdout, _, err := kubectl("-n", namespace, "get", "mantlebackup", mantleBackupName3, "-o", "json") - Expect(err).NotTo(HaveOccurred()) - var backup backupv1.MantleBackup - err = yaml.Unmarshal(stdout, &backup) - Expect(err).NotTo(HaveOccurred()) - Expect(meta.FindStatusCondition(backup.Status.Conditions, backupv1.BackupConditionReadyToUse).Status). - To(Equal(metav1.ConditionTrue)) - }) - - It("should delete MantleBackup resource", func() { - By("Delete MantleBackup") - _, _, err := kubectl("-n", namespace, "delete", "mantlebackup", mantleBackupName, "--wait=false") - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for RBD snapshot to be deleted") - Eventually(func() error { - stdout, stderr, err := kubectl( - "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", - "rbd", "snap", "ls", poolName+"/"+firstImageName, "--format=json") - if err != nil { - return fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) - } - var snapshots []controller.Snapshot - err = yaml.Unmarshal(stdout, &snapshots) - if err != nil { - return err - } - existSnapshot := false - for _, s := range snapshots { - if s.Name == mantleBackupName { - existSnapshot = true - break - } - } - if existSnapshot { - return fmt.Errorf("snapshot exists. snapshotName: %s", mantleBackupName) - } - - return nil - }).Should(Succeed()) - - By("Checking MantleBackup resource deletion") - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "mantlebackup", mantleBackupName) - if err != nil { - if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { - return nil - } - return fmt.Errorf("get mantlebackup %s failed. stderr: %s, err: %w", mantleBackupName, string(stderr), err) - } - return fmt.Errorf("MantleBackup resource %s still exists. stdout: %s", mantleBackupName, stdout) - }).Should(Succeed()) - }) - - It("should delete MantleBackup resource when backup target PVC is missing", func() { - By("Deleting MantleBackup resource") - _, _, err := kubectl("-n", namespace, "delete", "mantlebackup", mantleBackupName3) - Expect(err).NotTo(HaveOccurred()) - - By("Checking MantleBackup resource deletion") - Eventually(func() error { - stdout, stderr, err := kubectl("-n", namespace, "get", "mantlebackup", mantleBackupName3) - if err != nil { - if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { - return nil - } - return fmt.Errorf("get mantlebackup %s failed. stderr: %s, err: %w", mantleBackupName3, string(stderr), err) - } - return fmt.Errorf("MantleBackup resource %s still exists. stdout: %s", mantleBackupName3, stdout) - }).Should(Succeed()) - }) -}) diff --git a/go.mod b/go.mod index 1ddecb55..47f8cb75 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -50,6 +51,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.13.0 // indirect @@ -61,7 +63,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.28.3 // indirect k8s.io/component-base v0.28.3 // indirect diff --git a/go.sum b/go.sum index c93be16c..7e5d7e3a 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= diff --git a/e2e/.gitignore b/test/e2e/.gitignore similarity index 100% rename from e2e/.gitignore rename to test/e2e/.gitignore diff --git a/e2e/Makefile b/test/e2e/Makefile similarity index 79% rename from e2e/Makefile rename to test/e2e/Makefile index 7ead5252..e057a52d 100644 --- a/e2e/Makefile +++ b/test/e2e/Makefile @@ -1,4 +1,4 @@ -include ../versions.mk +include ../../versions.mk SHELL := /bin/bash BINDIR := $(shell pwd)/bin @@ -10,8 +10,8 @@ MINIKUBE := $(BINDIR)/minikube HELM := $(BINDIR)/helm KUBECTL := $(BINDIR)/kubectl-$(KUBERNETES_VERSION) MINIKUBE_HOME = $(BINDIR) -NS := rook-ceph -NS2 := rook-ceph2 +CEPH_CLUSTER1_NAMESPACE := rook-ceph +CEPH_CLUSTER2_NAMESPACE := rook-ceph2 NODE_NAME := minikube-worker SUDO := sudo DD := dd @@ -110,9 +110,10 @@ launch-minikube: .PHONY: delete-minikube delete-minikube: + $(SUDO) -E $(MINIKUBE) stop || true $(SUDO) -E $(MINIKUBE) delete || true # Is it really OK to remove these files? - rm -rf /tmp/juju-* + $(SUDO) rm -rf /tmp/juju-* .PHONY: create-loop-dev create-loop-dev: | $(TMPDIR) @@ -142,7 +143,7 @@ wait-deploy-ready: DEPLOY= wait-deploy-ready: is_ok="false"; \ for ((i=0;i<$(TIMEOUT_SECS);i+=$(POLLING_INTERVAL))); do \ - available_replicas=$$($(KUBECTL) -n $(NS) get deploy $(DEPLOY) -o json | jq -r ".status.availableReplicas"); \ + available_replicas=$$($(KUBECTL) -n $(CEPH_CLUSTER1_NAMESPACE) get deploy $(DEPLOY) -o json | jq -r ".status.availableReplicas"); \ if [ "$$available_replicas" = 1 ]; then \ is_ok="true"; \ break; \ @@ -158,7 +159,7 @@ wait-deploy-ready: .PHONY: launch-rook-ceph launch-rook-ceph: create-loop-dev $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ - --create-namespace --namespace $(NS) -f testdata/values.yaml --wait \ + --create-namespace --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values.yaml --wait \ rook-ceph rook-ceph sed \ -e "s%{LOOP_DEV}%$(loop-dev)%" \ @@ -168,42 +169,42 @@ launch-rook-ceph: create-loop-dev > testdata/persistentvolumes.yaml $(KUBECTL) apply -f testdata/persistentvolumes.yaml $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ - --namespace $(NS) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster + --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ - --create-namespace --namespace $(NS2) -f testdata/values-cluster.yaml \ + --create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \ --set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \ --wait rook-ceph-cluster2 rook-ceph-cluster - $(MAKE) wait-deploy-ready NS=$(NS) DEPLOY=rook-ceph-operator - $(MAKE) wait-deploy-ready NS=$(NS) DEPLOY=rook-ceph-osd-0 - $(MAKE) wait-deploy-ready NS=$(NS2) DEPLOY=rook-ceph-osd-0 + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0 + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER2_NAMESPACE) DEPLOY=rook-ceph-osd-0 .PHONY: delete-rook-ceph delete-rook-ceph: - $(HELM) uninstall --namespace $(NS2) rook-ceph-cluster2 --wait || : - $(HELM) uninstall --namespace $(NS) rook-ceph-cluster --wait || : + $(HELM) uninstall --namespace $(CEPH_CLUSTER2_NAMESPACE) rook-ceph-cluster2 --wait || : + $(HELM) uninstall --namespace $(CEPH_CLUSTER1_NAMESPACE) rook-ceph-cluster --wait || : $(SUDO) rm -rf /var/lib/rook $(SUDO) rm -rf /var/lib/rook2 ls /dev/mapper/ceph-* | xargs -I% -- $(SUDO) dmsetup remove % $(SUDO) rm -rf /dev/mapper/ceph-* - $(HELM) uninstall --namespace $(NS) rook-ceph --wait || : + $(HELM) uninstall --namespace $(CEPH_CLUSTER1_NAMESPACE) rook-ceph --wait || : $(KUBECTL) delete -f testdata/persistentvolumes.yaml || : $(MAKE) delete-loop-dev .PHONY: setup-components setup-components: - $(MAKE) -C .. docker-build + $(MAKE) -C ../.. docker-build # We can't use `minikube image load` due to a minikube's problem. # ref. https://github.com/kubernetes/minikube/issues/17785 docker save controller:latest | $(SUDO) ctr -n k8s.io images import - - $(HELM) upgrade --install mantle-cluster-wide ../charts/mantle-cluster-wide/ --wait - $(HELM) upgrade --install --namespace=$(NS) mantle ../charts/mantle/ --wait - $(HELM) upgrade --install --namespace=$(NS2) mantle2 ../charts/mantle/ --wait + $(HELM) upgrade --install mantle-cluster-wide ../../charts/mantle-cluster-wide/ --wait + $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait + $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 ../../charts/mantle/ --wait .PHONY: delete-components delete-components: - $(HELM) uninstall --namespace=$(NS2) mantle2 --wait || true - $(HELM) uninstall --namespace=$(NS) mantle --wait || true + $(HELM) uninstall --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 --wait || true + $(HELM) uninstall --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle --wait || true $(HELM) uninstall mantle-cluster-wide --wait || true .PHONY: do_test diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go new file mode 100644 index 00000000..6bef938e --- /dev/null +++ b/test/e2e/backup_test.go @@ -0,0 +1,262 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "strings" + + backupv1 "github.com/cybozu-go/mantle/api/v1" + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +const ( + kubectlIsNotFoundMessage = "Error from server (NotFound):" +) + +type backupTest struct { + poolName string + storageClassName1 string + storageClassName2 string + tenantNamespace1 string + tenantNamespace2 string + + pvcName1 string + pvcName2 string + mantleBackupName1 string + mantleBackupName2 string + mantleBackupName3 string +} + +func backupTestSuite() { + test := &backupTest{ + poolName: util.GetUniqueName("pool-"), + storageClassName1: util.GetUniqueName("sc-"), + storageClassName2: util.GetUniqueName("sc-"), + tenantNamespace1: util.GetUniqueName("ns-"), + tenantNamespace2: util.GetUniqueName("ns-"), + + pvcName1: "rbd-pvc1", + pvcName2: "rbd-pvc2", + mantleBackupName1: "mantlebackup-test1", + mantleBackupName2: "mantlebackup-test2", + mantleBackupName3: "mantlebackup-test3", + } + + Describe("setup environment", test.setupEnv) + Describe("test case 1", test.testCase1) + Describe("teardown environment", test.teardownEnv) +} + +func (test *backupTest) setupEnv() { + It("creating common resources", func() { + for _, ns := range []string{test.tenantNamespace1, test.tenantNamespace2} { + err := createNamespace(ns) + Expect(err).NotTo(HaveOccurred()) + } + + err := applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName1) + Expect(err).NotTo(HaveOccurred()) + + err = applyRBDPoolAndSCTemplate(cephCluster2Namespace, test.poolName, test.storageClassName2) + Expect(err).NotTo(HaveOccurred()) + + for _, name := range []string{test.pvcName1, test.pvcName2} { + By(fmt.Sprintf("Creating PVC, PV and RBD image (%s)", name)) + err = applyPVCTemplate(test.tenantNamespace1, name, test.storageClassName1) + Expect(err).NotTo(HaveOccurred()) + + pvName, err := getPVFromPVC(test.tenantNamespace1, name) + Expect(err).NotTo(HaveOccurred()) + + imageName, err := getImageNameFromPVName(pvName) + Expect(err).NotTo(HaveOccurred()) + + // Create a new RBD image in cephNamespace2 with the same name as imageName. + Eventually(func() error { + return createRBDImage(cephCluster2Namespace, test.poolName, imageName, 100) + }).Should(Succeed()) + } + }) + + It("waiting for mantle-controller to get ready", func() { + Eventually(func() error { + return checkDeploymentReady(cephCluster1Namespace, "mantle-controller") + }).Should(Succeed()) + }) +} + +func (test *backupTest) teardownEnv() { + It("deleting MantleBackups", func() { + for _, mantleBackup := range []string{test.mantleBackupName1, test.mantleBackupName2, test.mantleBackupName3} { + _, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "mantlebackup", mantleBackup) + } + }) + + It("deleting PVCs", func() { + for _, pvc := range []string{test.pvcName1, test.pvcName2} { + _, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "pvc", pvc) + } + + By("Deleting RBD images in " + cephCluster2Namespace) + stdout, _, err := kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--", + "rbd", "ls", test.poolName, "--format=json") + if err == nil { + imageNames := []string{} + if err := json.Unmarshal(stdout, &imageNames); err == nil { + for _, imageName := range imageNames { + _, _, _ = kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--", + "rbd", "rm", test.poolName+"/"+imageName) + } + } + } + }) + + It("deleting common resources", func() { + _, _, _ = kubectl("delete", "sc", test.storageClassName1, "--wait=false") + _, _, _ = kubectl("delete", "sc", test.storageClassName2, "--wait=false") + _, _, _ = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false") + _, _, _ = kubectl("delete", "-n", cephCluster2Namespace, "cephblockpool", test.poolName, "--wait=false") + }) +} + +func (test *backupTest) testCase1() { + var firstImageName string + + createMantleBackupAndGetImage := func(mantleBackupName string) string { + By("Creating MantleBackup") + err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName1, mantleBackupName) + Expect(err).NotTo(HaveOccurred()) + + pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName1) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for RBD snapshot to be created") + imageName := "" + Eventually(func() error { + imageName, err = getImageNameFromPVName(pvName) + if err != nil { + return err + } + + return checkSnapshotExist(cephCluster1Namespace, test.poolName, imageName, mantleBackupName) + }).Should(Succeed()) + + By("Checking that the mantle-controller deployed for a certain Rook/Ceph cluster (i.e., " + + cephCluster1Namespace + ") doesn't create a snapshot for a MantleBackup for a different Rook/Ceph cluster (i.e., " + + cephCluster2Namespace + ")") + Consistently(func() error { + err := checkSnapshotExist(cephCluster2Namespace, test.poolName, imageName, mantleBackupName) + if err == nil { + return fmt.Errorf("a wrong snapshot exists. snapshotName: %s", mantleBackupName) + } + return nil + }).Should(Succeed()) + + return imageName + } + + It("should create MantleBackup resource", func() { + firstImageName = createMantleBackupAndGetImage(test.mantleBackupName1) + }) + + It("should create multiple MantleBackup resources for the same PVC", func() { + createMantleBackupAndGetImage(test.mantleBackupName2) + }) + + It("should create MantleBackups resources for different PVCs", func() { + By("Creating a third MantleBackup for the other PVC") + err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName2, test.mantleBackupName3) + Expect(err).NotTo(HaveOccurred()) + + pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName2) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for RBD snapshot to be created") + Eventually(func() error { + imageName, err := getImageNameFromPVName(pvName) + if err != nil { + return err + } + + return checkSnapshotExist(cephCluster1Namespace, test.poolName, imageName, test.mantleBackupName3) + }).Should(Succeed()) + }) + + It("should not delete MantleBackup resource when delete backup target PVC", func() { + By("Deleting backup target PVC") + _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "pvc", test.pvcName2) + Expect(err).NotTo(HaveOccurred()) + + By("Checking backup target PVC deletion") + Eventually(func() error { + stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "pvc", test.pvcName2) + if err != nil { + if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { + return nil + } + return fmt.Errorf("get pvc %s failed. stderr: %s, err: %w", test.pvcName2, string(stderr), err) + } + return fmt.Errorf("PVC %s still exists. stdout: %s", test.pvcName2, stdout) + }).Should(Succeed()) + + By("Checking that the status.conditions of the MantleBackup resource remain \"Bound\"") + stdout, _, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3, "-o", "json") + Expect(err).NotTo(HaveOccurred()) + var backup backupv1.MantleBackup + err = yaml.Unmarshal(stdout, &backup) + Expect(err).NotTo(HaveOccurred()) + Expect(meta.FindStatusCondition(backup.Status.Conditions, backupv1.BackupConditionReadyToUse).Status). + To(Equal(metav1.ConditionTrue)) + }) + + It("should delete MantleBackup resource", func() { + By("Delete MantleBackup") + _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName1, "--wait=false") + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for RBD snapshot to be deleted") + Eventually(func() error { + err := checkSnapshotExist(cephCluster1Namespace, test.poolName, firstImageName, test.mantleBackupName1) + if err == nil { + return fmt.Errorf("snapshot exists. snapshotName: %s", test.mantleBackupName1) + } + + return nil + }).Should(Succeed()) + + By("Checking MantleBackup resource deletion") + Eventually(func() error { + stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName1) + if err != nil { + if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { + return nil + } + return fmt.Errorf("get mantlebackup %s failed. stderr: %s, err: %w", test.mantleBackupName1, string(stderr), err) + } + return fmt.Errorf("MantleBackup resource %s still exists. stdout: %s", test.mantleBackupName1, stdout) + }).Should(Succeed()) + }) + + It("should delete MantleBackup resource when backup target PVC is missing", func() { + By("Deleting MantleBackup resource") + _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName3) + Expect(err).NotTo(HaveOccurred()) + + By("Checking MantleBackup resource deletion") + Eventually(func() error { + stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3) + if err != nil { + if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { + return nil + } + return fmt.Errorf("get mantlebackup %s failed. stderr: %s, err: %w", test.mantleBackupName3, string(stderr), err) + } + return fmt.Errorf("MantleBackup resource %s still exists. stdout: %s", test.mantleBackupName3, stdout) + }).Should(Succeed()) + }) +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go new file mode 100644 index 00000000..5bed6e91 --- /dev/null +++ b/test/e2e/suite_test.go @@ -0,0 +1,33 @@ +package e2e + +import ( + _ "embed" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + cephCluster1Namespace = "rook-ceph" + cephCluster2Namespace = "rook-ceph2" +) + +func TestMtest(t *testing.T) { + if os.Getenv("E2ETEST") == "" { + t.Skip("Run under e2e/") + } + + RegisterFailHandler(Fail) + + SetDefaultEventuallyPollingInterval(time.Second) + SetDefaultEventuallyTimeout(3 * time.Minute) + + RunSpecs(t, "rbd backup system test") +} + +var _ = Describe("Mantle", func() { + Context("backup", backupTestSuite) +}) diff --git a/e2e/testdata/mantlebackup-template.yaml b/test/e2e/testdata/mantlebackup-template.yaml similarity index 100% rename from e2e/testdata/mantlebackup-template.yaml rename to test/e2e/testdata/mantlebackup-template.yaml diff --git a/e2e/testdata/persistentvolumes-template.yaml b/test/e2e/testdata/persistentvolumes-template.yaml similarity index 100% rename from e2e/testdata/persistentvolumes-template.yaml rename to test/e2e/testdata/persistentvolumes-template.yaml diff --git a/e2e/testdata/pvc-template.yaml b/test/e2e/testdata/pvc-template.yaml similarity index 81% rename from e2e/testdata/pvc-template.yaml rename to test/e2e/testdata/pvc-template.yaml index f9f7a5f7..0bcdf27e 100644 --- a/e2e/testdata/pvc-template.yaml +++ b/test/e2e/testdata/pvc-template.yaml @@ -9,4 +9,4 @@ spec: resources: requests: storage: 1Gi - storageClassName: rook-ceph-block + storageClassName: %s diff --git a/e2e/testdata/rbd-pool-sc-template.yaml b/test/e2e/testdata/rbd-pool-sc-template.yaml similarity index 100% rename from e2e/testdata/rbd-pool-sc-template.yaml rename to test/e2e/testdata/rbd-pool-sc-template.yaml diff --git a/e2e/testdata/storageclass-test.yaml b/test/e2e/testdata/storageclass-test.yaml similarity index 100% rename from e2e/testdata/storageclass-test.yaml rename to test/e2e/testdata/storageclass-test.yaml diff --git a/e2e/testdata/values-cluster.yaml b/test/e2e/testdata/values-cluster.yaml similarity index 100% rename from e2e/testdata/values-cluster.yaml rename to test/e2e/testdata/values-cluster.yaml diff --git a/e2e/testdata/values.yaml b/test/e2e/testdata/values.yaml similarity index 100% rename from e2e/testdata/values.yaml rename to test/e2e/testdata/values.yaml diff --git a/test/e2e/util.go b/test/e2e/util.go new file mode 100644 index 00000000..8f06378d --- /dev/null +++ b/test/e2e/util.go @@ -0,0 +1,166 @@ +package e2e + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "os" + "os/exec" + "time" + + "github.com/cybozu-go/mantle/internal/controller" + corev1 "k8s.io/api/core/v1" +) + +var ( + //go:embed testdata/pvc-template.yaml + testPVCTemplate string + //go:embed testdata/rbd-pool-sc-template.yaml + testRBDPoolSCTemplate string + //go:embed testdata/mantlebackup-template.yaml + testMantleBackupTemplate string + + kubectlPath = os.Getenv("KUBECTL") +) + +func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { + var stdout, stderr bytes.Buffer + command := exec.Command(cmd, args...) + command.Stdout = &stdout + command.Stderr = &stderr + + if len(input) != 0 { + command.Stdin = bytes.NewReader(input) + } + + err := command.Run() + return stdout.Bytes(), stderr.Bytes(), err +} + +func kubectl(args ...string) ([]byte, []byte, error) { + if len(kubectlPath) == 0 { + panic("KUBECTL environment variable is not set") + } + return execAtLocal(kubectlPath, nil, args...) +} + +func kubectlWithInput(input []byte, args ...string) ([]byte, []byte, error) { + if len(kubectlPath) == 0 { + panic("KUBECTL environment variable is not set") + } + return execAtLocal(kubectlPath, input, args...) +} + +func applyMantleBackupTemplate(namespace, pvcName, backupName string) error { + manifest := fmt.Sprintf(testMantleBackupTemplate, backupName, backupName, namespace, pvcName) + _, _, err := kubectlWithInput([]byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply mantlebackup failed. err: %w", err) + } + return nil +} + +func applyPVCTemplate(namespace, name, storageClassName string) error { + manifest := fmt.Sprintf(testPVCTemplate, name, storageClassName) + _, _, err := kubectlWithInput([]byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply pvc failed. err: %w", err) + } + return nil +} + +func applyRBDPoolAndSCTemplate(namespace, poolName, storageClassName string) error { + manifest := fmt.Sprintf(testRBDPoolSCTemplate, poolName, namespace, + storageClassName, poolName, namespace, namespace, namespace) + _, _, err := kubectlWithInput([]byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return err + } + return nil +} + +func checkDeploymentReady(namespace, name string) error { + _, stderr, err := kubectl("-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m") + if err != nil { + return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func checkSnapshotExist(namespace, poolName, imageName, backupName string) error { + stdout, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "snap", "ls", poolName+"/"+imageName, "--format=json") + if err != nil { + return fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) + } + var snapshots []controller.Snapshot + err = json.Unmarshal(stdout, &snapshots) + if err != nil { + return err + } + + for _, s := range snapshots { + if s.Name == backupName { + return nil + } + } + + return fmt.Errorf("snapshot not exists. snapshotName: %s", backupName) +} + +func createNamespace(name string) error { + _, _, err := kubectl("create", "ns", name) + if err != nil { + return fmt.Errorf("kubectl create ns failed. err: %w", err) + } + return nil +} + +func createRBDImage(namespace, poolName, imageName string, siz uint) error { + _, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "create", "--size", fmt.Sprintf("%d", siz), poolName+"/"+imageName) + if err != nil { + return fmt.Errorf("rbd create failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func getImageNameFromPVName(pvName string) (string, error) { + stdout, stderr, err := kubectl("get", "pv", pvName, "-o", "json") + if err != nil { + return "", fmt.Errorf("kubectl get pv failed. stderr: %s, err: %w", string(stderr), err) + } + var pv corev1.PersistentVolume + err = json.Unmarshal(stdout, &pv) + if err != nil { + return "", err + } + imageName := pv.Spec.CSI.VolumeAttributes["imageName"] + return imageName, nil +} + +func getPVFromPVC(namespace, pvcName string) (string, error) { + for i := 0; i < 60; i++ { + stdout, stderr, err := kubectl("-n", namespace, "get", "pvc", pvcName, "-o", "json") + if err != nil { + return "", fmt.Errorf("kubectl get pvc failed. stderr: %s, err: %w", string(stderr), err) + } + + var pvc corev1.PersistentVolumeClaim + err = json.Unmarshal(stdout, &pvc) + if err != nil { + return "", fmt.Errorf("yaml unmarshal failed. err: %w", err) + } + + if pvc.Status.Phase == "Bound" { + return pvc.Spec.VolumeName, nil + } + + time.Sleep(5 * time.Second) + } + + return "", fmt.Errorf("timeout to get pv from pvc. namespace: %s, pvcName: %s", namespace, pvcName) +} diff --git a/test/util/util.go b/test/util/util.go new file mode 100644 index 00000000..a6529cf9 --- /dev/null +++ b/test/util/util.go @@ -0,0 +1,24 @@ +package util + +import ( + "fmt" + "math/rand" +) + +var ( + usedResourceNames = make(map[string]bool) +) + +func GetUniqueName(prefix string) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + buf := make([]byte, 8) + for i := range buf { + buf[i] = letters[rand.Intn(len(letters))] + } + name := fmt.Sprintf("%s%s", prefix, string(buf)) + if usedResourceNames[name] { + return GetUniqueName(prefix) + } + usedResourceNames[name] = true + return name +}