e2e: cephfs rados namespace test

Signed-off-by: Praveen M <[email protected]>
ceph · Oct 1, 2024 · 459bca1 · 459bca1
1 parent 995ef49
commit 459bca1
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 58 deletions.
diff --git a/e2e/cephfs.go b/e2e/cephfs.go
@@ -298,55 +298,68 @@ var _ = Describe(cephfsType, func() {
 				framework.Failf("failed getting cephFS metadata pool name: %v", getErr)
 			}
 
-			By("checking provisioner deployment is running", func() {
-				err := waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
+			By("verify rados objects are within a namespace", func() {
+				framework.Logf("updating configmap with rados namespace %s", radosNamespace)
+				radosNamespace = "cephfs-ns"
+				err := createRadosNamespace(f)
 				if err != nil {
-					framework.Failf("timeout waiting for deployment %s: %v", cephFSDeploymentName, err)
+					framework.Failf("failed to create rados namespace: %v", err)
 				}
-			})
-
-			By("checking nodeplugin daemonset pods are running", func() {
-				err := waitForDaemonSets(cephFSDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err = createConfigMap(cephFSDirPath, f.ClientSet, f)
 				if err != nil {
-					framework.Failf("timeout waiting for daemonset %s: %v", cephFSDeamonSetName, err)
+					framework.Failf("failed to create configmap: %v", err)
 				}
-			})
 
-			// test only if ceph-csi is deployed via helm
-			if helmTest {
-				By("verify PVC and app binding on helm installation", func() {
-					err := validatePVCAndAppBinding(pvcPath, appPath, f)
-					if err != nil {
-						framework.Failf("failed to validate CephFS pvc and application binding: %v", err)
-					}
-					//  Deleting the storageclass and secret created by helm
-					err = deleteResource(cephFSExamplePath + "storageclass.yaml")
-					if err != nil {
-						framework.Failf("failed to delete CephFS storageclass: %v", err)
-					}
-					err = deleteResource(cephFSExamplePath + "secret.yaml")
-					if err != nil {
-						framework.Failf("failed to delete CephFS storageclass: %v", err)
-					}
-				})
-			}
+				// delete csi pods
+				err = deletePodWithLabel("app in (ceph-csi-cephfs, csi-cephfsplugin, csi-cephfsplugin-provisioner)",
+					cephCSINamespace, false)
+				if err != nil {
+					framework.Failf("failed to delete pods with labels: %v", err)
+				}
+				// wait for csi pods to come up
+				err = waitForDaemonSets(cephFSDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					framework.Failf("timeout waiting for daemonset pods: %v", err)
+				}
+				err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to delete CephFS storageclass: %v", err)
+				}
 
-			By("verify mountOptions support", func() {
-				err := createCephfsStorageClass(f.ClientSet, f, true, nil)
+				// create a PVC and bind it to an app
+				err = createCephfsStorageClass(f.ClientSet, f, true, nil)
 				if err != nil {
 					framework.Failf("failed to create CephFS storageclass: %v", err)
 				}
-
-				err = verifySeLinuxMountOption(f, pvcPath, appPath,
-					cephFSDeamonSetName, cephFSContainerName, cephCSINamespace)
+				pvc, pod, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
 				if err != nil {
-					framework.Failf("failed to verify mount options: %v", err)
+					framework.Failf("failed to validate CephFS pvc and application binding: %v", err)
 				}
 
+				validateOmapCount(f, 1, cephfsType, metadataPool, volumesType)
+
+				// delete resources
+				err = deletePod(pod.Name, pod.Namespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to delete application: %v", err)
+				}
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to delete PVC: %v", err)
+				}
 				err = deleteResource(cephFSExamplePath + "storageclass.yaml")
 				if err != nil {
 					framework.Failf("failed to delete CephFS storageclass: %v", err)
 				}
+
+				validateOmapCount(f, 1, cephfsType, metadataPool, volumesType)
+
+				// reset radosNamespace
+				radosNamespace = ""
+				err = createConfigMap(cephFSDirPath, f.ClientSet, f)
+				if err != nil {
+					framework.Failf("failed to create configmap: %v", err)
+				}
 			})
 
 			By("validate fuseMountOptions", func() {

diff --git a/e2e/cephfs_helper.go b/e2e/cephfs_helper.go
@@ -187,6 +187,15 @@ func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeC
 	return nil
 }
 
+func cephfsOptions(pool string) string {
+	if radosNamespace != "" {
+		return "--pool=" + pool + " --namespace=" + radosNamespace
+	}
+
+	// default namespace is csi
+	return "--pool=" + pool + " --namespace=csi" 
+}
+
 type cephfsSubVolume struct {
 	Name string `json:"name"`
 }

diff --git a/e2e/configmap.go b/e2e/configmap.go
@@ -60,6 +60,9 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra
 		RBD: cephcsi.RBD{
 			RadosNamespace: radosNamespace,
 		},
+		CephFS: cephcsi.CephFS{
+			RadosNamespace: radosNamespace,
+		},
 		ReadAffinity: cephcsi.ReadAffinity{
 			Enabled: true,
 			CrushLocationLabels: []string{

diff --git a/e2e/utils.go b/e2e/utils.go
@@ -172,15 +172,12 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 
 	radosListCommands := []radosListCommand{
 		{
-			volumeMode: volumesType,
-			driverType: cephfsType,
-			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
-			radosLsCmdFilter: fmt.Sprintf(
-				"rados ls --pool=%s --namespace csi | grep -v default | grep -v csi.volume.group. | grep -c ^csi.volume.",
-				pool),
-			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi", pool),
-			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi | wc -l",
-				pool),
+			volumeMode:           volumesType,
+			driverType:           cephfsType,
+			radosLsCmd:           "rados ls " + cephfsOptions(pool),
+			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.volume.", cephfsOptions(pool)),
+			radosLsKeysCmd:       "rados listomapkeys csi.volumes.default " + cephfsOptions(pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default %s | wc -l", cephfsOptions(pool)),
 		},
 		{
 			volumeMode: volumesType,
@@ -193,14 +190,12 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default %s | wc -l", rbdOptions(pool)),
 		},
 		{
-			volumeMode: snapsType,
-			driverType: cephfsType,
-			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
-			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.snap.",
-				pool),
-			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi", pool),
-			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi | wc -l",
-				pool),
+			volumeMode:           snapsType,
+			driverType:           cephfsType,
+			radosLsCmd:           "rados ls " + cephfsOptions(pool),
+			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.snap.", cephfsOptions(pool)),
+			radosLsKeysCmd:       "rados listomapkeys csi.snaps.default " + cephfsOptions(pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default %s | wc -l", cephfsOptions(pool)),
 		},
 		{
 			volumeMode:           snapsType,
@@ -211,14 +206,12 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default %s | wc -l", rbdOptions(pool)),
 		},
 		{
-			volumeMode: groupSnapsType,
-			driverType: cephfsType,
-			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
-			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.volume.group.",
-				pool),
-			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.groups.default --pool=%s --namespace csi", pool),
-			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.groups.default --pool=%s --namespace csi | wc -l",
-				pool),
+			volumeMode:           groupSnapsType,
+			driverType:           cephfsType,
+			radosLsCmd:           "rados ls" + cephfsOptions(pool),
+			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.volume.group.", cephfsOptions(pool)),
+			radosLsKeysCmd:       "rados listomapkeys csi.groups.default " + cephfsOptions(pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.groups.default %s | wc -l", cephfsOptions(pool)),
 		},
 	}