deploy: use allowedTopologies in storage classes, II

The previous attempt with a fixed label did not work when downgrading
to 0.8.0:

pmem-csi-controller-0/[email protected]: I0118 11:38:34.703703       1 tracing.go:19] GRPC call: /csi.v1.Controller/CreateVolume
pmem-csi-controller-0/[email protected]: I0118 11:38:34.703770       1 tracing.go:20] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"pmem-csi.intel.com/driver":"started"}}],"requisite":[{"segments":{"pmem-csi.intel.com/driver":"started"}}]},"capacity_range":{"required_bytes":115343360},"name":"pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0","parameters":{"eraseafter":"true"},"volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
pmem-csi-controller-0/[email protected]: I0118 11:38:34.704225       1 controllerserver-master.go:181] Controller CreateVolume: Name:pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0 required_bytes:115343360 limit_bytes:0
pmem-csi-controller-0/[email protected]: I0118 11:38:34.704251       1 controllerserver-master.go:86] Controller CreateVolume: Create VolumeID:pvc-26-328dad453a5df7f8348d57f9f7422e6b084a58303f5400af6f92906a based on name:pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0
pmem-csi-controller-0/[email protected]: W0118 11:38:34.704273       1 controllerserver-master.go:236] failed to connect to : No node registered with id:

This attempt uses the node selector label for the PMEM-CSI node pod in
the allowedTopologies field. Unfortunately, it also doesn't work:

pmem-csi-intel-com-node-q94vv/[email protected]: I0118 14:46:54.540845       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"skew-4425", Name:"direct-testingv4nl9", UID:"fb94e3a9-62e5-44ec-b48b-33471452142c", APIVersion:"v1", ResourceVersion:"1204", FieldPath:""}): type: 'Warning' reason: 'ProvisioningFailed' failed to provision volume with StorageClass "pmem-pmem-csi-sc-ext4-skew-4425-unused": error generating accessibility requirements: selected node '"pmem-csi-pmem-govm-worker2"' topology 'map[pmem-csi.intel.com/node:pmem-csi-pmem-govm-worker2]' is not in allowed topologies: [map[feature.node.kubernetes.io/memory-nv.dax:true]]

deploy/common/pmem-storageclass-cache.yaml

@@ -2,9 +2,9 @@
 
 allowedTopologies:
 - matchLabelExpressions:
-  - key: pmem-csi.intel.com/driver
+  - key: storage
     values:
-    - started
+    - pmem
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

deploy/common/pmem-storageclass-default.yaml

@@ -2,9 +2,9 @@
 
 allowedTopologies:
 - matchLabelExpressions:
-  - key: pmem-csi.intel.com/driver
+  - key: storage
     values:
-    - started
+    - pmem
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

deploy/common/pmem-storageclass-ext4.yaml

@@ -2,9 +2,9 @@
 
 allowedTopologies:
 - matchLabelExpressions:
-  - key: pmem-csi.intel.com/driver
+  - key: storage
     values:
-    - started
+    - pmem
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

deploy/common/pmem-storageclass-late-binding.yaml

@@ -2,9 +2,9 @@
 
 allowedTopologies:
 - matchLabelExpressions:
-  - key: pmem-csi.intel.com/driver
+  - key: storage
     values:
-    - started
+    - pmem
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

deploy/common/pmem-storageclass-xfs.yaml

@@ -2,9 +2,9 @@
 
 allowedTopologies:
 - matchLabelExpressions:
-  - key: pmem-csi.intel.com/driver
+  - key: storage
     values:
-    - started
+    - pmem
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

deploy/kustomize/storageclass/pmem-storageclass.yaml

@@ -7,8 +7,8 @@ provisioner: pmem-csi.intel.com
 # with late binding only get scheduled to pmem
 allowedTopologies:
 - matchLabelExpressions:
-  # The "pmem-csi.intel.com" part is the driver name.
-  # It must match the "provisioner" field above.
-  - key: pmem-csi.intel.com/driver
+  # This must be the same label that is also used for
+  # scheduling PMEM-CSI node pods.
+  - key: storage
     values:
-    - started
+    - pmem

pkg/pmem-csi-driver/nodeserver.go

@@ -93,12 +93,7 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque
 		NodeId: ns.cs.nodeID,
 		AccessibleTopology: &csi.Topology{
 			Segments: map[string]string{
-				// This part varies by node.
 				DriverTopologyKey: ns.cs.nodeID,
-				// This part is the same on all nodes. It is used
-				// by storage class label selectors to choose
-				// nodes which have the driver.
-				DriverStartedKey: DriverStartedValue,
 			},
 		},
 	}, nil

pkg/pmem-csi-driver/pmem-csi-driver.go

@@ -70,20 +70,12 @@ const (
 	Node DriverMode = "node"
 	// Just the webhooks, using metrics instead of gRPC over TCP.
 	Webhooks DriverMode = "webhooks"
-
-	// DriverStartedValue is the fixed key used for DriverStartedKey.
-	DriverStartedValue = "started"
 )
 
 var (
-	// DriverTopologyKey is the key to use for per-node topology constraints.
+	//PmemDriverTopologyKey key to use for topology constraint
 	DriverTopologyKey = ""
 
-	// DriverStartedKey is an additional key with the same value on each node.
-	// It's useful for selecting nodes with PMEM-CSI in cases (like storage
-	// classes) where a value is required because "key exists" is not valid.
-	DriverStartedKey = ""
-
 	// Mirrored after https://github.com/kubernetes/component-base/blob/dae26a37dccb958eac96bc9dedcecf0eb0690f0f/metrics/version.go#L21-L37
 	// just with less information.
 	buildInfo = prometheus.NewGaugeVec(
@@ -160,7 +152,6 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) {
 	}
 
 	DriverTopologyKey = cfg.DriverName + "/node"
-	DriverStartedKey = cfg.DriverName + "/driver"
 
 	// Should GetCSIDriver get called more than once per process,
 	// all of them will record their version.

test/e2e/driver/driver.go

@@ -31,6 +31,8 @@ import (
 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 
+	"github.com/intel/pmem-csi/test/test-config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -136,7 +138,13 @@ func (m *manifestDriver) GetDynamicProvisionStorageClass(config *testsuites.PerT
 	Expect(ok).To(BeTrue(), "storage class from %s", scManifest)
 	sc.Provisioner = m.csiDriverName
 	sc.Name = config.Prefix + "-" + sc.Name
-	sc.AllowedTopologies[0].MatchLabelExpressions[0].Key = m.csiDriverName + "/driver"
+	nodeLabel := testconfig.GetOrFail("TEST_PMEM_NODE_LABEL")
+	parts := strings.SplitN(nodeLabel, "=", 2)
+	if len(parts) < 2 {
+		panic(fmt.Sprintf("expected label=value: TEST_PMEM_NODE_LABEL=%q", nodeLabel))
+	}
+	sc.AllowedTopologies[0].MatchLabelExpressions[0].Key = parts[0]
+	sc.AllowedTopologies[0].MatchLabelExpressions[0].Values[0] = parts[1]
 
 	// Add additional parameters, if any.
 	for name, value := range m.parameters {

test/setup-deployment.sh

@@ -69,7 +69,17 @@ for deploy in ${DEPLOY[@]}; do
         paths+=" $path"
     fi
     if [ -f "$path" ]; then
-        ${KUBECTL} apply -f - <"$path"
+        case "$path" in
+            *storageclass*)
+                # Patch the node selector label into the storage class instead of the default storage=pmem.
+                sed -e "s;: storage\$;: \"$(echo $TEST_PMEM_NODE_LABEL | cut -d= -f1)\";" \
+                    -e "s;- pmem\$;- \"$(echo $TEST_PMEM_NODE_LABEL | cut -d= -f2)\";" \
+                    "$path" | ${KUBECTL} apply -f -
+                ;;
+            *)
+                ${KUBECTL} apply -f - <"$path"
+                ;;
+            esac
     elif [ -d "$path" ]; then
         # A kustomize base. We need to copy all files over into the cluster, otherwise
         # `kubectl kustomize` won't work.