From 7c4ee8639eb7729a6ea455a1d1dcbca0af26d5de Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 18 Jan 2021 15:53:59 +0100 Subject: [PATCH] deploy: use allowedTopologies in storage classes, II The previous attempt with a fixed label did not work when downgrading to 0.8.0: pmem-csi-controller-0/pmem-driver@pmem..ker1: I0118 11:38:34.703703 1 tracing.go:19] GRPC call: /csi.v1.Controller/CreateVolume pmem-csi-controller-0/pmem-driver@pmem..ker1: I0118 11:38:34.703770 1 tracing.go:20] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"pmem-csi.intel.com/driver":"started"}}],"requisite":[{"segments":{"pmem-csi.intel.com/driver":"started"}}]},"capacity_range":{"required_bytes":115343360},"name":"pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0","parameters":{"eraseafter":"true"},"volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]} pmem-csi-controller-0/pmem-driver@pmem..ker1: I0118 11:38:34.704225 1 controllerserver-master.go:181] Controller CreateVolume: Name:pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0 required_bytes:115343360 limit_bytes:0 pmem-csi-controller-0/pmem-driver@pmem..ker1: I0118 11:38:34.704251 1 controllerserver-master.go:86] Controller CreateVolume: Create VolumeID:pvc-26-328dad453a5df7f8348d57f9f7422e6b084a58303f5400af6f92906a based on name:pvc-269396e3-8ad6-4b4a-8296-9944cdd3acf0 pmem-csi-controller-0/pmem-driver@pmem..ker1: W0118 11:38:34.704273 1 controllerserver-master.go:236] failed to connect to : No node registered with id: This attempt uses the node selector label for the PMEM-CSI node pod in the allowedTopologies field. Unfortunately, it also doesn't work: pmem-csi-intel-com-node-q94vv/external-provisioner@pmem..ker2: I0118 14:46:54.540845 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"skew-4425", Name:"direct-testingv4nl9", UID:"fb94e3a9-62e5-44ec-b48b-33471452142c", APIVersion:"v1", ResourceVersion:"1204", FieldPath:""}): type: 'Warning' reason: 'ProvisioningFailed' failed to provision volume with StorageClass "pmem-pmem-csi-sc-ext4-skew-4425-unused": error generating accessibility requirements: selected node '"pmem-csi-pmem-govm-worker2"' topology 'map[pmem-csi.intel.com/node:pmem-csi-pmem-govm-worker2]' is not in allowed topologies: [map[feature.node.kubernetes.io/memory-nv.dax:true]] --- deploy/bindata_generated.go | 20 +++++++++---------- deploy/common/pmem-storageclass-cache.yaml | 4 ++-- deploy/common/pmem-storageclass-default.yaml | 4 ++-- deploy/common/pmem-storageclass-ext4.yaml | 4 ++-- .../pmem-storageclass-late-binding.yaml | 4 ++-- deploy/common/pmem-storageclass-xfs.yaml | 4 ++-- .../storageclass/pmem-storageclass.yaml | 8 ++++---- pkg/pmem-csi-driver/nodeserver.go | 5 ----- pkg/pmem-csi-driver/pmem-csi-driver.go | 11 +--------- test/e2e/driver/driver.go | 10 +++++++++- test/setup-deployment.sh | 12 ++++++++++- 11 files changed, 45 insertions(+), 41 deletions(-) diff --git a/deploy/bindata_generated.go b/deploy/bindata_generated.go index 4208429c9e..2b431d6c1e 100644 --- a/deploy/bindata_generated.go +++ b/deploy/bindata_generated.go @@ -101,7 +101,7 @@ func deployKubernetes117DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610711837, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610970360, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -121,7 +121,7 @@ func deployKubernetes117LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610711838, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610970361, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -141,7 +141,7 @@ func deployKubernetes118DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610711842, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610970365, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -161,7 +161,7 @@ func deployKubernetes118LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610711843, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610970366, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -181,7 +181,7 @@ func deployKubernetes119AlphaDirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 12949, mode: os.FileMode(420), modTime: time.Unix(1610711853, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 12949, mode: os.FileMode(420), modTime: time.Unix(1610970377, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -201,7 +201,7 @@ func deployKubernetes119AlphaLvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 12883, mode: os.FileMode(420), modTime: time.Unix(1610711854, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 12883, mode: os.FileMode(420), modTime: time.Unix(1610970378, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -221,7 +221,7 @@ func deployKubernetes119DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610711848, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 12635, mode: os.FileMode(420), modTime: time.Unix(1610970371, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -241,7 +241,7 @@ func deployKubernetes119LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610711849, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 12569, mode: os.FileMode(420), modTime: time.Unix(1610970372, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -261,7 +261,7 @@ func deployKustomizeWebhookWebhookYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kustomize/webhook/webhook.yaml", size: 1143, mode: os.FileMode(420), modTime: time.Unix(1610703232, 0)} + info := bindataFileInfo{name: "deploy/kustomize/webhook/webhook.yaml", size: 1143, mode: os.FileMode(420), modTime: time.Unix(1610912692, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -281,7 +281,7 @@ func deployKustomizeSchedulerSchedulerServiceYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kustomize/scheduler/scheduler-service.yaml", size: 277, mode: os.FileMode(420), modTime: time.Unix(1610703232, 0)} + info := bindataFileInfo{name: "deploy/kustomize/scheduler/scheduler-service.yaml", size: 277, mode: os.FileMode(420), modTime: time.Unix(1610912692, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/deploy/common/pmem-storageclass-cache.yaml b/deploy/common/pmem-storageclass-cache.yaml index e4e9346e27..f500f25eb7 100644 --- a/deploy/common/pmem-storageclass-cache.yaml +++ b/deploy/common/pmem-storageclass-cache.yaml @@ -2,9 +2,9 @@ allowedTopologies: - matchLabelExpressions: - - key: pmem-csi.intel.com/driver + - key: storage values: - - started + - pmem apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/common/pmem-storageclass-default.yaml b/deploy/common/pmem-storageclass-default.yaml index 3774452d74..72f93ea742 100644 --- a/deploy/common/pmem-storageclass-default.yaml +++ b/deploy/common/pmem-storageclass-default.yaml @@ -2,9 +2,9 @@ allowedTopologies: - matchLabelExpressions: - - key: pmem-csi.intel.com/driver + - key: storage values: - - started + - pmem apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/common/pmem-storageclass-ext4.yaml b/deploy/common/pmem-storageclass-ext4.yaml index c5f271682f..65c4073e30 100644 --- a/deploy/common/pmem-storageclass-ext4.yaml +++ b/deploy/common/pmem-storageclass-ext4.yaml @@ -2,9 +2,9 @@ allowedTopologies: - matchLabelExpressions: - - key: pmem-csi.intel.com/driver + - key: storage values: - - started + - pmem apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/common/pmem-storageclass-late-binding.yaml b/deploy/common/pmem-storageclass-late-binding.yaml index 7de5e22c8d..33adde0217 100644 --- a/deploy/common/pmem-storageclass-late-binding.yaml +++ b/deploy/common/pmem-storageclass-late-binding.yaml @@ -2,9 +2,9 @@ allowedTopologies: - matchLabelExpressions: - - key: pmem-csi.intel.com/driver + - key: storage values: - - started + - pmem apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/common/pmem-storageclass-xfs.yaml b/deploy/common/pmem-storageclass-xfs.yaml index f10c357667..9ac4b24f07 100644 --- a/deploy/common/pmem-storageclass-xfs.yaml +++ b/deploy/common/pmem-storageclass-xfs.yaml @@ -2,9 +2,9 @@ allowedTopologies: - matchLabelExpressions: - - key: pmem-csi.intel.com/driver + - key: storage values: - - started + - pmem apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/kustomize/storageclass/pmem-storageclass.yaml b/deploy/kustomize/storageclass/pmem-storageclass.yaml index 4dffc9c16f..399ad1aeb8 100644 --- a/deploy/kustomize/storageclass/pmem-storageclass.yaml +++ b/deploy/kustomize/storageclass/pmem-storageclass.yaml @@ -7,8 +7,8 @@ provisioner: pmem-csi.intel.com # with late binding only get scheduled to pmem allowedTopologies: - matchLabelExpressions: - # The "pmem-csi.intel.com" part is the driver name. - # It must match the "provisioner" field above. - - key: pmem-csi.intel.com/driver + # This must be the same label that is also used for + # scheduling PMEM-CSI node pods. + - key: storage values: - - started + - pmem diff --git a/pkg/pmem-csi-driver/nodeserver.go b/pkg/pmem-csi-driver/nodeserver.go index da69068e91..0f6d445b54 100644 --- a/pkg/pmem-csi-driver/nodeserver.go +++ b/pkg/pmem-csi-driver/nodeserver.go @@ -93,12 +93,7 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque NodeId: ns.cs.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - // This part varies by node. DriverTopologyKey: ns.cs.nodeID, - // This part is the same on all nodes. It is used - // by storage class label selectors to choose - // nodes which have the driver. - DriverStartedKey: DriverStartedValue, }, }, }, nil diff --git a/pkg/pmem-csi-driver/pmem-csi-driver.go b/pkg/pmem-csi-driver/pmem-csi-driver.go index 84d3d9d880..76dcfbe44c 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver.go @@ -70,20 +70,12 @@ const ( Node DriverMode = "node" // Just the webhooks, using metrics instead of gRPC over TCP. Webhooks DriverMode = "webhooks" - - // DriverStartedValue is the fixed key used for DriverStartedKey. - DriverStartedValue = "started" ) var ( - // DriverTopologyKey is the key to use for per-node topology constraints. + //PmemDriverTopologyKey key to use for topology constraint DriverTopologyKey = "" - // DriverStartedKey is an additional key with the same value on each node. - // It's useful for selecting nodes with PMEM-CSI in cases (like storage - // classes) where a value is required because "key exists" is not valid. - DriverStartedKey = "" - // Mirrored after https://github.com/kubernetes/component-base/blob/dae26a37dccb958eac96bc9dedcecf0eb0690f0f/metrics/version.go#L21-L37 // just with less information. buildInfo = prometheus.NewGaugeVec( @@ -160,7 +152,6 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { } DriverTopologyKey = cfg.DriverName + "/node" - DriverStartedKey = cfg.DriverName + "/driver" // Should GetCSIDriver get called more than once per process, // all of them will record their version. diff --git a/test/e2e/driver/driver.go b/test/e2e/driver/driver.go index 1065eaffca..4b4e093adf 100644 --- a/test/e2e/driver/driver.go +++ b/test/e2e/driver/driver.go @@ -31,6 +31,8 @@ import ( "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" + "github.com/intel/pmem-csi/test/test-config" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -136,7 +138,13 @@ func (m *manifestDriver) GetDynamicProvisionStorageClass(config *testsuites.PerT Expect(ok).To(BeTrue(), "storage class from %s", scManifest) sc.Provisioner = m.csiDriverName sc.Name = config.Prefix + "-" + sc.Name - sc.AllowedTopologies[0].MatchLabelExpressions[0].Key = m.csiDriverName + "/driver" + nodeLabel := testconfig.GetOrFail("TEST_PMEM_NODE_LABEL") + parts := strings.SplitN(nodeLabel, "=", 2) + if len(parts) < 2 { + panic(fmt.Sprintf("expected label=value: TEST_PMEM_NODE_LABEL=%q", nodeLabel)) + } + sc.AllowedTopologies[0].MatchLabelExpressions[0].Key = parts[0] + sc.AllowedTopologies[0].MatchLabelExpressions[0].Values[0] = parts[1] // Add additional parameters, if any. for name, value := range m.parameters { diff --git a/test/setup-deployment.sh b/test/setup-deployment.sh index 3701e54437..5a54d02d62 100755 --- a/test/setup-deployment.sh +++ b/test/setup-deployment.sh @@ -69,7 +69,17 @@ for deploy in ${DEPLOY[@]}; do paths+=" $path" fi if [ -f "$path" ]; then - ${KUBECTL} apply -f - <"$path" + case "$path" in + *storageclass*) + # Patch the node selector label into the storage class instead of the default storage=pmem. + sed -e "s;: storage\$;: \"$(echo $TEST_PMEM_NODE_LABEL | cut -d= -f1)\";" \ + -e "s;- pmem\$;- \"$(echo $TEST_PMEM_NODE_LABEL | cut -d= -f2)\";" \ + "$path" | ${KUBECTL} apply -f - + ;; + *) + ${KUBECTL} apply -f - <"$path" + ;; + esac elif [ -d "$path" ]; then # A kustomize base. We need to copy all files over into the cluster, otherwise # `kubectl kustomize` won't work.