diff --git a/Makefile b/Makefile index e922063458..e701855dae 100644 --- a/Makefile +++ b/Makefile @@ -184,9 +184,6 @@ KUSTOMIZE += deploy/common/pmem-storageclass-cache.yaml=deploy/kustomize/storage KUSTOMIZE += deploy/common/pmem-storageclass-late-binding.yaml=deploy/kustomize/storageclass-late-binding KUSTOMIZE += deploy/operator/pmem-csi-operator.yaml=deploy/kustomize/operator -# Special one-off deployment with device mode = fake. -KUSTOMIZE += deploy/kubernetes-1.19/pmem-csi-fake.yaml=deploy/kustomize/kubernetes-base-fake - KUSTOMIZE_OUTPUT := $(foreach item,$(KUSTOMIZE),$(firstword $(subst =, ,$(item)))) # This function takes the name of a .yaml output file and returns the @@ -204,14 +201,14 @@ $(KUSTOMIZE_OUTPUT): _work/kustomize $(KUSTOMIZE_INPUT) mkdir -p ${@D} $(call KUSTOMIZE_INVOCATION,$<,$@) >$@ if echo "$@" | grep '/pmem-csi-' | grep -qv '\-operator'; then \ - dir=$$(echo "$@" | tr - / | sed -e 's;kubernetes/;kubernetes-;' -e 's;/alpha/;-alpha/;' -e 's/.yaml//' -e 's;/pmem/csi/;/;') && \ + dir=$$(echo "$@" | tr - / | sed -e 's;kubernetes/;kubernetes-;' -e 's;/alpha/;-alpha/;' -e 's;/distributed/;-distributed/;' -e 's/.yaml//' -e 's;/pmem/csi/;/;') && \ mkdir -p $$dir && \ cp $@ $$dir/pmem-csi.yaml && \ echo 'resources: [ pmem-csi.yaml ]' > $$dir/kustomization.yaml; \ fi kustomize: _work/go-bindata clean_kustomize_output $(KUSTOMIZE_OUTPUT) - $< -o deploy/bindata_generated.go -pkg deploy deploy/kubernetes-*/*/pmem-csi.yaml + $< -o deploy/bindata_generated.go -pkg deploy deploy/kubernetes-*/*/pmem-csi.yaml deploy/kustomize/webhook/webhook.yaml deploy/kustomize/scheduler/scheduler-service.yaml clean_kustomize_output: rm -rf deploy/kubernetes-* diff --git a/deploy/bindata_generated.go b/deploy/bindata_generated.go index 8f42003788..a66e86e2ec 100644 --- a/deploy/bindata_generated.go +++ b/deploy/bindata_generated.go @@ -7,8 +7,9 @@ // deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml // deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml // deploy/kubernetes-1.19/direct/pmem-csi.yaml -// deploy/kubernetes-1.19/fake/pmem-csi.yaml // deploy/kubernetes-1.19/lvm/pmem-csi.yaml +// deploy/kustomize/webhook/webhook.yaml +// deploy/kustomize/scheduler/scheduler-service.yaml package deploy import ( @@ -85,7 +86,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _deployKubernetes117DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\x78\xdb\x03\x5a\x01\x7e\xc8\x26\xbe\xd4\xe8\xc6\x31\x92\xec\xbd\x06\x0c\x35\x96\x59\x53\x24\x4b\x8e\xbc\xf1\x15\xfd\xdf\x0b\x52\x92\xad\x2f\xcb\x1f\x97\x0f\x14\xf5\x3e\xac\x43\x71\x38\xbf\xf9\xe0\xcc\x6f\x6c\xff\x91\xdc\x80\x04\x43\x11\x62\xf2\x9d\xe3\x92\x7c\x4a\xe9\x0a\xc8\x2a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x18\x50\xcd\x7f\x05\x63\xb9\x92\x11\x59\x8f\x06\x2b\x2e\xe3\x88\x3c\x80\x59\x73\x06\x97\x8c\xa9\x4c\xe2\x20\x05\xa4\x31\x45\x1a\x0d\x08\x11\xf4\x19\x84\x75\xef\x08\xd1\x29\xa4\x01\xb3\x7c\xc8\x25\x82\x18\x32\x95\x86\x31\x68\xa1\x36\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x20\x44\xd2\x14\xa2\xad\x60\xc0\x94\x44\xa3\x84\x00\x53\x3c\xb3\x9a\xb2\xca\x86\x41\x10\x04\x35\x88\xe6\x99\xb2\x21\xcd\x70\xa9\x0c\xff\x8d\xba\x43\x87\xab\xbf\xda\x21\x57\xe1\x16\xfc\xbd\x12\xf0\x86\x90\xe1\x05\xc1\x48\x2a\xdc\xa6\x35\x77\xa8\xc0\x04\x6c\x91\xec\x31\xc0\x64\x02\x6c\x34\x08\x08\xd5\xfc\xc6\xa8\x4c\x7b\x20\x01\xf9\xf4\x69\x40\x88\x01\xab\x32\xc3\xa0\x58\x03\x19\x6b\xc5\x25\xda\x01\x21\x6b\x30\xcf\xc5\x72\x02\xe8\xff\xff\x4e\x91\x2d\xfd\x3b\xc1\x6d\xbe\x14\x83\x00\x04\xff\x36\xd3\x31\x2d\xde\x32\x03\xee\x6d\x4b\x27\x53\xca\xc4\x5c\x56\xdd\xd6\x06\x21\x80\x5a\x78\x2b\x04\x16\x95\xa1\x09\xec\x55\xce\x2c\x2f\xb6\x30\xaa\x29\xe3\xc8\xf7\x41\xd9\x02\xd8\x61\x2a\x94\x36\xa0\xe8\xed\xf3\x02\xea\x51\xa1\xd0\x2a\xee\x50\xdc\x12\xa5\x5a\xdb\xb6\xb0\x01\x2d\x38\xa3\x16\xba\x22\x79\x46\x46\x5f\x89\xcc\x22\x98\x8f\x48\x6c\x93\x49\x09\xe6\xa4\x2c\xd6\xce\x32\x8b\x20\x71\xad\x44\x96\x9e\x17\xc0\x93\x62\xd5\x50\xc8\x04\xe5\xe9\xf1\x5a\x8b\x5c\x39\x39\x57\xcb\x44\x15\xd4\xee\xbd\x30\x0d\x75\xc7\x55\x81\x35\xb4\x4a\xc0\x89\xd9\xde\xb6\x45\x52\x6d\x97\x0a\x87\x87\x8c\x2a\x42\x56\x6c\xef\xb3\xea\xb5\x74\xb8\x16\xd0\x36\xf8\x80\xaa\xc3\x65\x44\xaa\xf8\x35\x83\x72\xc2\x71\xe7\x75\xac\x2f\x5c\xc6\x5c\x26\x6f\xd9\x6b\x2d\xaf\x5f\x6d\x25\xa0\xaf\x71\x29\x01\xf7\xb0\x70\xda\x4b\xff\xf4\x18\x33\x20\xa4\xd2\x7d\x8f\x6f\x9a\x36\x7b\xfe\x27\x30\xf4\xb5\xa5\x93\x7b\xbc\x0b\x65\xa8\x14\xd8\x8f\x89\xc3\xb9\xde\xae\x76\x86\x93\x0a\xfa\x3b\xf8\xbd\xc9\x26\x3f\x92\x46\x5a\x0d\xcc\x29\xd3\xca\xe0\xb6\xc3\x1b\x8c\xc8\xe8\xe2\xe2\xe2\xc2\xa3\x40\x6a\x12\xc0\x79\x6d\xd5\x82\x00\x86\xca\xe4\x38\xa9\xd6\xfb\x74\x9e\x61\xc4\x07\xba\x2b\x05\x34\x9c\xd9\x33\x7c\x35\xea\xf4\xd5\xe8\xad\x7d\x45\x08\x6e\x34\x44\x64\xa6\x62\x70\x5a\x5b\xce\x73\x3c\x6c\x77\x9d\x1f\x90\x22\x2c\x32\xf1\x00\x1f\x3a\xbb\x94\x8e\x2c\xe9\x60\x44\x46\x2d\x3f\xa5\xae\x67\x7c\xad\xe0\x3a\xe0\xba\xb3\x80\xdb\x3c\xa3\x66\x3d\xf8\x11\x52\x2d\x28\x42\x81\xaa\xe2\x34\x0f\x49\x4a\x85\xbe\x00\x6d\x51\x76\x02\xb1\xcc\x50\x17\x27\x77\x34\xe5\x12\x8c\x2d\x76\x8b\x9a\x85\x07\x6d\x3c\xcb\xca\xbd\x82\xdf\xe1\x79\xa9\xd4\x2a\x22\x3c\x91\xca\x80\xdf\x5a\xc6\xc6\x83\x59\x2c\xb8\xe4\xb8\xd9\xc1\x73\xdd\xfe\xb2\xb5\xea\x22\xf9\xaf\x8c\x1b\x88\xaf\x33\xc3\x65\xf2\xc0\x96\x10\x67\x82\xcb\x64\xea\x0f\x2e\x96\x27\x2f\xc0\x32\x87\xa9\x2a\x99\x9f\xf9\x50\xc4\xfe\x11\x4c\x6a\xeb\x8f\x83\x3c\x15\x26\x2f\xda\x80\xb5\x75\x4f\x97\x3b\x56\xb0\x89\xba\x0c\xec\xf0\x5f\xf9\x52\x1a\x0c\x75\xd9\x46\x66\x0a\xa7\xb2\xf5\x7c\x4d\x45\x06\x2d\x55\x9e\x05\x49\xf5\xa9\x6b\x79\x41\x85\x85\xf2\xc9\x2e\xd0\xe5\x11\x6e\xca\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x33\x97\xe1\x36\xf0\xb1\xe1\xeb\x0a\xe8\x80\x04\xeb\xf1\x9f\xab\x7f\x0a\x95\x24\x5c\x26\xc1\x42\x99\x94\xe2\x18\xe1\x05\xab\x8f\x53\x15\xc3\xb8\xc3\xfa\x80\x04\xe5\x2c\x3d\xce\x24\x7f\x89\xc2\x30\x64\x96\x87\xf5\x64\x1b\x5a\xc5\x56\x55\x19\x03\x09\xb7\x68\x36\x93\x52\x16\x99\x8e\xc2\xf0\x62\xe8\xff\x45\xbb\x5e\x51\xec\x77\x21\xe5\xf1\xf8\x87\x1f\xff\xf1\xed\xcb\xe4\x69\x76\x77\x3d\x79\x9a\x5d\xde\x4e\x7e\xaa\xee\x61\xf4\x17\x2e\x60\x1c\x32\x30\x68\x43\x46\x87\xcc\xd4\x4c\x70\xeb\xd5\x1d\x28\x6c\x73\xcb\x0a\x36\xcd\x1d\x2b\xd8\x54\x77\xe4\x7e\x74\x35\x68\xfc\xc3\x8f\xf3\xdb\xc9\xed\xd3\xd5\xc3\xf4\xe9\xfa\x7e\xfa\xeb\xe4\xbe\x0d\xa9\x28\xff\x5f\xfd\xcc\x34\x8e\x76\x65\xdd\xbd\x40\xae\xab\x91\xcb\x6b\x5e\xdd\xbe\x41\x23\x7d\x7e\x31\x2a\xad\x67\xd0\x82\x83\x88\x0b\x3a\x53\x7d\x35\xfa\x5d\xfd\xa1\x17\x9a\x53\x5c\x46\xfe\x76\x0e\x9d\x77\x5d\xc9\x6a\xa1\x79\x9c\xdc\xdf\x4e\x67\x97\x8f\xd3\xbb\xd9\xd3\xd7\xbb\x9b\xa7\xf9\xe5\xe3\xdf\x9b\x98\x22\x12\x62\xaa\x43\x04\x93\x16\x1f\x78\xb8\x5c\x6a\x1d\xd5\xe5\xab\xf6\x51\xed\x2b\xd7\x3a\xe8\xe6\xee\x7a\xf2\xe5\xdb\x4d\x5b\xf6\xe5\xe7\x8b\xbf\xe5\x75\xe7\x6a\x36\xde\xb9\x99\xa7\x34\x81\x88\xf8\x03\x9b\xf7\x21\x62\x54\x52\xb3\xa9\xef\x9d\x67\x42\xcc\x95\xe0\x6c\x13\x91\xe9\x62\xa6\x70\x6e\xc0\x82\xdc\x25\x4a\xa5\x3b\x35\xae\xd5\xb6\x91\x97\x90\xb7\xb7\x76\xde\xe8\xeb\xd5\x93\x76\x24\x21\x7f\x59\x60\x99\xe1\xb8\xb9\x72\xf3\xda\x0b\xd6\xcb\x22\x8d\xef\xa4\xd8\xdc\x2b\xe5\x73\xd9\x6e\x2c\x42\x1a\x11\x34\xd9\x2e\x7a\x95\x50\xdc\x82\xb5\xce\x24\x1f\xeb\xde\x38\xe5\xa3\xe2\xad\x63\xa4\x35\x0b\x52\xb7\x52\xc8\xfb\x5b\xd1\x82\x5f\xde\x65\x7f\xc1\xf6\x49\x5a\xde\x92\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x5b\xb2\x98\xea\x8a\x44\x40\xa8\x49\x6a\xa8\x9b\xd5\xcd\xc7\x9c\xc6\xb1\xab\xf8\xe3\x63\x0a\x54\xb0\x00\x8a\x99\x81\x20\xa1\x08\x76\xfc\xa8\xb4\x12\x2a\xd9\x8c\x6b\x8e\x76\xfb\xac\x8b\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xe3\x9f\xd3\xda\x72\x0c\x0b\x9a\x09\x0c\x16\xd6\x11\xae\x31\xbc\xe0\x5f\x6a\xcf\x8b\xa4\xd8\x82\xf6\xc5\x63\xd4\xcc\x6a\x37\xa4\x24\xcc\xb8\x11\xcb\xf2\x24\x28\xa6\xf5\xb0\x31\xf8\x44\xeb\xcf\xc3\x8b\xe1\xe7\xb3\xd2\xbc\x6b\xb4\x39\x29\xdf\x47\x6f\x9e\xef\x47\xa5\xee\x09\x09\xb8\x17\x8f\xc9\xe4\xa5\x9d\x29\xe9\xd0\x34\x30\xf8\x47\xdf\x2c\x98\x7c\xa4\xd9\x9e\x54\x9d\xf4\xfa\x58\xa1\x7b\xa1\x12\x8e\x42\x54\x19\x49\x40\x60\xb1\x00\x86\x8e\x54\x14\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x60\x3b\x5c\x65\xcf\x60\x24\x20\xf8\x89\x3b\xa5\x6e\x5c\x1d\x54\x3d\x54\x3d\x35\xd5\xb8\xb9\xe6\x26\x22\x32\x13\xa2\x59\xd9\xf6\xf8\x25\xe8\xbd\xf0\x16\x98\x81\x5a\xf0\xf2\x95\x86\xd5\x5b\xe1\xfc\xa9\xed\x80\xf4\xef\xff\x34\x00\x95\xb7\xbd\x7f\x16\xb9\xa6\x90\x2a\xf9\xb6\x93\x88\xf3\xf6\x81\x19\xe4\xc4\xa9\xa3\x38\xf1\x4c\x78\x1f\x3a\x4c\x54\xa0\xbf\xef\x18\xf1\xea\x5c\x38\x06\x77\x4d\x6f\xa9\xa4\x09\x98\x71\x8e\xf5\x77\x72\xe5\x9a\x73\x7a\x58\x72\xab\xf3\x1c\x45\x75\xb7\xa5\xa3\x41\xa0\x0b\xa9\xf9\xdd\xf5\xd3\x74\xfe\x93\xe7\xd1\xa3\x23\x78\x77\x47\x4d\x6a\x73\xf0\x77\xe1\xd7\x16\x29\x7a\xba\x32\x0e\xd7\xd4\x84\x82\x3f\x87\x47\xf0\xec\xd3\x58\xb9\xb3\x76\x0e\x86\x81\x44\x9a\xc0\x78\x54\x37\xf3\x7f\x9d\xb3\x57\x12\xe0\x1d\xa0\x20\xc5\xcc\x0e\xb5\x8a\x2b\xda\x5e\x9d\xf2\xbf\xe2\x18\xf2\x7f\x3d\x3d\x68\xc3\xd7\x5c\x40\x02\x71\x83\xbd\xd4\xf8\xcb\xc5\xdb\x0f\x12\xe5\xd5\x76\x94\x45\x00\x86\x39\xe3\xb0\x61\x9d\xc2\xd4\x39\x5b\x2e\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xaf\xd4\x5c\x49\x2a\xda\x7e\x71\xfb\x7d\x91\xeb\x99\x2d\x5a\x40\xf2\x2f\xa3\xcf\xd5\xe9\xc4\x7b\xb4\x75\xcf\x4f\x9e\xc1\xf5\xcc\x4e\x31\xac\x5b\x32\x31\xac\x7b\xf4\xd8\x4d\x5b\x8b\xdd\xf4\x22\xeb\xa0\xc7\x07\x07\xb3\xd2\x79\x3d\x37\xf8\x0c\x17\xba\xc3\x7c\x17\x38\x6d\xc0\x2b\x22\x58\x76\xb8\x3c\x2f\x75\xad\x91\x34\xb3\x6d\x4f\xab\xe8\xec\xca\x9d\x03\x64\x7d\x57\x67\x6f\xf8\xdd\x35\xf0\xf0\xa0\xe7\xf3\x27\x2f\x25\x5b\xeb\x4d\xb4\x1e\x0d\x3f\x0f\x2f\xce\xaa\x4d\xcd\xb3\x3e\x7c\x50\x3b\x98\x89\xd5\xa0\xb7\xa4\x6b\x19\xb1\x3b\xa3\xfa\x29\xf1\x0e\x4c\xe1\xd8\x3c\x24\xfb\xa6\xa7\xa5\xb2\xb9\xe6\x6a\x6d\xed\xaf\x6c\xbd\x77\x24\xff\xca\xe5\xda\xdf\x0a\x65\x36\x77\xe6\xaa\xfc\xc5\x41\xaf\x17\xce\x01\xf2\x54\x32\xc0\xf0\x64\xfd\x7b\xfc\x78\x96\x3b\xfa\x0a\xfd\x71\x68\x3a\xcb\xfb\x69\x58\xea\xb5\xfe\x38\xb5\x8d\x0a\x1f\xec\xad\xe1\x47\x8e\xc3\x5e\xb0\x39\x0a\x1f\x61\xc4\x2b\xe4\xd3\x9e\x4a\xdb\xa3\xbc\xde\x89\x8e\xd3\x52\xef\x54\xbd\xc7\xd7\xdb\xd6\x91\x97\xa2\x68\x6b\xcd\x8f\x06\xea\xbf\x5b\x09\xd7\xa3\x67\x40\xba\xfd\x11\xc2\xc3\xf4\x3a\xa7\x5e\x6f\xf6\x49\x41\x25\x2e\xe5\xe4\x4a\x11\x29\x5b\xde\x17\x5f\x6c\x45\xc4\x7f\xbd\xe3\xbf\xfd\x8d\xa7\x72\xa1\xee\xa4\x2f\x8c\xdb\x92\x99\x57\x9d\xaf\x7c\x01\x6c\xc3\x04\xdc\xaa\xb8\xfc\x85\xcc\x7c\xfb\x2b\x2c\xff\xe7\x44\x2f\x21\x05\x43\xc5\xe0\xbf\x01\x00\x00\xff\xff\x99\xe3\x8f\xb5\x93\x2a\x00\x00") +var _deployKubernetes117DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x05\x2a\x2b\xee\x07\xb0\x10\xe0\x87\x5c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8d\x65\xd6\x14\xc9\x92\x23\xef\xfa\x8a\xfe\xef\x05\x25\x59\xa6\x24\x5b\x96\x15\xe7\xa3\x67\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x1f\x3f\xd2\xfa\x23\xf9\x0c\x12\x0c\x45\x88\xc9\x37\x8e\x0b\x72\x96\xd2\x25\x90\x65\x66\x51\xa5\xfc\x37\x38\xfb\x33\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\x59\x4d\x46\x4b\x2e\xe3\x88\x3c\x82\x59\x71\x06\x17\x8c\xa9\x4c\xe2\x28\x05\xa4\x31\x45\x1a\x8d\x08\x11\xf4\x19\x84\x75\xdf\x08\xd1\x29\xa4\x01\xb3\x7c\xcc\x25\x82\x18\x33\x95\x86\x31\x68\xa1\xd6\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x6a\x62\x90\x4f\x0c\x98\x4a\x03\xa6\x24\x1a\x25\x04\x98\x52\xca\x6a\xca\x3c\xd1\x51\x10\x04\x1f\xc6\xd8\x6f\xf0\xbc\x50\x6a\x69\x7b\x9a\x6a\x9e\x29\x1b\xd3\x0c\x17\xca\xf0\xdf\xa8\x5b\x7c\xbc\xfc\x64\xc7\x5c\x85\x95\x13\x0f\x4a\xc0\x9b\x98\x0e\xdf\x11\x8c\xa4\xc2\x89\xaf\xb8\xb3\x0f\x4c\xc0\xe6\xc9\x1e\x57\x4c\x26\xc0\x46\xa3\x80\x50\xcd\x3f\x1b\x95\xe9\xdc\xa4\x80\x9c\x9d\x8d\x08\x31\x60\x55\x66\x18\x94\x63\x20\x63\xad\xb8\x44\x07\xcb\x0a\xcc\x73\x39\x9c\x00\xe6\xff\x7f\xa3\xc8\x16\xf9\x37\xc1\x6d\x31\x14\x83\x00\x84\xfc\x6b\xa6\x63\x5a\x7e\x65\x06\xdc\xd7\x96\x4e\xa6\x94\x89\xb9\xf4\x01\x6c\x1b\x21\x80\x5a\x78\x2d\x0b\x2c\x2a\x43\x13\xd8\xab\x9c\x59\x5e\x8a\x30\xaa\x29\xe3\xc8\xf7\x99\x52\x19\xb0\xb5\xa9\x54\xda\x30\x45\x57\xcf\x4b\x53\x7b\x6d\x85\x56\xf1\x0e\xc5\xad\xa9\x54\x6b\xdb\x9e\x6c\x40\x0b\xce\xa8\x85\x5d\x3b\xf9\x91\x63\x7b\x93\x96\x27\x8a\xe7\xdd\x20\xee\x08\xa4\x01\x98\x5c\x8a\xcc\x22\x98\xf7\x4d\x7b\x93\x49\x09\xe6\x38\x4c\x9c\x8f\x16\x41\xe2\x4a\x89\x2c\x1d\x16\xde\x47\x45\x72\x43\x21\x13\x94\xa7\xfd\xb5\x96\x99\x74\x74\x26\x6f\xd2\x58\x50\xbb\xb7\x9c\x34\xd4\xf5\xab\x91\x2b\x68\x15\xc8\x23\x6b\x41\xdb\x17\x49\xb5\x5d\x28\x1c\x1f\x72\xaa\xdc\xb2\x52\xbc\xcb\xab\x53\xe9\x70\x5d\xbd\xed\xf0\x01\x55\x87\x8b\xac\x54\xf1\x29\x37\xe5\x88\xe5\x3e\x7e\xa6\x57\x45\xf0\x44\xd9\x7d\x14\x3c\x6f\x90\xd1\xfa\x60\x6e\xf7\xcd\xbb\x63\x94\xbc\x84\x15\xb4\x6a\xc9\xb1\x31\x3c\xac\xe5\xfe\xc4\x65\xcc\x65\xf2\x36\xec\xdd\xf2\x7a\x67\x51\x02\xba\xba\xb0\x12\xf0\x00\x73\x67\xc7\x06\xcb\x0e\xb7\x46\x84\x78\x44\x62\x08\xb7\xb5\xd9\xf3\xbf\x80\x61\x9e\x06\x3b\x0f\x0d\xa7\x3c\x98\x7c\xc0\xed\xd9\xd6\x84\x77\xd9\x97\x1a\x2f\x7b\xc9\x5e\x9c\xfe\xdc\xe5\x55\xe7\xf7\xce\x97\xa1\xe8\xfb\x0d\x66\x20\x03\xfc\x68\xf9\xf1\x4e\xbb\x52\x4b\x93\x57\xde\x8e\x66\x9b\x7e\xc3\xb4\x68\xde\x9c\x7c\x8c\xfb\x1d\xab\x81\x39\xb5\x5a\x19\xac\x0e\x60\x06\x23\x32\x39\x3f\x3f\x3f\xcf\xed\x41\x6a\x12\xc0\x59\x6d\xd4\x82\x00\x86\xca\x14\x16\x53\xad\xc7\xcb\xec\x19\x8c\x04\x84\x3c\xa6\xb8\xb4\x48\xa5\xaf\x69\xeb\xcd\x9e\x29\x0d\x1f\x6a\x96\x0f\x00\xe5\x43\xc0\x9f\x02\x1a\xce\xf6\x45\x48\x17\xf6\x93\x9d\xd8\x4f\xfe\x3f\xb0\x27\x04\xd7\x1a\x22\x72\xa7\x62\x70\xb6\xb7\x36\x83\x6a\x6d\xb7\x75\xe7\x11\x29\xc2\x3c\x13\x8f\xb0\xff\x1e\xb1\x6d\x33\x53\xa9\x56\x32\xb7\xa0\x61\xef\x1b\x40\xd2\x96\xd7\xd4\x60\xa0\xe6\xde\xf6\xbe\x5b\x2e\x6f\x6e\x92\x22\x32\x69\x85\x4b\xea\xb8\xed\x17\x0f\xd8\x41\x70\x1d\x0f\xd8\x20\x28\x6c\x91\xaa\x77\xbd\x10\x41\x48\xb5\xa0\x08\xa5\x9f\x5e\x1c\xe5\xf6\x4a\xa9\x30\xef\x1c\x95\xdf\x3b\x4d\xb2\xcc\x50\x17\xba\x6e\x69\xca\x25\x18\x5b\x4a\x8b\x1a\x66\xc7\x05\xe4\x60\x94\x87\xe0\xdc\x3f\x38\x07\xee\xca\xde\x89\x65\x43\x8c\x08\x4f\xa4\x32\x90\x8b\x6e\x62\xd2\x7d\xb6\x98\x6e\x46\x02\xc2\x54\x9a\x52\x19\x6f\x71\x0d\x48\x98\x59\x13\x0a\xc5\xa8\x08\x9f\xb9\x0c\x2b\x4f\x63\xc3\x57\x9e\x97\x01\x09\x56\xd3\xbf\xfa\x7f\x0a\x95\x24\x5c\x26\xc1\x5c\x99\x94\xe2\x14\xe1\x3b\xfa\x8f\x53\x15\xc3\xd4\x6b\xda\xd5\x83\x62\x61\x87\xeb\xf4\x87\x1f\x67\xb7\xd7\xb7\x4f\x97\x8f\x37\x4f\x57\x0f\x37\xbf\x5e\x3f\x3c\xdd\x5d\xdc\x5e\xff\xc9\x97\x76\x27\xc8\xc7\x32\xa7\xa6\xff\x39\x2b\x8f\x99\x67\xd1\x99\x33\xf4\xec\xbf\xbe\x28\xa3\x3f\x73\x01\xd3\x90\x81\x41\x1b\x32\x3a\x66\xa6\x66\x91\x1b\xf7\x25\x50\xd8\xa6\xc8\x12\xd6\x4d\x89\x25\xac\x7d\x09\xcb\x16\x10\x67\x02\xcc\x97\xfc\x78\x3f\x8d\x3e\x6d\x1a\xf7\xc6\xef\xa2\x09\x6d\x1e\x6f\x9b\x8b\xfb\x80\x5c\xf9\xd8\x17\xc1\xf5\xf5\xfa\xe1\xf6\xe6\xee\xe2\xeb\xcd\xfd\xdd\xd3\x97\xfb\xcf\x4f\xb3\x8b\xaf\xff\xa8\x84\x08\x59\x51\x91\x41\x44\xc2\x18\x56\x21\x82\x49\xcb\x1f\x07\x1c\xfe\xad\xa5\x76\xc1\xd9\x5e\xaa\x23\x01\xaa\x85\xee\xaf\xf2\xc9\x8f\xb3\x8b\xcb\xd6\x0a\x3f\x1b\x95\x46\xde\x20\x21\x73\x0e\x22\x2e\x79\x64\x6b\x7c\x46\x71\x11\x55\x25\x62\x5c\xd5\xd2\x4a\x96\xa7\x34\x81\x88\xe4\xd6\x34\x03\x30\x62\x54\x52\xb3\xae\xcb\xce\x32\x21\x66\x4a\x70\xb6\x8e\xc8\xcd\xfc\x4e\xe1\xcc\x80\x05\xb9\xdd\x4a\x2f\x6b\x1b\x71\x5c\x75\xff\x8d\xbf\x55\x9a\xcc\x1a\x64\xc0\x5f\x69\xcb\x2c\x8a\x8f\x05\x96\x19\x8e\xeb\x4b\x25\x5d\xdc\xfb\x6e\x1b\xa0\xf1\xbd\x14\xeb\x07\xa5\xf2\x68\xb3\x6b\x8b\x90\x46\x04\x4d\xb6\xf5\xd8\xdb\xc7\x5b\xb0\xd6\xb9\x94\x83\xd4\xb9\xc9\xc5\x4d\xd2\xad\x23\xca\x35\x0f\x52\x37\x52\xce\xcf\xe3\xb6\x65\x7e\x99\x86\x79\x06\x8c\x0e\x78\x60\x32\x79\x61\xef\x94\x74\xf6\x37\xac\xce\x1f\xfd\x62\xc1\x14\xcc\xb4\x5a\xc9\xa7\xf0\x7b\xbb\x47\xa3\x14\xa0\x12\x60\xea\x1d\x22\x20\x30\x9f\x03\x43\xc7\x64\x1e\xcb\x34\xab\x74\x2f\x61\x1d\xe5\x97\x74\xf9\xb9\xa5\x51\x70\x53\xea\x4e\x23\x23\x1f\x26\x6f\xd5\xbd\x18\xe4\x28\x18\xa8\x6d\x5f\x31\xd2\xa3\x07\x06\x85\xe4\x01\xb2\x75\x45\x21\x55\x72\x30\xd5\x72\x0e\xbf\x1a\xc9\xea\x58\xfc\xf5\xe9\x55\xa9\xbc\x8b\x58\xbd\x13\x95\xaa\x60\x19\x46\xc5\x3f\x10\x2f\xf2\x3c\x79\x3d\x46\x74\x40\xc9\xef\x94\x0b\xc5\xe0\x6a\xde\x2d\x95\x34\x01\x33\x2d\x6c\x7d\x21\x57\xaa\x01\x19\x90\x60\xf3\xa6\xc1\x34\x93\xfc\x7b\x14\x86\x21\xb3\xdc\xfd\x1b\x5b\xc5\x96\x4d\x8a\xc4\xe3\xe9\x0f\x3f\xfe\xf3\x97\x9f\xae\x9f\xee\xee\xaf\xae\xdb\x34\xca\xba\x43\x9f\xeb\x11\xd3\x70\x45\x4d\x28\xf8\x73\xd8\x83\x7c\x1d\x47\xd5\x1c\x5c\x33\x30\x0c\x24\xd2\x04\xa6\x93\x97\xf2\xa2\xba\x3b\xc3\x49\x48\xe3\x5e\xa2\xfe\xd0\x63\x28\x2e\x60\xc6\x0e\x4c\x57\xfd\x5f\x8f\x5a\xf5\xa4\x7b\x98\xea\xbd\x4c\xe0\x77\x45\x98\xb4\xe1\x2b\x2e\x20\x81\xb8\x41\x37\x6a\x84\xe3\xfc\x30\x77\xea\x42\xac\x0f\x77\xda\x24\x86\x2b\x60\x02\x30\xd4\x22\x4b\xb8\xb4\x61\xa3\xbe\x7a\x25\x8c\x94\xf3\x8d\xd2\x34\xc9\x95\x46\xe4\x27\x5e\x54\x03\xae\x24\x15\x6d\x5c\x9c\x7c\x9e\xd6\x41\xcc\x4d\x5f\x43\x8a\x77\x34\x86\xea\x74\xd3\x3b\xb4\xc5\xb0\x6a\x4d\x89\x61\xd5\x31\xc3\xae\xdb\x14\xd3\xae\xbb\x74\xd4\x41\x2b\x67\x28\xb6\x84\x3e\x30\x74\xe4\xd4\x00\x30\xdc\x62\x79\x35\xf4\x34\x07\x84\x9a\xa4\x16\x17\xcd\x02\x1e\x94\x7b\x11\x18\x48\xb8\xc5\x82\xb9\x06\xba\x56\x50\x9b\x71\xb3\xa7\x64\xee\xac\xe1\x79\x02\xd3\x38\x36\x60\xed\x74\x77\xa5\xdf\x59\x23\x5f\x5c\x95\xca\x52\xb2\xfc\x64\xc7\x09\x33\x2e\xc0\x2d\x4f\x82\xf2\x98\x1d\x6e\x1a\x7c\x59\x14\x2a\xef\x4d\xb4\x9a\x8c\xff\x32\x3e\x1f\x54\x65\x9a\x6b\x9d\xec\x54\xd5\xeb\x80\x34\x24\x12\xfd\x4d\x6f\xcd\xae\x45\xc4\x51\x31\x75\x78\xcf\x9d\xd4\x1c\x28\x66\x06\x82\x84\x22\xd8\xe9\x57\xa5\x95\x50\xc9\x7a\x5a\xf3\xdc\xc9\x15\xfb\x54\x31\xa8\xb6\x80\x75\x25\x19\x03\xdc\xbb\x02\x4f\x53\x88\xb9\x4b\x8d\x4a\x66\x4e\x85\xad\x0b\x21\x4f\x41\x65\x38\xfd\x7b\x5a\x1b\x8e\x61\x4e\x33\x81\xc1\xdc\xe2\x5a\xc3\x14\xbe\xe3\xdf\x6a\xcf\xcb\x86\x50\xf9\x9b\x13\x80\x49\x77\x70\xbf\x7b\xef\x2f\x93\xa3\x4c\x8c\xe5\x27\xeb\x2a\x47\x4e\xe4\x9a\x49\xe2\xfd\xde\xf8\x92\x06\xbc\xeb\xf7\xcb\xa3\x3a\xf1\xe4\xd5\xaf\x2e\x4e\x96\x64\xfe\xcd\xde\x76\xa1\x12\xd4\xa2\x66\x1d\x77\xc3\xd0\xba\x9b\x6d\xdd\x04\x2c\x94\x2d\x4c\xf4\xe9\x47\x77\xf3\xef\x6c\x3e\xc5\xef\x2e\x57\x79\xbb\x51\x66\x7d\x6f\x2e\x37\xaf\xb4\x75\x7a\x3e\xc4\x90\xa7\xb2\xcc\xac\xc3\xa3\xf5\xef\x2d\x50\x03\xe0\xe8\xe2\x42\xfd\xac\xd9\xc9\x80\x8e\xb3\xa5\x4e\x87\xfa\xa9\x6d\x90\xa0\x5e\x0a\x4f\xb0\xf7\x7b\xe8\x46\x87\xf2\x3a\x1d\xeb\xa7\xa5\x4e\xd7\x3a\x97\xaf\x73\xb7\x9e\x01\x5c\x72\xbb\xe6\x55\x57\xfd\x9d\xb0\x70\x35\x79\x06\xa4\xd5\xeb\x0d\x8f\x37\x57\xc5\x49\xe2\xd5\x7e\xfa\xf5\xf6\x65\x73\xd8\xa7\x88\x94\x2d\x1e\xe0\xdf\x19\x37\xee\x40\xb1\xe9\x5f\x5a\xc5\x37\x72\xae\xee\x65\x5e\xb8\xaa\x92\x56\x54\x88\x2f\x7c\x0e\x6c\xcd\x04\xdc\xaa\x78\xf3\x3e\xdb\xac\x7a\x81\x2f\xff\xf3\x5a\x2f\x20\x05\x43\xc5\xe8\x7f\x01\x00\x00\xff\xff\x5b\xc9\x12\x9a\x7d\x32\x00\x00") func deployKubernetes117DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -100,12 +101,12 @@ func deployKubernetes117DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 10899, mode: os.FileMode(436), modTime: time.Unix(1610052987, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 12925, mode: os.FileMode(420), modTime: time.Unix(1611067491, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes117LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\x3c\xed\x02\xad\x00\x3f\x64\x12\x6f\x6a\x74\x92\x18\x49\x66\x5f\x03\x86\x3a\x96\x59\x53\x24\x4b\x1e\x69\xe2\x2d\xfa\xdf\x0b\x52\x92\xad\x9b\x15\xdb\x9b\x0b\x8a\x7a\x1e\xc6\xa1\x78\x78\x3e\x9e\xeb\x77\x6c\xff\x91\x5c\x81\x04\x43\x11\x62\xf2\x9d\xe3\x8a\x7c\x4a\xe9\x1a\xc8\x3a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\xc9\x27\xa3\x35\x97\x71\x44\xee\xc1\xe4\x9c\xc1\x39\x63\x2a\x93\x38\x4a\x01\x69\x4c\x91\x46\x23\x42\x04\x7d\x02\x61\xdd\x3b\x42\x74\x0a\x69\xc0\x2c\x1f\x73\x89\x20\xc6\x4c\xa5\x61\x0c\x5a\xa8\x4d\x0a\x12\x23\x22\xf2\x34\xd0\x46\xc5\x19\x43\xae\xe4\x88\x10\x49\x53\x88\xb6\x52\x01\x53\x12\x8d\x12\x02\x4c\xf9\xcc\x6a\xca\x6a\x1b\x46\x41\x10\x34\xf0\x99\x27\xca\xc6\x34\xc3\x95\x32\xfc\x37\xea\x0e\x1d\xaf\xff\x6a\xc7\x5c\x85\x5b\xe4\x77\x4a\xc0\x5b\xe1\x85\x67\x04\x23\xa9\x70\x9b\x72\xee\x20\x81\x09\xd8\x32\xd9\x83\xde\x64\x02\x6c\x34\x0a\x08\xd5\xfc\xca\xa8\x4c\x7b\x14\x01\xf9\xf4\x69\x44\x88\x01\xab\x32\xc3\xa0\x5c\x03\x19\x6b\xc5\x25\xda\x11\x21\x39\x98\xa7\x72\x39\x01\xf4\xff\x7f\xa7\xc8\x56\xfe\x9d\xe0\xb6\x58\x8a\x41\x00\x82\x7f\x9b\xe9\x98\x96\x6f\x99\x01\xf7\xb6\xa3\x93\x29\x65\x62\x2e\xeb\x36\xeb\x82\x10\x40\x2d\xbc\x15\x02\x8b\xca\xd0\x04\xf6\x2a\x67\x96\x97\x5b\x18\xd5\x94\x71\xe4\xfb\xa0\x6c\x01\xec\x30\x95\x4a\x5b\x50\xf4\xf6\x79\x09\xf5\x20\x57\x68\x15\xf7\x28\xee\x88\x52\xad\x6d\x57\xd8\x80\x16\x9c\x51\x0b\x7d\x9e\x3c\x21\x9c\x2f\x44\x66\x11\xcc\xbb\x47\xb5\xc9\xa4\x04\x73\x54\x08\x6b\x77\x2d\x8b\x20\x31\x57\x22\x4b\x4f\xf3\xde\x51\x8e\x6a\x29\x64\x82\xf2\xf4\x70\xad\x65\xa0\x1c\x1d\xa8\x55\x94\x0a\x6a\xf7\x66\x4b\x4b\xdd\x61\x25\x20\x87\x4e\xfe\x1f\x19\xea\xdd\xbb\x48\xaa\xed\x4a\xe1\xf8\xa5\x4b\x95\x2e\x2b\xb7\x0f\xdd\xea\xb5\x74\xb8\xe2\xdf\xbd\xf0\x0b\xaa\x5e\xae\x21\x52\xc5\xaf\xe9\x94\x23\x8e\x3b\xad\x57\x7d\xe1\x32\xe6\x32\x79\xb3\x16\x6b\x79\x33\xaf\x95\x80\xa1\x96\xa5\x04\xdc\xc1\xd2\xa9\xae\x8c\x33\x70\x93\x11\x21\xb5\xa6\x7b\x78\xbb\xb4\xd9\xd3\x3f\x81\xa1\x2f\x2c\xbd\x7c\xe3\x5d\x98\x42\xad\xb4\x7e\x80\x13\x4e\x35\x75\xbd\x21\x1c\x55\xca\xdf\xc1\xe8\x6d\xfa\xf8\x61\xbc\xd1\x6a\x60\x4e\x93\x56\x06\xb7\x5d\xdd\x60\x44\x26\x67\x67\x67\x67\x1e\x02\x52\x93\x00\x2e\x1a\xab\x16\x04\x30\x54\xa6\x00\x49\xb5\xde\xa7\xf3\xd8\x1b\x7c\x94\xa1\x52\x40\xc3\x99\x3d\xc1\x4a\x93\x5e\x2b\x4d\xde\xd4\x4a\x84\xe0\x46\x43\x44\x6e\x54\x0c\x4e\x65\xc7\x6c\x8e\x72\xed\xf2\xf7\x1e\x29\xc2\x32\x13\xf7\xf0\x71\x03\x4a\x65\xc2\x8a\xf6\x45\x64\xd2\xb1\x50\xea\xda\xc3\xd7\x1a\xa8\x17\x8c\x76\x3c\x6a\x5b\x44\xd1\xcd\x00\x78\x84\x54\x0b\x8a\x50\x42\xaa\x99\xcb\xe3\x91\x52\xa1\x2f\x37\x5b\x88\xbd\x28\x2c\x33\xd4\x79\xc8\x1d\x4d\xb9\x04\x63\xcb\xdd\xa2\x71\xbd\x17\x2f\x78\xfc\x15\xf7\x4a\x7d\x87\xa7\x95\x52\xeb\x88\xf0\x44\x2a\x03\x7e\x6b\xe5\x15\x8f\x64\xb9\xe4\x92\xe3\x66\x87\xcd\xb5\xf4\xf3\xce\xaa\xf3\xe1\xbf\x32\x6e\x20\xbe\xcc\x0c\x97\xc9\x3d\x5b\x41\x9c\x09\x2e\x93\xb9\x3f\xb8\x5c\x9e\x3d\x03\xcb\x1c\xa6\xba\x64\x71\xe6\x7d\xe9\xf5\x07\x30\xa9\x6d\x3e\x0e\x8a\x20\x98\x3d\x6b\x03\xd6\x36\xcd\x5c\xed\x58\xc3\x26\xea\xbb\x60\x8f\xf1\xaa\x97\xd2\x6e\xd0\x57\xc6\xa5\x0c\xce\x65\xe7\x79\x4e\x45\x06\x1d\x55\x9e\xea\x48\xf5\xa9\x6f\x79\x49\x85\x85\xea\xc9\xce\xcb\xd5\x11\x6e\x8e\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x13\x97\xe1\xd6\xeb\xb1\xe1\x79\x0d\x74\x40\x82\x7c\xfa\xe7\xfa\x9f\x42\x25\x09\x97\x49\xb0\x54\x26\xa5\x38\x45\x78\xc6\xfa\xe3\x54\xc5\x30\xed\xb9\x7d\x40\x82\x6a\x5a\x9e\x66\x92\x3f\x47\x61\x18\x32\xcb\xc3\x66\xa4\x8d\xad\x62\xeb\xba\x8c\x81\x84\x5b\x34\x9b\x59\x25\x8b\x4c\x47\x61\x78\x36\xf6\xff\xa2\x5d\x67\x28\xf7\x3b\x97\xf2\x78\xfa\xc3\x8f\xff\xf8\xf6\x65\xf6\x78\x73\x7b\x39\x7b\xbc\x39\xbf\x9e\xfd\x54\xdf\xc3\xe8\x2f\x5c\xc0\x34\x64\x60\xd0\x86\x8c\x8e\x99\x69\x5c\xc1\xad\xd7\x77\xa0\xb0\xed\x2d\x6b\xd8\xb4\x77\xac\x61\x53\xdf\x51\xd8\xd1\x55\x9f\xe9\x0f\x3f\x2e\xae\x67\xd7\x8f\x17\xf7\xf3\xc7\xcb\xbb\xf9\xaf\xb3\xbb\x2e\xa4\xb2\xe4\x7f\xf5\x83\xd1\x34\xda\x95\x72\xf7\x02\x99\xd7\x3d\x57\x54\xbb\xe6\xfd\x46\xad\xf0\xf9\xc5\xa8\xb4\x19\x41\x4b\x0e\x22\x2e\x99\x4b\xfd\xd5\x6a\x70\xcd\x87\x5e\x68\x41\x71\x15\xf9\xec\x1c\x3b\xeb\xba\x7a\xd5\x41\xf3\x30\xbb\xbb\x9e\xdf\x9c\x3f\xcc\x6f\x6f\x1e\xbf\xde\x5e\x3d\x2e\xce\x1f\xfe\xde\xc6\x14\x91\x10\x53\x1d\x22\x98\xb4\xfc\x48\xc3\xc5\x52\xe7\xa8\x3e\x5b\x75\x8f\xea\xa6\x5c\xe7\xa0\xab\xdb\xcb\xd9\x97\x6f\x57\x5d\xd9\xe7\x9f\xcf\xfe\x56\xd4\x9d\x8b\x9b\xe9\xce\xcc\x3c\xa5\x09\x44\xc4\x1f\xd8\xce\x87\x88\x51\x49\xcd\xa6\xb9\x77\x91\x09\xb1\x50\x82\xb3\x4d\x44\xe6\xcb\x1b\x85\x0b\x03\x16\xe4\x2e\x50\x6a\x7d\xa9\x95\x56\xdb\xe6\x5d\x41\xde\x66\xed\xa2\xd5\xcb\xeb\x27\xed\x88\x41\xf1\xb2\xc0\x32\xc3\x71\x73\xe1\x86\xb2\x67\x6c\x96\x45\x1a\xdf\x4a\xb1\xb9\x53\xca\xc7\xb2\xdd\x58\x84\x34\x22\x68\xb2\x9d\xf7\x6a\xae\xb8\x06\x6b\xdd\x95\xbc\xaf\x07\xfd\x54\xcc\x83\xd7\x8e\x7c\x36\x6e\x90\xba\x95\x52\xde\x67\x45\x07\x7e\x95\xcb\x3e\xc1\xf6\x49\x5a\xde\x91\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x3b\xb2\x98\xea\x9a\x44\x40\xa8\x49\x1a\xa8\xdb\xd5\xcd\xfb\x9c\xc6\xb1\xab\xf8\xd3\x43\x0a\x54\xb0\x04\x8a\x99\x81\x20\xa1\x08\x76\xfa\xa0\xb4\x12\x2a\xd9\x4c\x1b\x86\x76\xfb\xac\xf3\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xd3\x9f\xd3\xc6\x72\x0c\x4b\x9a\x09\x0c\x96\xd6\xf1\xac\x29\x3c\xe3\x5f\x1a\xcf\xcb\xa0\xd8\x82\xf6\xc5\x63\xd2\x8e\x6a\x37\x8f\x24\xcc\xb8\x51\xca\xf2\x24\x28\x47\xf2\xb0\x35\xe3\x44\xf9\xe7\xf1\xd9\xf8\xf3\x49\x61\xde\x37\xc5\x1c\x15\xef\x93\x37\x8f\xf7\x83\x42\xf7\x88\x00\xdc\x8b\xc7\x64\xf2\xdc\xde\x28\xe9\xd0\xb4\x30\xf8\x47\xdf\x2c\x98\x62\x80\xd9\x9e\x54\x1f\xea\x86\x28\xa1\x7b\xa1\x12\x8e\x42\xd4\x19\x49\x40\x60\xb9\x04\x86\x8e\x54\x94\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x0c\x3b\x5e\x67\x4f\x60\x24\x20\xf8\xc9\x3a\xa5\x6e\x32\x1d\xd5\x2d\x54\x3f\x35\xd5\xb8\xb9\xe4\x26\x22\x32\x13\xa2\x5d\xd9\xf6\xd8\x25\x18\x4c\x78\x0b\xcc\x40\xc3\x79\xc5\x4a\xeb\xd6\x5b\xe1\xe2\xa9\xed\x81\xf4\xef\xff\xb4\x00\x55\xd9\x3e\x3c\x82\x5c\x52\x48\x95\x7c\xc3\x01\xc4\x99\xfa\x85\xd1\xe3\xc8\x61\xa3\x3c\xf1\x14\x6c\x1f\x3a\x43\xd4\x70\xbf\xe3\xf4\xf0\xea\x14\x38\x06\x97\x9d\xd7\x54\xd2\x04\xcc\x54\xe4\xe9\xef\xe4\xc7\x0d\xb3\x0c\x30\xe3\x4e\xb7\x39\x88\xde\x6e\xcb\x45\x8b\x34\x97\x52\x8b\xdb\xcb\xc7\xf9\xe2\x27\xcf\x9d\x27\x07\x70\xed\x9e\x3a\xd4\xe5\xdd\xef\xc2\xa9\x2d\x52\xf4\x14\x65\x1a\xe6\xd4\x84\x82\x3f\x85\x07\x70\xeb\xe3\x98\xb8\xbb\xed\x02\x0c\x03\x89\x34\x81\xe9\xa4\x79\xcd\xff\x75\x9e\x5e\x0b\x80\x77\x80\x82\x14\x33\x3b\xd6\x2a\xae\x69\x7b\x75\x9a\xff\x8a\xa3\xc7\xff\xf5\xc4\xa0\x0d\xcf\xb9\x80\x04\xe2\x16\x63\x69\x70\x96\xb3\xb7\x1f\x1e\xaa\xd4\x76\x34\x45\x00\x86\x05\xcb\xb0\x61\x93\xb6\x34\x79\x5a\x21\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xe6\x06\x7c\x2b\xa1\xa2\x6b\x17\xb7\xdf\x17\xb9\x81\x79\xa2\x03\xa4\xf8\x8a\xf9\x54\x9d\x4e\x7c\x40\x5b\xff\xcc\xe4\x59\xdb\xc0\xbc\x14\x43\xde\x91\x89\x21\x1f\xd0\x63\x37\x5d\x2d\x76\x33\x88\xac\x87\x12\xbf\x38\x8c\x55\xc6\x1b\xc8\xe0\x13\x4c\xe8\x0e\xf3\x5d\xe0\xb8\xa1\xae\xf4\x60\xd5\xe1\x8a\xb8\xd4\x8d\x46\xd2\x8e\xb6\x3d\xad\xa2\xb7\x2b\xf7\x0e\x8d\xcd\x5d\xbd\xbd\xe1\x77\xd7\xc0\x97\x87\x3b\x1f\x3f\x45\x29\xd9\xde\xde\x44\xf9\x64\xfc\x79\x7c\x76\x52\x6d\x6a\x9f\xf5\xe1\xc3\xd9\x8b\x91\x58\x77\x7a\x47\xba\x11\x11\xbb\x33\xea\x9f\x0c\xef\xc0\x94\x86\x2d\x5c\xb2\x6f\x62\x5a\x29\x5b\x68\xae\xd7\xd6\xe1\xca\x36\x98\x23\xc5\xb7\x2b\x97\x3e\x2b\x94\xd9\xdc\x9a\x8b\xea\xa7\x04\x83\x56\x38\x05\xc8\x63\xc5\x00\xc3\xa3\xf5\xef\xb1\xe3\x49\xe6\x18\x2a\xf4\x87\xa1\xe9\x2d\xef\xc7\x61\x69\xd6\xfa\xc3\xd4\xb6\x2a\x7c\xb0\xb7\x86\x1f\x38\x02\x7b\xc1\xf6\xf8\x7b\xc0\x25\x5e\x21\x9e\xf6\x54\xda\x01\xe5\xcd\x4e\x74\x98\x96\x66\xa7\x1a\x3c\xbe\xd9\xb6\x0e\x4c\x8a\xb2\xad\xb5\x3f\x0e\x68\xfe\x20\x25\xcc\x27\x4f\x80\x74\xfb\x03\x83\xfb\xf9\x65\x41\xbd\xde\xe6\xd3\x81\x9a\x53\xaa\x99\x95\x22\x52\xb6\xba\x2b\xbf\xc9\x8a\x88\xff\x3e\xc7\x7f\xc5\x1b\xcf\xe5\x52\xdd\x4a\x5f\x15\xb7\xf5\xb2\x28\x39\x5f\xf9\x12\xd8\x86\x09\xb8\x56\x71\xf5\xbb\x97\xc5\xf6\xb7\x55\xfe\xcf\x99\x5e\x41\x0a\x86\x8a\xd1\x7f\x03\x00\x00\xff\xff\x3a\x78\x18\xc6\x60\x2a\x00\x00") +var _deployKubernetes117LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xd3\x7b\xb8\x02\x95\x15\xf7\x0f\xb0\x10\xe0\x87\x6c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8e\x65\xd6\x14\xc9\x92\x94\x76\x7d\x45\xbf\x7b\x41\x49\x96\x29\xc9\x56\x24\xc5\xde\xa4\x6b\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x7f\x7e\xa4\xf5\x47\xf4\x09\x04\x68\x6c\x81\xa2\xaf\xcc\xae\xd0\x59\x82\xd7\x80\xd6\xa9\xb1\x32\x61\xbf\xc3\xd9\x9f\x11\x95\x48\x48\x8b\x80\x32\xfb\x87\xc9\x04\x2b\xf6\x1b\x68\xc3\xa4\x88\x50\x36\x9b\xac\x99\xa0\x11\x7a\x04\x9d\x31\x02\x17\x84\xc8\x54\xd8\x49\x02\x16\x53\x6c\x71\x34\x41\x88\xe3\x67\xe0\xc6\x7d\x43\x48\x25\x90\x04\xc4\xb0\x29\x13\x16\xf8\x94\xc8\x24\xa4\xa0\xb8\xdc\x24\x20\x6c\x84\x78\x96\x04\x4a\x4b\x9a\x12\xcb\xa4\x98\x20\x24\x70\x02\x51\x35\x2b\xc8\x67\x05\x44\x26\x01\x91\xc2\x6a\xc9\x39\xe8\x52\xca\x28\x4c\x3c\xd1\x49\x10\x04\xef\xc3\xd2\xaf\xf0\xbc\x92\x72\x6d\x7a\xda\xa9\x9f\x31\x99\xe2\xd4\xae\xa4\x66\xbf\x63\xb7\xf8\x74\xfd\xc1\x4c\x99\x0c\x2b\x0f\x1e\x24\x87\xd3\xdb\x0d\xdf\x2c\x68\x81\xb9\x13\xcf\x98\x33\x0e\x74\x40\x96\xf1\x01\x3f\x74\xca\xc1\x44\x93\x00\x61\xc5\x3e\x69\x99\xaa\xdc\x9e\x00\x9d\x9d\x4d\x10\xd2\x60\x64\xaa\x09\x94\x63\x20\xa8\x92\x4c\x58\x87\x49\x06\xfa\xb9\x1c\x8e\xc1\xe6\xff\x7f\xc5\x96\xac\xf2\x6f\x9c\x99\x62\x88\x02\x07\x0b\xf9\xd7\x54\x51\x5c\x7e\x25\x1a\xdc\xd7\x96\x4e\x22\xa5\xa6\x4c\xf8\xe8\xb5\x8d\xe0\x80\x0d\x9c\xca\x02\x63\xa5\xc6\x31\x1c\x54\x4e\x0c\x2b\x45\x08\x56\x98\x30\xcb\x0e\x99\x52\x19\xb0\xb3\xa9\x54\xda\x30\x45\x55\xcf\x4b\x53\x7b\x6d\x85\x92\x74\x8f\xe2\xd6\x54\xac\x94\x69\x4f\xd6\xa0\x38\x23\xd8\xc0\xbe\x9d\x7c\xb7\x81\xbd\x4d\xc8\x23\x05\xf3\x7e\x04\xf7\x44\xd1\x08\x40\x2e\x79\x6a\x2c\xe8\x37\x4c\x78\x9d\x0a\x01\x7a\x18\x20\xce\x41\x63\x41\xd8\x4c\xf2\x34\x19\x17\xd8\x83\x62\xb8\xa1\x90\x70\xcc\x92\xfe\x5a\xcb\x1c\x1a\x9c\xc3\xdb\x04\xe6\xd8\x1c\x2c\x24\x0d\x75\xfd\xaa\x63\x06\xad\xd2\x38\xb0\x0a\xb4\x7d\x11\x58\x99\x95\xb4\xd3\x97\x9c\x2a\xb7\xac\x14\xef\xf2\xea\x58\x3a\x5c\x27\x6f\x3b\xfc\x82\xaa\x97\xcb\xab\x90\xf4\x98\x9b\x32\x60\xb9\x77\x9e\xe6\x55\xf9\x3b\x52\x6a\x0f\xc2\xe6\x3b\xa4\xb3\x7a\x31\xb1\xfb\x26\xdd\x10\x25\xaf\x21\x03\xad\x42\x32\x34\x80\xc7\x75\xda\x8f\x4c\x50\x26\xe2\xef\xc0\xd5\x0d\xab\xf7\x14\xc9\xa1\xab\xf9\x4a\x0e\x0f\xb0\x74\x46\x6c\x81\xec\xf0\x69\x82\x90\x47\x1e\xc6\xf0\x59\x93\x3e\xff\x0b\x88\xcd\x73\x60\xef\x11\xe1\x98\xc7\x90\xf7\xb6\x37\xbb\x6a\xf0\x26\x9b\x52\xe3\x62\xaf\xd9\x88\xe3\x9f\xb2\xbc\xa2\xfc\xa6\x99\x32\x16\x7a\xbf\xa9\x8c\x64\x7d\xef\x2d\x33\xde\x62\x4b\x6a\x09\x72\xe2\xbd\x68\xb6\xe6\xef\x98\x10\xcd\xeb\x91\x77\x70\x83\x63\x14\x10\xa7\x53\x49\x6d\xab\xb3\x96\xb6\x11\x9a\x9d\x9f\x9f\x9f\xe7\xc6\x58\xac\x63\xb0\x8b\xda\xa8\x01\x0e\xc4\x4a\x5d\x98\x8b\x95\x9a\xae\xd3\x67\xd0\x02\x2c\xe4\xd1\xc4\x84\xb1\x58\xf8\x9a\x76\xae\x1c\x98\xd2\xf0\xa1\x66\xf9\x50\x44\xde\x1e\xf8\x04\xac\x66\xe4\x50\x60\x74\xa1\x3e\xdb\x8b\xfa\xec\xff\x00\x75\x84\xec\x46\x41\x84\xee\x24\x05\x67\x78\x6b\x1b\xb0\x52\x66\x57\x68\x1e\x2d\xb6\xb0\x4c\xf9\x23\x1c\xbe\x20\x6c\x1b\x4c\x64\xa2\xa4\xc8\xd5\x37\x8c\xfd\x0e\x78\xb4\xe5\x15\xd6\x36\x90\x4b\x6f\x6f\xdf\x26\x7f\xb7\xb7\x44\x11\x9a\xb5\x02\x25\x71\x04\xf6\xb3\x87\xea\x28\xac\x86\xa3\x35\x1c\x07\x53\xa4\xe7\x5d\x2f\x38\x2c\x24\x8a\x63\x0b\xa5\x93\x5e\x04\xe5\xc6\x0a\x21\x6d\xde\x27\x2a\xa7\xf7\xda\x63\x88\xc6\x2e\x68\xdd\xd2\x98\x09\xd0\xa6\x94\xe6\x35\xc0\x86\x85\xe2\x68\x88\xc7\x80\xdc\x3f\x2c\xc7\x6c\xc9\xc1\x59\x65\xef\x8b\x10\x8b\x85\xd4\x90\x8b\x6e\xa3\xd1\x7d\x76\x80\x6e\x47\x02\x44\x64\x92\x60\x41\x77\xa0\x06\x28\x4c\x8d\x0e\xb9\x24\x98\x87\xcf\x4c\x84\x95\x9b\x54\xb3\xcc\x73\x31\x40\x41\x36\xff\xab\xff\x27\x97\x71\xcc\x44\x1c\x2c\xa5\x4e\xb0\x9d\x5b\xf8\x66\xfd\xc7\x89\xa4\x30\xf7\xfa\x73\xf5\xa0\x58\xd8\x81\x3a\xff\xe9\xe7\xc5\xed\xf5\xed\xd3\xe5\xe3\xcd\xd3\xd5\xc3\xcd\x6f\xd7\x0f\x4f\x77\x17\xb7\xd7\x7f\xf2\xa5\xdd\x01\xf1\xb1\xcc\xa6\xf9\x7f\xce\xca\x53\xe4\x59\x74\xe6\x0c\x3d\xfb\xaf\x2f\x4a\xf0\x2f\x8c\xc3\x3c\x24\xa0\xad\x09\x09\x9e\x12\x5d\xb3\xc8\x8d\xfb\x12\x96\x9b\xa6\xc8\x1a\x36\x4d\x89\x35\x6c\x7c\x09\x43\x56\x40\x53\x0e\xfa\x73\x7e\x7a\x9f\x47\x1f\xb6\x6d\x7a\xeb\x77\xd1\x78\xb6\x8f\x77\x0d\xc5\x7d\x40\x64\x3e\xf6\x45\x64\x7d\xb9\x7e\xb8\xbd\xb9\xbb\xf8\x72\x73\x7f\xf7\xf4\xf9\xfe\xd3\xd3\xe2\xe2\xcb\x3f\x2a\x21\x84\x32\xcc\x53\x88\x50\x48\x21\x0b\x2d\xe8\xa4\xbc\xf2\x77\xf8\xb7\x96\xda\x07\x67\x7b\xa9\x8e\xe8\xaf\x16\xba\xbf\xca\x27\x3f\x2e\x2e\x2e\x5b\x2b\xfc\xa2\x65\x12\x79\x83\x08\x2d\x19\x70\x5a\x52\xc6\xd6\xf8\x02\xdb\x55\x54\xd5\x87\x69\x55\x45\x2b\x59\x96\xe0\x18\x22\x94\x5b\xd3\x0c\xc0\x88\x60\x81\xf5\xa6\x2e\xbb\x48\x39\x5f\x48\xce\xc8\x26\x42\x37\xcb\x3b\x69\x17\x1a\x0c\x88\xdd\x56\x7a\x29\xdb\x88\xe3\xaa\xe3\x6f\xfd\xad\xd2\x64\xd1\x20\x00\xfe\x4a\x3b\x36\x51\x7c\x0c\x90\x54\x33\xbb\xb9\x94\xc2\xc5\xbd\xef\xb6\x06\x4c\xef\x05\xdf\x3c\x48\x99\x47\x9b\xd9\x18\x0b\x49\x84\xac\x4e\x77\x1e\x7b\xfb\x78\x0b\xc6\x38\x97\x72\x90\x3a\x37\xb9\xb8\x28\xba\x75\x9c\xb8\xe6\x41\xe2\x46\xca\xf9\x79\xdc\xb6\xcc\x2f\xd3\x30\xcf\x80\xc9\x0b\x1e\xe8\x54\x5c\x98\x3b\x29\x9c\xfd\x0d\xab\xf3\x47\xbf\x1a\xd0\x05\x0f\xad\x56\xf2\xd9\xfa\xc1\xd6\xd1\x28\x05\x56\x72\xd0\xf5\xf6\x10\x20\x58\x2e\x81\x58\x47\x60\x1e\xcb\x34\xab\x74\xaf\x61\x13\xe5\x77\x70\xf9\x11\xa5\x51\x6d\x13\xec\x0e\x1e\x13\x1f\x26\x6f\xd5\x83\x18\xe4\x28\x68\xa8\x6d\x5f\x31\xd2\xa3\x01\x06\x85\xe4\x0b\x1c\xeb\x0a\x43\x22\xc5\x68\x86\xe5\x1c\x3e\x19\xb7\xea\x58\xfc\xc4\xac\xaa\xd4\xdc\xc5\xa7\xde\x88\x41\x55\x98\x8c\xe0\xde\xef\x88\x0e\x79\x6e\x9c\x8e\x08\xbd\xa0\xe4\x47\xa4\x40\x14\x5c\xa9\xbb\xc5\x02\xc7\xa0\xe7\x3c\x4b\x5e\xc9\x8f\x6a\x10\x06\x28\xd8\xbe\x33\x30\x4f\x05\xfb\x16\x85\x61\x48\x0c\x73\xff\xa6\x46\x92\x75\x93\x16\x31\x3a\xff\xe9\xe7\x7f\xfe\xfa\xf1\xfa\xe9\xee\xfe\xea\xba\x4d\x9d\x8c\x3b\xdf\xb9\xbe\x30\x0f\x33\xac\x43\xce\x9e\xc3\x1e\x84\x6b\x18\x3d\x73\x58\x2d\x40\x13\x10\x16\xc7\x30\x9f\xbd\x96\x0b\xd5\xdd\x19\x4f\x3c\x1a\x97\x0f\xf5\x87\x1e\x2b\x71\xd1\x32\x75\x60\xba\x8a\x7f\x3a\x3a\xd5\x93\xe2\xd9\x44\x1d\xec\xfe\x3f\x14\x49\x52\x9a\x65\x8c\x43\x0c\xb4\x41\x31\x6a\x24\xe3\xfc\x65\xbe\xd4\x85\x58\x1f\xbe\xb4\x4d\x0c\x57\xba\x38\xd8\x50\xf1\x34\x66\xc2\x84\x8d\xca\xea\x15\x2f\x54\xce\xd7\x52\xe1\x38\x57\x1a\xa1\x8f\x8c\x32\x0d\x79\xb9\xc2\xbc\x8d\x8b\x93\xcf\xd3\x3a\xa0\x4c\xf7\x35\xa4\x78\xe1\x62\xac\x4e\x37\xbd\x43\x1b\x85\xac\x35\x85\x42\xd6\x31\xc3\x6c\xda\xb4\xd2\x6c\xba\x74\xd4\x41\x2b\x67\x48\xb2\x86\x3e\x30\x74\xe4\xd4\x08\x30\xdc\x62\x79\x35\xf4\x34\x07\x08\xeb\xb8\x16\x17\xcd\x02\x1e\x94\x7b\x11\x68\x88\x99\xb1\x05\x5b\x0d\x54\xad\xa0\x36\xe3\xe6\x40\xc9\xdc\x5b\xc3\xf3\x04\xc6\x94\x6a\x30\x66\xbe\xbf\xd2\xef\xad\x91\xaf\xae\x4a\x65\x29\x59\x7f\x30\xd3\x98\x68\x17\xe0\x86\xc5\x41\x79\xb4\x0e\xb7\xad\xbd\x2c\x0a\x95\xf7\x3a\xca\x66\xd3\xbf\x4c\xcf\x47\x55\x99\xe6\x5a\x47\x3b\x49\xf5\x3a\x14\x8d\x89\x44\x7f\xd3\x5b\xb3\x6b\x11\x31\x28\xa6\x5e\xde\x73\x27\xb5\x04\x6c\x53\x0d\x41\x8c\x2d\x98\xf9\x17\xa9\x24\x97\xf1\x66\x5e\xf3\xdc\xc9\x15\xfb\x54\x71\xa7\xb6\x80\x71\x25\xd9\x06\xf6\xe0\x0a\x2c\x49\x80\x32\x97\x1a\x95\xcc\x12\x73\x53\x17\xb2\x2c\x01\x99\xda\xf9\xdf\x6b\x7c\x27\xa0\xb0\xc4\x29\xb7\xc1\xd2\xd8\x8d\x82\x39\x7c\xb3\x7f\xab\x3d\x2f\x1b\x42\xe5\x6f\x4e\x00\x66\xdd\xc1\xfd\xe6\xbd\xbf\x4c\x8e\x32\x31\xd6\x1f\x8c\xab\x1c\x39\x91\x6b\x26\x89\xf7\x5b\xe2\x6b\x1a\xf0\xbe\xdf\x26\x07\x75\xe2\xd9\xc9\xaf\x2b\x8e\x96\x64\xfe\x6d\xde\x6e\xa1\x12\xd4\xa2\x66\x0d\xbb\x55\x68\x5d\xc6\xb6\x4e\xff\x2b\x69\x0a\x13\x7d\xfa\xd1\xdd\xfc\x3b\x9b\x4f\xf1\x13\xcb\x55\xde\x6e\xa4\xde\xdc\xeb\xcb\xed\x2b\x6a\x9d\x9e\x8f\x31\xe4\xa9\x2c\x33\x9b\x70\xb0\xfe\x83\x05\x6a\x04\x1c\x5d\x5c\xa8\x9f\x35\x7b\x19\xd0\x30\x5b\xea\x74\xa8\x9f\xda\x06\x09\xea\xa5\xf0\x08\x7b\x7f\x80\x6e\x74\x28\xaf\xd3\xb1\x7e\x5a\xea\x74\xad\x73\xf9\x3a\x77\xeb\x19\xc0\x25\xb7\x6b\x5e\x6f\xd5\x5f\xf3\x0a\xb3\xd9\x33\x58\x5c\xbd\xba\xf0\x78\x73\x55\x9c\x24\x4e\xf3\xfb\xae\xb7\x29\xdb\x63\x3e\xb6\x16\x93\xd5\x03\xfc\x3b\x65\xda\x9d\x26\xb6\xcd\x4b\x49\x7a\x23\x96\xf2\x5e\xe4\x55\xab\xaa\x67\x45\x79\xf8\xcc\x96\x40\x36\x84\xc3\xad\xa4\xdb\xf7\xd3\x16\xd5\x0b\x79\xf9\x9f\xd7\x6a\x05\x09\x68\xcc\x27\xff\x0b\x00\x00\xff\xff\x2a\x5b\x31\x74\x3b\x32\x00\x00") func deployKubernetes117LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -120,12 +121,12 @@ func deployKubernetes117LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 10848, mode: os.FileMode(436), modTime: time.Unix(1610052990, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 12859, mode: os.FileMode(420), modTime: time.Unix(1611067492, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes118DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\x78\xdb\x03\x5a\x01\x7e\xc8\x26\xbe\xd4\xe8\xc6\x31\x92\xec\xbd\x06\x0c\x35\x96\x59\x53\x24\x4b\x8e\xbc\xf1\x15\xfd\xdf\x0b\x52\x92\xad\x2f\xcb\x1f\x97\x0f\x14\xf5\x3e\xac\x43\x71\x38\xbf\xf9\xe0\xcc\x6f\x6c\xff\x91\xdc\x80\x04\x43\x11\x62\xf2\x9d\xe3\x92\x7c\x4a\xe9\x0a\xc8\x2a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x18\x50\xcd\x7f\x05\x63\xb9\x92\x11\x59\x8f\x06\x2b\x2e\xe3\x88\x3c\x80\x59\x73\x06\x97\x8c\xa9\x4c\xe2\x20\x05\xa4\x31\x45\x1a\x0d\x08\x11\xf4\x19\x84\x75\xef\x08\xd1\x29\xa4\x01\xb3\x7c\xc8\x25\x82\x18\x32\x95\x86\x31\x68\xa1\x36\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x20\x44\xd2\x14\xa2\xad\x60\xc0\x94\x44\xa3\x84\x00\x53\x3c\xb3\x9a\xb2\xca\x86\x41\x10\x04\x35\x88\xe6\x99\xb2\x21\xcd\x70\xa9\x0c\xff\x8d\xba\x43\x87\xab\xbf\xda\x21\x57\xe1\x16\xfc\xbd\x12\xf0\x86\x90\xe1\x05\xc1\x48\x2a\xdc\xa6\x35\x77\xa8\xc0\x04\x6c\x91\xec\x31\xc0\x64\x02\x6c\x34\x08\x08\xd5\xfc\xc6\xa8\x4c\x7b\x20\x01\xf9\xf4\x69\x40\x88\x01\xab\x32\xc3\xa0\x58\x03\x19\x6b\xc5\x25\xda\x01\x21\x6b\x30\xcf\xc5\x72\x02\xe8\xff\xff\x4e\x91\x2d\xfd\x3b\xc1\x6d\xbe\x14\x83\x00\x04\xff\x36\xd3\x31\x2d\xde\x32\x03\xee\x6d\x4b\x27\x53\xca\xc4\x5c\x56\xdd\xd6\x06\x21\x80\x5a\x78\x2b\x04\x16\x95\xa1\x09\xec\x55\xce\x2c\x2f\xb6\x30\xaa\x29\xe3\xc8\xf7\x41\xd9\x02\xd8\x61\x2a\x94\x36\xa0\xe8\xed\xf3\x02\xea\x51\xa1\xd0\x2a\xee\x50\xdc\x12\xa5\x5a\xdb\xb6\xb0\x01\x2d\x38\xa3\x16\xba\x22\x79\x46\x46\x5f\x89\xcc\x22\x98\x8f\x48\x6c\x93\x49\x09\xe6\xa4\x2c\xd6\xce\x32\x8b\x20\x71\xad\x44\x96\x9e\x17\xc0\x93\x62\xd5\x50\xc8\x04\xe5\xe9\xf1\x5a\x8b\x5c\x39\x39\x57\xcb\x44\x15\xd4\xee\xbd\x30\x0d\x75\xc7\x55\x81\x35\xb4\x4a\xc0\x89\xd9\xde\xb6\x45\x52\x6d\x97\x0a\x87\x87\x8c\x2a\x42\x56\x6c\xef\xb3\xea\xb5\x74\xb8\x16\xd0\x36\xf8\x80\xaa\xc3\x65\x44\xaa\xf8\x35\x83\x72\xc2\x71\xe7\x75\xac\x2f\x5c\xc6\x5c\x26\x6f\xd9\x6b\x2d\xaf\x5f\x6d\x25\xa0\xaf\x71\x29\x01\xf7\xb0\x70\xda\x4b\xff\xf4\x18\x33\x20\xa4\xd2\x7d\x8f\x6f\x9a\x36\x7b\xfe\x27\x30\xf4\xb5\xa5\x93\x7b\xbc\x0b\x65\xa8\x14\xd8\x8f\x89\xc3\xb9\xde\xae\x76\x86\x93\x0a\xfa\x3b\xf8\xbd\xc9\x26\x3f\x92\x46\x5a\x0d\xcc\x29\xd3\xca\xe0\xb6\xc3\x1b\x8c\xc8\xe8\xe2\xe2\xe2\xc2\xa3\x40\x6a\x12\xc0\x79\x6d\xd5\x82\x00\x86\xca\xe4\x38\xa9\xd6\xfb\x74\x9e\x61\xc4\x07\xba\x2b\x05\x34\x9c\xd9\x33\x7c\x35\xea\xf4\xd5\xe8\xad\x7d\x45\x08\x6e\x34\x44\x64\xa6\x62\x70\x5a\x5b\xce\x73\x3c\x6c\x77\x9d\x1f\x90\x22\x2c\x32\xf1\x00\x1f\x3a\xbb\x94\x8e\x2c\xe9\x60\x44\x46\x2d\x3f\xa5\xae\x67\x7c\xad\xe0\x3a\xe0\xba\xb3\x80\xdb\x3c\xa3\x66\x3d\xf8\x11\x52\x2d\x28\x42\x81\xaa\xe2\x34\x0f\x49\x4a\x85\xbe\x00\x6d\x51\x76\x02\xb1\xcc\x50\x17\x27\x77\x34\xe5\x12\x8c\x2d\x76\x8b\x9a\x85\x07\x6d\x3c\xcb\xca\xbd\x82\xdf\xe1\x79\xa9\xd4\x2a\x22\x3c\x91\xca\x80\xdf\x5a\xc6\xc6\x83\x59\x2c\xb8\xe4\xb8\xd9\xc1\x73\xdd\xfe\xb2\xb5\xea\x22\xf9\xaf\x8c\x1b\x88\xaf\x33\xc3\x65\xf2\xc0\x96\x10\x67\x82\xcb\x64\xea\x0f\x2e\x96\x27\x2f\xc0\x32\x87\xa9\x2a\x99\x9f\xf9\x50\xc4\xfe\x11\x4c\x6a\xeb\x8f\x83\x3c\x15\x26\x2f\xda\x80\xb5\x75\x4f\x97\x3b\x56\xb0\x89\xba\x0c\xec\xf0\x5f\xf9\x52\x1a\x0c\x75\xd9\x46\x66\x0a\xa7\xb2\xf5\x7c\x4d\x45\x06\x2d\x55\x9e\x05\x49\xf5\xa9\x6b\x79\x41\x85\x85\xf2\xc9\x2e\xd0\xe5\x11\x6e\xca\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x33\x97\xe1\x36\xf0\xb1\xe1\xeb\x0a\xe8\x80\x04\xeb\xf1\x9f\xab\x7f\x0a\x95\x24\x5c\x26\xc1\x42\x99\x94\xe2\x18\xe1\x05\xab\x8f\x53\x15\xc3\xb8\xc3\xfa\x80\x04\xe5\x2c\x3d\xce\x24\x7f\x89\xc2\x30\x64\x96\x87\xf5\x64\x1b\x5a\xc5\x56\x55\x19\x03\x09\xb7\x68\x36\x93\x52\x16\x99\x8e\xc2\xf0\x62\xe8\xff\x45\xbb\x5e\x51\xec\x77\x21\xe5\xf1\xf8\x87\x1f\xff\xf1\xed\xcb\xe4\x69\x76\x77\x3d\x79\x9a\x5d\xde\x4e\x7e\xaa\xee\x61\xf4\x17\x2e\x60\x1c\x32\x30\x68\x43\x46\x87\xcc\xd4\x4c\x70\xeb\xd5\x1d\x28\x6c\x73\xcb\x0a\x36\xcd\x1d\x2b\xd8\x54\x77\xe4\x7e\x74\x35\x68\xfc\xc3\x8f\xf3\xdb\xc9\xed\xd3\xd5\xc3\xf4\xe9\xfa\x7e\xfa\xeb\xe4\xbe\x0d\xa9\x28\xff\x5f\xfd\xcc\x34\x8e\x76\x65\xdd\xbd\x40\xae\xab\x91\xcb\x6b\x5e\xdd\xbe\x41\x23\x7d\x7e\x31\x2a\xad\x67\xd0\x82\x83\x88\x0b\x3a\x53\x7d\x35\xfa\x5d\xfd\xa1\x17\x9a\x53\x5c\x46\xfe\x76\x0e\x9d\x77\x5d\xc9\x6a\xa1\x79\x9c\xdc\xdf\x4e\x67\x97\x8f\xd3\xbb\xd9\xd3\xd7\xbb\x9b\xa7\xf9\xe5\xe3\xdf\x9b\x98\x22\x12\x62\xaa\x43\x04\x93\x16\x1f\x78\xb8\x5c\x6a\x1d\xd5\xe5\xab\xf6\x51\xed\x2b\xd7\x3a\xe8\xe6\xee\x7a\xf2\xe5\xdb\x4d\x5b\xf6\xe5\xe7\x8b\xbf\xe5\x75\xe7\x6a\x36\xde\xb9\x99\xa7\x34\x81\x88\xf8\x03\x9b\xf7\x21\x62\x54\x52\xb3\xa9\xef\x9d\x67\x42\xcc\x95\xe0\x6c\x13\x91\xe9\x62\xa6\x70\x6e\xc0\x82\xdc\x25\x4a\xa5\x3b\x35\xae\xd5\xb6\x91\x97\x90\xb7\xb7\x76\xde\xe8\xeb\xd5\x93\x76\x24\x21\x7f\x59\x60\x99\xe1\xb8\xb9\x72\xf3\xda\x0b\xd6\xcb\x22\x8d\xef\xa4\xd8\xdc\x2b\xe5\x73\xd9\x6e\x2c\x42\x1a\x11\x34\xd9\x2e\x7a\x95\x50\xdc\x82\xb5\xce\x24\x1f\xeb\xde\x38\xe5\xa3\xe2\xad\x63\xa4\x35\x0b\x52\xb7\x52\xc8\xfb\x5b\xd1\x82\x5f\xde\x65\x7f\xc1\xf6\x49\x5a\xde\x92\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x5b\xb2\x98\xea\x8a\x44\x40\xa8\x49\x6a\xa8\x9b\xd5\xcd\xc7\x9c\xc6\xb1\xab\xf8\xe3\x63\x0a\x54\xb0\x00\x8a\x99\x81\x20\xa1\x08\x76\xfc\xa8\xb4\x12\x2a\xd9\x8c\x6b\x8e\x76\xfb\xac\x8b\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xe3\x9f\xd3\xda\x72\x0c\x0b\x9a\x09\x0c\x16\xd6\x11\xae\x31\xbc\xe0\x5f\x6a\xcf\x8b\xa4\xd8\x82\xf6\xc5\x63\xd4\xcc\x6a\x37\xa4\x24\xcc\xb8\x11\xcb\xf2\x24\x28\xa6\xf5\xb0\x31\xf8\x44\xeb\xcf\xc3\x8b\xe1\xe7\xb3\xd2\xbc\x6b\xb4\x39\x29\xdf\x47\x6f\x9e\xef\x47\xa5\xee\x09\x09\xb8\x17\x8f\xc9\xe4\xa5\x9d\x29\xe9\xd0\x34\x30\xf8\x47\xdf\x2c\x98\x7c\xa4\xd9\x9e\x54\x9d\xf4\xfa\x58\xa1\x7b\xa1\x12\x8e\x42\x54\x19\x49\x40\x60\xb1\x00\x86\x8e\x54\x14\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x60\x3b\x5c\x65\xcf\x60\x24\x20\xf8\x89\x3b\xa5\x6e\x5c\x1d\x54\x3d\x54\x3d\x35\xd5\xb8\xb9\xe6\x26\x22\x32\x13\xa2\x59\xd9\xf6\xf8\x25\xe8\xbd\xf0\x16\x98\x81\x5a\xf0\xf2\x95\x86\xd5\x5b\xe1\xfc\xa9\xed\x80\xf4\xef\xff\x34\x00\x95\xb7\xbd\x7f\x16\xb9\xa6\x90\x2a\xf9\xb6\x93\x88\xf3\xf6\x81\x19\xe4\xc4\xa9\xa3\x38\xf1\x4c\x78\x1f\x3a\x4c\x54\xa0\xbf\xef\x18\xf1\xea\x5c\x38\x06\x77\x4d\x6f\xa9\xa4\x09\x98\x71\x8e\xf5\x77\x72\xe5\x9a\x73\x7a\x58\x72\xab\xf3\x1c\x45\x75\xb7\xa5\xa3\x41\xa0\x0b\xa9\xf9\xdd\xf5\xd3\x74\xfe\x93\xe7\xd1\xa3\x23\x78\x77\x47\x4d\x6a\x73\xf0\x77\xe1\xd7\x16\x29\x7a\xba\x32\x0e\xd7\xd4\x84\x82\x3f\x87\x47\xf0\xec\xd3\x58\xb9\xb3\x76\x0e\x86\x81\x44\x9a\xc0\x78\x54\x37\xf3\x7f\x9d\xb3\x57\x12\xe0\x1d\xa0\x20\xc5\xcc\x0e\xb5\x8a\x2b\xda\x5e\x9d\xf2\xbf\xe2\x18\xf2\x7f\x3d\x3d\x68\xc3\xd7\x5c\x40\x02\x71\x83\xbd\xd4\xf8\xcb\xc5\xdb\x0f\x12\xe5\xd5\x76\x94\x45\x00\x86\x39\xe3\xb0\x61\x9d\xc2\xd4\x39\x5b\x2e\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xaf\xd4\x5c\x49\x2a\xda\x7e\x71\xfb\x7d\x91\xeb\x99\x2d\x5a\x40\xf2\x2f\xa3\xcf\xd5\xe9\xc4\x7b\xb4\x75\xcf\x4f\x9e\xc1\xf5\xcc\x4e\x31\xac\x5b\x32\x31\xac\x7b\xf4\xd8\x4d\x5b\x8b\xdd\xf4\x22\xeb\xa0\xc7\x07\x07\xb3\xd2\x79\x3d\x37\xf8\x0c\x17\xba\xc3\x7c\x17\x38\x6d\xc0\x2b\x22\x58\x76\xb8\x3c\x2f\x75\xad\x91\x34\xb3\x6d\x4f\xab\xe8\xec\xca\x9d\x03\x64\x7d\x57\x67\x6f\xf8\xdd\x35\xf0\xf0\xa0\xe7\xf3\x27\x2f\x25\x5b\xeb\x4d\xb4\x1e\x0d\x3f\x0f\x2f\xce\xaa\x4d\xcd\xb3\x3e\x7c\x50\x3b\x98\x89\xd5\xa0\xb7\xa4\x6b\x19\xb1\x3b\xa3\xfa\x29\xf1\x0e\x4c\xe1\xd8\x3c\x24\xfb\xa6\xa7\xa5\xb2\xb9\xe6\x6a\x6d\xed\xaf\x6c\xbd\x77\x24\xff\xca\xe5\xda\xdf\x0a\x65\x36\x77\xe6\xaa\xfc\xc5\x41\xaf\x17\xce\x01\xf2\x54\x32\xc0\xf0\x64\xfd\x7b\xfc\x78\x96\x3b\xfa\x0a\xfd\x71\x68\x3a\xcb\xfb\x69\x58\xea\xb5\xfe\x38\xb5\x8d\x0a\x1f\xec\xad\xe1\x47\x8e\xc3\x5e\xb0\x39\x0a\x1f\x61\xc4\x2b\xe4\xd3\x9e\x4a\xdb\xa3\xbc\xde\x89\x8e\xd3\x52\xef\x54\xbd\xc7\xd7\xdb\xd6\x91\x97\xa2\x68\x6b\xcd\x8f\x06\xea\xbf\x5b\x09\xd7\xa3\x67\x40\xba\xfd\x11\xc2\xc3\xf4\x3a\xa7\x5e\x6f\xf6\x49\x41\x25\x2e\xe5\xe4\x4a\x11\x29\x5b\xde\x17\x5f\x6c\x45\xc4\x7f\xbd\xe3\xbf\xfd\x8d\xa7\x72\xa1\xee\xa4\x2f\x8c\xdb\x92\x99\x57\x9d\xaf\x7c\x01\x6c\xc3\x04\xdc\xaa\xb8\xfc\x85\xcc\x7c\xfb\x2b\x2c\xff\xe7\x44\x2f\x21\x05\x43\xc5\xe0\xbf\x01\x00\x00\xff\xff\x99\xe3\x8f\xb5\x93\x2a\x00\x00") +var _deployKubernetes118DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x05\x2a\x2b\xee\x07\xb0\x10\xe0\x87\x5c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8d\x65\xd6\x14\xc9\x92\x23\xef\xfa\x8a\xfe\xef\x05\x25\x59\xa6\x24\x5b\x96\x15\xe7\xa3\x67\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x1f\x3f\xd2\xfa\x23\xf9\x0c\x12\x0c\x45\x88\xc9\x37\x8e\x0b\x72\x96\xd2\x25\x90\x65\x66\x51\xa5\xfc\x37\x38\xfb\x33\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\x59\x4d\x46\x4b\x2e\xe3\x88\x3c\x82\x59\x71\x06\x17\x8c\xa9\x4c\xe2\x28\x05\xa4\x31\x45\x1a\x8d\x08\x11\xf4\x19\x84\x75\xdf\x08\xd1\x29\xa4\x01\xb3\x7c\xcc\x25\x82\x18\x33\x95\x86\x31\x68\xa1\xd6\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x6a\x62\x90\x4f\x0c\x98\x4a\x03\xa6\x24\x1a\x25\x04\x98\x52\xca\x6a\xca\x3c\xd1\x51\x10\x04\x1f\xc6\xd8\x6f\xf0\xbc\x50\x6a\x69\x7b\x9a\x6a\x9e\x29\x1b\xd3\x0c\x17\xca\xf0\xdf\xa8\x5b\x7c\xbc\xfc\x64\xc7\x5c\x85\x95\x13\x0f\x4a\xc0\x9b\x98\x0e\xdf\x11\x8c\xa4\xc2\x89\xaf\xb8\xb3\x0f\x4c\xc0\xe6\xc9\x1e\x57\x4c\x26\xc0\x46\xa3\x80\x50\xcd\x3f\x1b\x95\xe9\xdc\xa4\x80\x9c\x9d\x8d\x08\x31\x60\x55\x66\x18\x94\x63\x20\x63\xad\xb8\x44\x07\xcb\x0a\xcc\x73\x39\x9c\x00\xe6\xff\x7f\xa3\xc8\x16\xf9\x37\xc1\x6d\x31\x14\x83\x00\x84\xfc\x6b\xa6\x63\x5a\x7e\x65\x06\xdc\xd7\x96\x4e\xa6\x94\x89\xb9\xf4\x01\x6c\x1b\x21\x80\x5a\x78\x2d\x0b\x2c\x2a\x43\x13\xd8\xab\x9c\x59\x5e\x8a\x30\xaa\x29\xe3\xc8\xf7\x99\x52\x19\xb0\xb5\xa9\x54\xda\x30\x45\x57\xcf\x4b\x53\x7b\x6d\x85\x56\xf1\x0e\xc5\xad\xa9\x54\x6b\xdb\x9e\x6c\x40\x0b\xce\xa8\x85\x5d\x3b\xf9\x91\x63\x7b\x93\x96\x27\x8a\xe7\xdd\x20\xee\x08\xa4\x01\x98\x5c\x8a\xcc\x22\x98\xf7\x4d\x7b\x93\x49\x09\xe6\x38\x4c\x9c\x8f\x16\x41\xe2\x4a\x89\x2c\x1d\x16\xde\x47\x45\x72\x43\x21\x13\x94\xa7\xfd\xb5\x96\x99\x74\x74\x26\x6f\xd2\x58\x50\xbb\xb7\x9c\x34\xd4\xf5\xab\x91\x2b\x68\x15\xc8\x23\x6b\x41\xdb\x17\x49\xb5\x5d\x28\x1c\x1f\x72\xaa\xdc\xb2\x52\xbc\xcb\xab\x53\xe9\x70\x5d\xbd\xed\xf0\x01\x55\x87\x8b\xac\x54\xf1\x29\x37\xe5\x88\xe5\x3e\x7e\xa6\x57\x45\xf0\x44\xd9\x7d\x14\x3c\x6f\x90\xd1\xfa\x60\x6e\xf7\xcd\xbb\x63\x94\xbc\x84\x15\xb4\x6a\xc9\xb1\x31\x3c\xac\xe5\xfe\xc4\x65\xcc\x65\xf2\x36\xec\xdd\xf2\x7a\x67\x51\x02\xba\xba\xb0\x12\xf0\x00\x73\x67\xc7\x06\xcb\x0e\xb7\x46\x84\x78\x44\x62\x08\xb7\xb5\xd9\xf3\xbf\x80\x61\x9e\x06\x3b\x0f\x0d\xa7\x3c\x98\x7c\xc0\xed\xd9\xd6\x84\x77\xd9\x97\x1a\x2f\x7b\xc9\x5e\x9c\xfe\xdc\xe5\x55\xe7\xf7\xce\x97\xa1\xe8\xfb\x0d\x66\x20\x03\xfc\x68\xf9\xf1\x4e\xbb\x52\x4b\x93\x57\xde\x8e\x66\x9b\x7e\xc3\xb4\x68\xde\x9c\x7c\x8c\xfb\x1d\xab\x81\x39\xb5\x5a\x19\xac\x0e\x60\x06\x23\x32\x39\x3f\x3f\x3f\xcf\xed\x41\x6a\x12\xc0\x59\x6d\xd4\x82\x00\x86\xca\x14\x16\x53\xad\xc7\xcb\xec\x19\x8c\x04\x84\x3c\xa6\xb8\xb4\x48\xa5\xaf\x69\xeb\xcd\x9e\x29\x0d\x1f\x6a\x96\x0f\x00\xe5\x43\xc0\x9f\x02\x1a\xce\xf6\x45\x48\x17\xf6\x93\x9d\xd8\x4f\xfe\x3f\xb0\x27\x04\xd7\x1a\x22\x72\xa7\x62\x70\xb6\xb7\x36\x83\x6a\x6d\xb7\x75\xe7\x11\x29\xc2\x3c\x13\x8f\xb0\xff\x1e\xb1\x6d\x33\x53\xa9\x56\x32\xb7\xa0\x61\xef\x1b\x40\xd2\x96\xd7\xd4\x60\xa0\xe6\xde\xf6\xbe\x5b\x2e\x6f\x6e\x92\x22\x32\x69\x85\x4b\xea\xb8\xed\x17\x0f\xd8\x41\x70\x1d\x0f\xd8\x20\x28\x6c\x91\xaa\x77\xbd\x10\x41\x48\xb5\xa0\x08\xa5\x9f\x5e\x1c\xe5\xf6\x4a\xa9\x30\xef\x1c\x95\xdf\x3b\x4d\xb2\xcc\x50\x17\xba\x6e\x69\xca\x25\x18\x5b\x4a\x8b\x1a\x66\xc7\x05\xe4\x60\x94\x87\xe0\xdc\x3f\x38\x07\xee\xca\xde\x89\x65\x43\x8c\x08\x4f\xa4\x32\x90\x8b\x6e\x62\xd2\x7d\xb6\x98\x6e\x46\x02\xc2\x54\x9a\x52\x19\x6f\x71\x0d\x48\x98\x59\x13\x0a\xc5\xa8\x08\x9f\xb9\x0c\x2b\x4f\x63\xc3\x57\x9e\x97\x01\x09\x56\xd3\xbf\xfa\x7f\x0a\x95\x24\x5c\x26\xc1\x5c\x99\x94\xe2\x14\xe1\x3b\xfa\x8f\x53\x15\xc3\xd4\x6b\xda\xd5\x83\x62\x61\x87\xeb\xf4\x87\x1f\x67\xb7\xd7\xb7\x4f\x97\x8f\x37\x4f\x57\x0f\x37\xbf\x5e\x3f\x3c\xdd\x5d\xdc\x5e\xff\xc9\x97\x76\x27\xc8\xc7\x32\xa7\xa6\xff\x39\x2b\x8f\x99\x67\xd1\x99\x33\xf4\xec\xbf\xbe\x28\xa3\x3f\x73\x01\xd3\x90\x81\x41\x1b\x32\x3a\x66\xa6\x66\x91\x1b\xf7\x25\x50\xd8\xa6\xc8\x12\xd6\x4d\x89\x25\xac\x7d\x09\xcb\x16\x10\x67\x02\xcc\x97\xfc\x78\x3f\x8d\x3e\x6d\x1a\xf7\xc6\xef\xa2\x09\x6d\x1e\x6f\x9b\x8b\xfb\x80\x5c\xf9\xd8\x17\xc1\xf5\xf5\xfa\xe1\xf6\xe6\xee\xe2\xeb\xcd\xfd\xdd\xd3\x97\xfb\xcf\x4f\xb3\x8b\xaf\xff\xa8\x84\x08\x59\x51\x91\x41\x44\xc2\x18\x56\x21\x82\x49\xcb\x1f\x07\x1c\xfe\xad\xa5\x76\xc1\xd9\x5e\xaa\x23\x01\xaa\x85\xee\xaf\xf2\xc9\x8f\xb3\x8b\xcb\xd6\x0a\x3f\x1b\x95\x46\xde\x20\x21\x73\x0e\x22\x2e\x79\x64\x6b\x7c\x46\x71\x11\x55\x25\x62\x5c\xd5\xd2\x4a\x96\xa7\x34\x81\x88\xe4\xd6\x34\x03\x30\x62\x54\x52\xb3\xae\xcb\xce\x32\x21\x66\x4a\x70\xb6\x8e\xc8\xcd\xfc\x4e\xe1\xcc\x80\x05\xb9\xdd\x4a\x2f\x6b\x1b\x71\x5c\x75\xff\x8d\xbf\x55\x9a\xcc\x1a\x64\xc0\x5f\x69\xcb\x2c\x8a\x8f\x05\x96\x19\x8e\xeb\x4b\x25\x5d\xdc\xfb\x6e\x1b\xa0\xf1\xbd\x14\xeb\x07\xa5\xf2\x68\xb3\x6b\x8b\x90\x46\x04\x4d\xb6\xf5\xd8\xdb\xc7\x5b\xb0\xd6\xb9\x94\x83\xd4\xb9\xc9\xc5\x4d\xd2\xad\x23\xca\x35\x0f\x52\x37\x52\xce\xcf\xe3\xb6\x65\x7e\x99\x86\x79\x06\x8c\x0e\x78\x60\x32\x79\x61\xef\x94\x74\xf6\x37\xac\xce\x1f\xfd\x62\xc1\x14\xcc\xb4\x5a\xc9\xa7\xf0\x7b\xbb\x47\xa3\x14\xa0\x12\x60\xea\x1d\x22\x20\x30\x9f\x03\x43\xc7\x64\x1e\xcb\x34\xab\x74\x2f\x61\x1d\xe5\x97\x74\xf9\xb9\xa5\x51\x70\x53\xea\x4e\x23\x23\x1f\x26\x6f\xd5\xbd\x18\xe4\x28\x18\xa8\x6d\x5f\x31\xd2\xa3\x07\x06\x85\xe4\x01\xb2\x75\x45\x21\x55\x72\x30\xd5\x72\x0e\xbf\x1a\xc9\xea\x58\xfc\xf5\xe9\x55\xa9\xbc\x8b\x58\xbd\x13\x95\xaa\x60\x19\x46\xc5\x3f\x10\x2f\xf2\x3c\x79\x3d\x46\x74\x40\xc9\xef\x94\x0b\xc5\xe0\x6a\xde\x2d\x95\x34\x01\x33\x2d\x6c\x7d\x21\x57\xaa\x01\x19\x90\x60\xf3\xa6\xc1\x34\x93\xfc\x7b\x14\x86\x21\xb3\xdc\xfd\x1b\x5b\xc5\x96\x4d\x8a\xc4\xe3\xe9\x0f\x3f\xfe\xf3\x97\x9f\xae\x9f\xee\xee\xaf\xae\xdb\x34\xca\xba\x43\x9f\xeb\x11\xd3\x70\x45\x4d\x28\xf8\x73\xd8\x83\x7c\x1d\x47\xd5\x1c\x5c\x33\x30\x0c\x24\xd2\x04\xa6\x93\x97\xf2\xa2\xba\x3b\xc3\x49\x48\xe3\x5e\xa2\xfe\xd0\x63\x28\x2e\x60\xc6\x0e\x4c\x57\xfd\x5f\x8f\x5a\xf5\xa4\x7b\x98\xea\xbd\x4c\xe0\x77\x45\x98\xb4\xe1\x2b\x2e\x20\x81\xb8\x41\x37\x6a\x84\xe3\xfc\x30\x77\xea\x42\xac\x0f\x77\xda\x24\x86\x2b\x60\x02\x30\xd4\x22\x4b\xb8\xb4\x61\xa3\xbe\x7a\x25\x8c\x94\xf3\x8d\xd2\x34\xc9\x95\x46\xe4\x27\x5e\x54\x03\xae\x24\x15\x6d\x5c\x9c\x7c\x9e\xd6\x41\xcc\x4d\x5f\x43\x8a\x77\x34\x86\xea\x74\xd3\x3b\xb4\xc5\xb0\x6a\x4d\x89\x61\xd5\x31\xc3\xae\xdb\x14\xd3\xae\xbb\x74\xd4\x41\x2b\x67\x28\xb6\x84\x3e\x30\x74\xe4\xd4\x00\x30\xdc\x62\x79\x35\xf4\x34\x07\x84\x9a\xa4\x16\x17\xcd\x02\x1e\x94\x7b\x11\x18\x48\xb8\xc5\x82\xb9\x06\xba\x56\x50\x9b\x71\xb3\xa7\x64\xee\xac\xe1\x79\x02\xd3\x38\x36\x60\xed\x74\x77\xa5\xdf\x59\x23\x5f\x5c\x95\xca\x52\xb2\xfc\x64\xc7\x09\x33\x2e\xc0\x2d\x4f\x82\xf2\x98\x1d\x6e\x1a\x7c\x59\x14\x2a\xef\x4d\xb4\x9a\x8c\xff\x32\x3e\x1f\x54\x65\x9a\x6b\x9d\xec\x54\xd5\xeb\x80\x34\x24\x12\xfd\x4d\x6f\xcd\xae\x45\xc4\x51\x31\x75\x78\xcf\x9d\xd4\x1c\x28\x66\x06\x82\x84\x22\xd8\xe9\x57\xa5\x95\x50\xc9\x7a\x5a\xf3\xdc\xc9\x15\xfb\x54\x31\xa8\xb6\x80\x75\x25\x19\x03\xdc\xbb\x02\x4f\x53\x88\xb9\x4b\x8d\x4a\x66\x4e\x85\xad\x0b\x21\x4f\x41\x65\x38\xfd\x7b\x5a\x1b\x8e\x61\x4e\x33\x81\xc1\xdc\xe2\x5a\xc3\x14\xbe\xe3\xdf\x6a\xcf\xcb\x86\x50\xf9\x9b\x13\x80\x49\x77\x70\xbf\x7b\xef\x2f\x93\xa3\x4c\x8c\xe5\x27\xeb\x2a\x47\x4e\xe4\x9a\x49\xe2\xfd\xde\xf8\x92\x06\xbc\xeb\xf7\xcb\xa3\x3a\xf1\xe4\xd5\xaf\x2e\x4e\x96\x64\xfe\xcd\xde\x76\xa1\x12\xd4\xa2\x66\x1d\x77\xc3\xd0\xba\x9b\x6d\xdd\x04\x2c\x94\x2d\x4c\xf4\xe9\x47\x77\xf3\xef\x6c\x3e\xc5\xef\x2e\x57\x79\xbb\x51\x66\x7d\x6f\x2e\x37\xaf\xb4\x75\x7a\x3e\xc4\x90\xa7\xb2\xcc\xac\xc3\xa3\xf5\xef\x2d\x50\x03\xe0\xe8\xe2\x42\xfd\xac\xd9\xc9\x80\x8e\xb3\xa5\x4e\x87\xfa\xa9\x6d\x90\xa0\x5e\x0a\x4f\xb0\xf7\x7b\xe8\x46\x87\xf2\x3a\x1d\xeb\xa7\xa5\x4e\xd7\x3a\x97\xaf\x73\xb7\x9e\x01\x5c\x72\xbb\xe6\x55\x57\xfd\x9d\xb0\x70\x35\x79\x06\xa4\xd5\xeb\x0d\x8f\x37\x57\xc5\x49\xe2\xd5\x7e\xfa\xf5\xf6\x65\x73\xd8\xa7\x88\x94\x2d\x1e\xe0\xdf\x19\x37\xee\x40\xb1\xe9\x5f\x5a\xc5\x37\x72\xae\xee\x65\x5e\xb8\xaa\x92\x56\x54\x88\x2f\x7c\x0e\x6c\xcd\x04\xdc\xaa\x78\xf3\x3e\xdb\xac\x7a\x81\x2f\xff\xf3\x5a\x2f\x20\x05\x43\xc5\xe8\x7f\x01\x00\x00\xff\xff\x5b\xc9\x12\x9a\x7d\x32\x00\x00") func deployKubernetes118DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -140,12 +141,12 @@ func deployKubernetes118DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 10899, mode: os.FileMode(436), modTime: time.Unix(1610052999, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 12925, mode: os.FileMode(420), modTime: time.Unix(1611067496, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes118LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\x3c\xed\x02\xad\x00\x3f\x64\x12\x6f\x6a\x74\x92\x18\x49\x66\x5f\x03\x86\x3a\x96\x59\x53\x24\x4b\x1e\x69\xe2\x2d\xfa\xdf\x0b\x52\x92\xad\x9b\x15\xdb\x9b\x0b\x8a\x7a\x1e\xc6\xa1\x78\x78\x3e\x9e\xeb\x77\x6c\xff\x91\x5c\x81\x04\x43\x11\x62\xf2\x9d\xe3\x8a\x7c\x4a\xe9\x1a\xc8\x3a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\xc9\x27\xa3\x35\x97\x71\x44\xee\xc1\xe4\x9c\xc1\x39\x63\x2a\x93\x38\x4a\x01\x69\x4c\x91\x46\x23\x42\x04\x7d\x02\x61\xdd\x3b\x42\x74\x0a\x69\xc0\x2c\x1f\x73\x89\x20\xc6\x4c\xa5\x61\x0c\x5a\xa8\x4d\x0a\x12\x23\x22\xf2\x34\xd0\x46\xc5\x19\x43\xae\xe4\x88\x10\x49\x53\x88\xb6\x52\x01\x53\x12\x8d\x12\x02\x4c\xf9\xcc\x6a\xca\x6a\x1b\x46\x41\x10\x34\xf0\x99\x27\xca\xc6\x34\xc3\x95\x32\xfc\x37\xea\x0e\x1d\xaf\xff\x6a\xc7\x5c\x85\x5b\xe4\x77\x4a\xc0\x5b\xe1\x85\x67\x04\x23\xa9\x70\x9b\x72\xee\x20\x81\x09\xd8\x32\xd9\x83\xde\x64\x02\x6c\x34\x0a\x08\xd5\xfc\xca\xa8\x4c\x7b\x14\x01\xf9\xf4\x69\x44\x88\x01\xab\x32\xc3\xa0\x5c\x03\x19\x6b\xc5\x25\xda\x11\x21\x39\x98\xa7\x72\x39\x01\xf4\xff\x7f\xa7\xc8\x56\xfe\x9d\xe0\xb6\x58\x8a\x41\x00\x82\x7f\x9b\xe9\x98\x96\x6f\x99\x01\xf7\xb6\xa3\x93\x29\x65\x62\x2e\xeb\x36\xeb\x82\x10\x40\x2d\xbc\x15\x02\x8b\xca\xd0\x04\xf6\x2a\x67\x96\x97\x5b\x18\xd5\x94\x71\xe4\xfb\xa0\x6c\x01\xec\x30\x95\x4a\x5b\x50\xf4\xf6\x79\x09\xf5\x20\x57\x68\x15\xf7\x28\xee\x88\x52\xad\x6d\x57\xd8\x80\x16\x9c\x51\x0b\x7d\x9e\x3c\x21\x9c\x2f\x44\x66\x11\xcc\xbb\x47\xb5\xc9\xa4\x04\x73\x54\x08\x6b\x77\x2d\x8b\x20\x31\x57\x22\x4b\x4f\xf3\xde\x51\x8e\x6a\x29\x64\x82\xf2\xf4\x70\xad\x65\xa0\x1c\x1d\xa8\x55\x94\x0a\x6a\xf7\x66\x4b\x4b\xdd\x61\x25\x20\x87\x4e\xfe\x1f\x19\xea\xdd\xbb\x48\xaa\xed\x4a\xe1\xf8\xa5\x4b\x95\x2e\x2b\xb7\x0f\xdd\xea\xb5\x74\xb8\xe2\xdf\xbd\xf0\x0b\xaa\x5e\xae\x21\x52\xc5\xaf\xe9\x94\x23\x8e\x3b\xad\x57\x7d\xe1\x32\xe6\x32\x79\xb3\x16\x6b\x79\x33\xaf\x95\x80\xa1\x96\xa5\x04\xdc\xc1\xd2\xa9\xae\x8c\x33\x70\x93\x11\x21\xb5\xa6\x7b\x78\xbb\xb4\xd9\xd3\x3f\x81\xa1\x2f\x2c\xbd\x7c\xe3\x5d\x98\x42\xad\xb4\x7e\x80\x13\x4e\x35\x75\xbd\x21\x1c\x55\xca\xdf\xc1\xe8\x6d\xfa\xf8\x61\xbc\xd1\x6a\x60\x4e\x93\x56\x06\xb7\x5d\xdd\x60\x44\x26\x67\x67\x67\x67\x1e\x02\x52\x93\x00\x2e\x1a\xab\x16\x04\x30\x54\xa6\x00\x49\xb5\xde\xa7\xf3\xd8\x1b\x7c\x94\xa1\x52\x40\xc3\x99\x3d\xc1\x4a\x93\x5e\x2b\x4d\xde\xd4\x4a\x84\xe0\x46\x43\x44\x6e\x54\x0c\x4e\x65\xc7\x6c\x8e\x72\xed\xf2\xf7\x1e\x29\xc2\x32\x13\xf7\xf0\x71\x03\x4a\x65\xc2\x8a\xf6\x45\x64\xd2\xb1\x50\xea\xda\xc3\xd7\x1a\xa8\x17\x8c\x76\x3c\x6a\x5b\x44\xd1\xcd\x00\x78\x84\x54\x0b\x8a\x50\x42\xaa\x99\xcb\xe3\x91\x52\xa1\x2f\x37\x5b\x88\xbd\x28\x2c\x33\xd4\x79\xc8\x1d\x4d\xb9\x04\x63\xcb\xdd\xa2\x71\xbd\x17\x2f\x78\xfc\x15\xf7\x4a\x7d\x87\xa7\x95\x52\xeb\x88\xf0\x44\x2a\x03\x7e\x6b\xe5\x15\x8f\x64\xb9\xe4\x92\xe3\x66\x87\xcd\xb5\xf4\xf3\xce\xaa\xf3\xe1\xbf\x32\x6e\x20\xbe\xcc\x0c\x97\xc9\x3d\x5b\x41\x9c\x09\x2e\x93\xb9\x3f\xb8\x5c\x9e\x3d\x03\xcb\x1c\xa6\xba\x64\x71\xe6\x7d\xe9\xf5\x07\x30\xa9\x6d\x3e\x0e\x8a\x20\x98\x3d\x6b\x03\xd6\x36\xcd\x5c\xed\x58\xc3\x26\xea\xbb\x60\x8f\xf1\xaa\x97\xd2\x6e\xd0\x57\xc6\xa5\x0c\xce\x65\xe7\x79\x4e\x45\x06\x1d\x55\x9e\xea\x48\xf5\xa9\x6f\x79\x49\x85\x85\xea\xc9\xce\xcb\xd5\x11\x6e\x8e\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x13\x97\xe1\xd6\xeb\xb1\xe1\x79\x0d\x74\x40\x82\x7c\xfa\xe7\xfa\x9f\x42\x25\x09\x97\x49\xb0\x54\x26\xa5\x38\x45\x78\xc6\xfa\xe3\x54\xc5\x30\xed\xb9\x7d\x40\x82\x6a\x5a\x9e\x66\x92\x3f\x47\x61\x18\x32\xcb\xc3\x66\xa4\x8d\xad\x62\xeb\xba\x8c\x81\x84\x5b\x34\x9b\x59\x25\x8b\x4c\x47\x61\x78\x36\xf6\xff\xa2\x5d\x67\x28\xf7\x3b\x97\xf2\x78\xfa\xc3\x8f\xff\xf8\xf6\x65\xf6\x78\x73\x7b\x39\x7b\xbc\x39\xbf\x9e\xfd\x54\xdf\xc3\xe8\x2f\x5c\xc0\x34\x64\x60\xd0\x86\x8c\x8e\x99\x69\x5c\xc1\xad\xd7\x77\xa0\xb0\xed\x2d\x6b\xd8\xb4\x77\xac\x61\x53\xdf\x51\xd8\xd1\x55\x9f\xe9\x0f\x3f\x2e\xae\x67\xd7\x8f\x17\xf7\xf3\xc7\xcb\xbb\xf9\xaf\xb3\xbb\x2e\xa4\xb2\xe4\x7f\xf5\x83\xd1\x34\xda\x95\x72\xf7\x02\x99\xd7\x3d\x57\x54\xbb\xe6\xfd\x46\xad\xf0\xf9\xc5\xa8\xb4\x19\x41\x4b\x0e\x22\x2e\x99\x4b\xfd\xd5\x6a\x70\xcd\x87\x5e\x68\x41\x71\x15\xf9\xec\x1c\x3b\xeb\xba\x7a\xd5\x41\xf3\x30\xbb\xbb\x9e\xdf\x9c\x3f\xcc\x6f\x6f\x1e\xbf\xde\x5e\x3d\x2e\xce\x1f\xfe\xde\xc6\x14\x91\x10\x53\x1d\x22\x98\xb4\xfc\x48\xc3\xc5\x52\xe7\xa8\x3e\x5b\x75\x8f\xea\xa6\x5c\xe7\xa0\xab\xdb\xcb\xd9\x97\x6f\x57\x5d\xd9\xe7\x9f\xcf\xfe\x56\xd4\x9d\x8b\x9b\xe9\xce\xcc\x3c\xa5\x09\x44\xc4\x1f\xd8\xce\x87\x88\x51\x49\xcd\xa6\xb9\x77\x91\x09\xb1\x50\x82\xb3\x4d\x44\xe6\xcb\x1b\x85\x0b\x03\x16\xe4\x2e\x50\x6a\x7d\xa9\x95\x56\xdb\xe6\x5d\x41\xde\x66\xed\xa2\xd5\xcb\xeb\x27\xed\x88\x41\xf1\xb2\xc0\x32\xc3\x71\x73\xe1\x86\xb2\x67\x6c\x96\x45\x1a\xdf\x4a\xb1\xb9\x53\xca\xc7\xb2\xdd\x58\x84\x34\x22\x68\xb2\x9d\xf7\x6a\xae\xb8\x06\x6b\xdd\x95\xbc\xaf\x07\xfd\x54\xcc\x83\xd7\x8e\x7c\x36\x6e\x90\xba\x95\x52\xde\x67\x45\x07\x7e\x95\xcb\x3e\xc1\xf6\x49\x5a\xde\x91\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x3b\xb2\x98\xea\x9a\x44\x40\xa8\x49\x1a\xa8\xdb\xd5\xcd\xfb\x9c\xc6\xb1\xab\xf8\xd3\x43\x0a\x54\xb0\x04\x8a\x99\x81\x20\xa1\x08\x76\xfa\xa0\xb4\x12\x2a\xd9\x4c\x1b\x86\x76\xfb\xac\xf3\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xd3\x9f\xd3\xc6\x72\x0c\x4b\x9a\x09\x0c\x96\xd6\xf1\xac\x29\x3c\xe3\x5f\x1a\xcf\xcb\xa0\xd8\x82\xf6\xc5\x63\xd2\x8e\x6a\x37\x8f\x24\xcc\xb8\x51\xca\xf2\x24\x28\x47\xf2\xb0\x35\xe3\x44\xf9\xe7\xf1\xd9\xf8\xf3\x49\x61\xde\x37\xc5\x1c\x15\xef\x93\x37\x8f\xf7\x83\x42\xf7\x88\x00\xdc\x8b\xc7\x64\xf2\xdc\xde\x28\xe9\xd0\xb4\x30\xf8\x47\xdf\x2c\x98\x62\x80\xd9\x9e\x54\x1f\xea\x86\x28\xa1\x7b\xa1\x12\x8e\x42\xd4\x19\x49\x40\x60\xb9\x04\x86\x8e\x54\x94\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x0c\x3b\x5e\x67\x4f\x60\x24\x20\xf8\xc9\x3a\xa5\x6e\x32\x1d\xd5\x2d\x54\x3f\x35\xd5\xb8\xb9\xe4\x26\x22\x32\x13\xa2\x5d\xd9\xf6\xd8\x25\x18\x4c\x78\x0b\xcc\x40\xc3\x79\xc5\x4a\xeb\xd6\x5b\xe1\xe2\xa9\xed\x81\xf4\xef\xff\xb4\x00\x55\xd9\x3e\x3c\x82\x5c\x52\x48\x95\x7c\xc3\x01\xc4\x99\xfa\x85\xd1\xe3\xc8\x61\xa3\x3c\xf1\x14\x6c\x1f\x3a\x43\xd4\x70\xbf\xe3\xf4\xf0\xea\x14\x38\x06\x97\x9d\xd7\x54\xd2\x04\xcc\x54\xe4\xe9\xef\xe4\xc7\x0d\xb3\x0c\x30\xe3\x4e\xb7\x39\x88\xde\x6e\xcb\x45\x8b\x34\x97\x52\x8b\xdb\xcb\xc7\xf9\xe2\x27\xcf\x9d\x27\x07\x70\xed\x9e\x3a\xd4\xe5\xdd\xef\xc2\xa9\x2d\x52\xf4\x14\x65\x1a\xe6\xd4\x84\x82\x3f\x85\x07\x70\xeb\xe3\x98\xb8\xbb\xed\x02\x0c\x03\x89\x34\x81\xe9\xa4\x79\xcd\xff\x75\x9e\x5e\x0b\x80\x77\x80\x82\x14\x33\x3b\xd6\x2a\xae\x69\x7b\x75\x9a\xff\x8a\xa3\xc7\xff\xf5\xc4\xa0\x0d\xcf\xb9\x80\x04\xe2\x16\x63\x69\x70\x96\xb3\xb7\x1f\x1e\xaa\xd4\x76\x34\x45\x00\x86\x05\xcb\xb0\x61\x93\xb6\x34\x79\x5a\x21\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xe6\x06\x7c\x2b\xa1\xa2\x6b\x17\xb7\xdf\x17\xb9\x81\x79\xa2\x03\xa4\xf8\x8a\xf9\x54\x9d\x4e\x7c\x40\x5b\xff\xcc\xe4\x59\xdb\xc0\xbc\x14\x43\xde\x91\x89\x21\x1f\xd0\x63\x37\x5d\x2d\x76\x33\x88\xac\x87\x12\xbf\x38\x8c\x55\xc6\x1b\xc8\xe0\x13\x4c\xe8\x0e\xf3\x5d\xe0\xb8\xa1\xae\xf4\x60\xd5\xe1\x8a\xb8\xd4\x8d\x46\xd2\x8e\xb6\x3d\xad\xa2\xb7\x2b\xf7\x0e\x8d\xcd\x5d\xbd\xbd\xe1\x77\xd7\xc0\x97\x87\x3b\x1f\x3f\x45\x29\xd9\xde\xde\x44\xf9\x64\xfc\x79\x7c\x76\x52\x6d\x6a\x9f\xf5\xe1\xc3\xd9\x8b\x91\x58\x77\x7a\x47\xba\x11\x11\xbb\x33\xea\x9f\x0c\xef\xc0\x94\x86\x2d\x5c\xb2\x6f\x62\x5a\x29\x5b\x68\xae\xd7\xd6\xe1\xca\x36\x98\x23\xc5\xb7\x2b\x97\x3e\x2b\x94\xd9\xdc\x9a\x8b\xea\xa7\x04\x83\x56\x38\x05\xc8\x63\xc5\x00\xc3\xa3\xf5\xef\xb1\xe3\x49\xe6\x18\x2a\xf4\x87\xa1\xe9\x2d\xef\xc7\x61\x69\xd6\xfa\xc3\xd4\xb6\x2a\x7c\xb0\xb7\x86\x1f\x38\x02\x7b\xc1\xf6\xf8\x7b\xc0\x25\x5e\x21\x9e\xf6\x54\xda\x01\xe5\xcd\x4e\x74\x98\x96\x66\xa7\x1a\x3c\xbe\xd9\xb6\x0e\x4c\x8a\xb2\xad\xb5\x3f\x0e\x68\xfe\x20\x25\xcc\x27\x4f\x80\x74\xfb\x03\x83\xfb\xf9\x65\x41\xbd\xde\xe6\xd3\x81\x9a\x53\xaa\x99\x95\x22\x52\xb6\xba\x2b\xbf\xc9\x8a\x88\xff\x3e\xc7\x7f\xc5\x1b\xcf\xe5\x52\xdd\x4a\x5f\x15\xb7\xf5\xb2\x28\x39\x5f\xf9\x12\xd8\x86\x09\xb8\x56\x71\xf5\xbb\x97\xc5\xf6\xb7\x55\xfe\xcf\x99\x5e\x41\x0a\x86\x8a\xd1\x7f\x03\x00\x00\xff\xff\x3a\x78\x18\xc6\x60\x2a\x00\x00") +var _deployKubernetes118LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xd3\x7b\xb8\x02\x95\x15\xf7\x0f\xb0\x10\xe0\x87\x6c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8e\x65\xd6\x14\xc9\x92\x94\x76\x7d\x45\xbf\x7b\x41\x49\x96\x29\xc9\x56\x24\xc5\xde\xa4\x6b\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x7f\x7e\xa4\xf5\x47\xf4\x09\x04\x68\x6c\x81\xa2\xaf\xcc\xae\xd0\x59\x82\xd7\x80\xd6\xa9\xb1\x32\x61\xbf\xc3\xd9\x9f\x11\x95\x48\x48\x8b\x80\x32\xfb\x87\xc9\x04\x2b\xf6\x1b\x68\xc3\xa4\x88\x50\x36\x9b\xac\x99\xa0\x11\x7a\x04\x9d\x31\x02\x17\x84\xc8\x54\xd8\x49\x02\x16\x53\x6c\x71\x34\x41\x88\xe3\x67\xe0\xc6\x7d\x43\x48\x25\x90\x04\xc4\xb0\x29\x13\x16\xf8\x94\xc8\x24\xa4\xa0\xb8\xdc\x24\x20\x6c\x84\x78\x96\x04\x4a\x4b\x9a\x12\xcb\xa4\x98\x20\x24\x70\x02\x51\x35\x2b\xc8\x67\x05\x44\x26\x01\x91\xc2\x6a\xc9\x39\xe8\x52\xca\x28\x4c\x3c\xd1\x49\x10\x04\xef\xc3\xd2\xaf\xf0\xbc\x92\x72\x6d\x7a\xda\xa9\x9f\x31\x99\xe2\xd4\xae\xa4\x66\xbf\x63\xb7\xf8\x74\xfd\xc1\x4c\x99\x0c\x2b\x0f\x1e\x24\x87\xd3\xdb\x0d\xdf\x2c\x68\x81\xb9\x13\xcf\x98\x33\x0e\x74\x40\x96\xf1\x01\x3f\x74\xca\xc1\x44\x93\x00\x61\xc5\x3e\x69\x99\xaa\xdc\x9e\x00\x9d\x9d\x4d\x10\xd2\x60\x64\xaa\x09\x94\x63\x20\xa8\x92\x4c\x58\x87\x49\x06\xfa\xb9\x1c\x8e\xc1\xe6\xff\x7f\xc5\x96\xac\xf2\x6f\x9c\x99\x62\x88\x02\x07\x0b\xf9\xd7\x54\x51\x5c\x7e\x25\x1a\xdc\xd7\x96\x4e\x22\xa5\xa6\x4c\xf8\xe8\xb5\x8d\xe0\x80\x0d\x9c\xca\x02\x63\xa5\xc6\x31\x1c\x54\x4e\x0c\x2b\x45\x08\x56\x98\x30\xcb\x0e\x99\x52\x19\xb0\xb3\xa9\x54\xda\x30\x45\x55\xcf\x4b\x53\x7b\x6d\x85\x92\x74\x8f\xe2\xd6\x54\xac\x94\x69\x4f\xd6\xa0\x38\x23\xd8\xc0\xbe\x9d\x7c\xb7\x81\xbd\x4d\xc8\x23\x05\xf3\x7e\x04\xf7\x44\xd1\x08\x40\x2e\x79\x6a\x2c\xe8\x37\x4c\x78\x9d\x0a\x01\x7a\x18\x20\xce\x41\x63\x41\xd8\x4c\xf2\x34\x19\x17\xd8\x83\x62\xb8\xa1\x90\x70\xcc\x92\xfe\x5a\xcb\x1c\x1a\x9c\xc3\xdb\x04\xe6\xd8\x1c\x2c\x24\x0d\x75\xfd\xaa\x63\x06\xad\xd2\x38\xb0\x0a\xb4\x7d\x11\x58\x99\x95\xb4\xd3\x97\x9c\x2a\xb7\xac\x14\xef\xf2\xea\x58\x3a\x5c\x27\x6f\x3b\xfc\x82\xaa\x97\xcb\xab\x90\xf4\x98\x9b\x32\x60\xb9\x77\x9e\xe6\x55\xf9\x3b\x52\x6a\x0f\xc2\xe6\x3b\xa4\xb3\x7a\x31\xb1\xfb\x26\xdd\x10\x25\xaf\x21\x03\xad\x42\x32\x34\x80\xc7\x75\xda\x8f\x4c\x50\x26\xe2\xef\xc0\xd5\x0d\xab\xf7\x14\xc9\xa1\xab\xf9\x4a\x0e\x0f\xb0\x74\x46\x6c\x81\xec\xf0\x69\x82\x90\x47\x1e\xc6\xf0\x59\x93\x3e\xff\x0b\x88\xcd\x73\x60\xef\x11\xe1\x98\xc7\x90\xf7\xb6\x37\xbb\x6a\xf0\x26\x9b\x52\xe3\x62\xaf\xd9\x88\xe3\x9f\xb2\xbc\xa2\xfc\xa6\x99\x32\x16\x7a\xbf\xa9\x8c\x64\x7d\xef\x2d\x33\xde\x62\x4b\x6a\x09\x72\xe2\xbd\x68\xb6\xe6\xef\x98\x10\xcd\xeb\x91\x77\x70\x83\x63\x14\x10\xa7\x53\x49\x6d\xab\xb3\x96\xb6\x11\x9a\x9d\x9f\x9f\x9f\xe7\xc6\x58\xac\x63\xb0\x8b\xda\xa8\x01\x0e\xc4\x4a\x5d\x98\x8b\x95\x9a\xae\xd3\x67\xd0\x02\x2c\xe4\xd1\xc4\x84\xb1\x58\xf8\x9a\x76\xae\x1c\x98\xd2\xf0\xa1\x66\xf9\x50\x44\xde\x1e\xf8\x04\xac\x66\xe4\x50\x60\x74\xa1\x3e\xdb\x8b\xfa\xec\xff\x00\x75\x84\xec\x46\x41\x84\xee\x24\x05\x67\x78\x6b\x1b\xb0\x52\x66\x57\x68\x1e\x2d\xb6\xb0\x4c\xf9\x23\x1c\xbe\x20\x6c\x1b\x4c\x64\xa2\xa4\xc8\xd5\x37\x8c\xfd\x0e\x78\xb4\xe5\x15\xd6\x36\x90\x4b\x6f\x6f\xdf\x26\x7f\xb7\xb7\x44\x11\x9a\xb5\x02\x25\x71\x04\xf6\xb3\x87\xea\x28\xac\x86\xa3\x35\x1c\x07\x53\xa4\xe7\x5d\x2f\x38\x2c\x24\x8a\x63\x0b\xa5\x93\x5e\x04\xe5\xc6\x0a\x21\x6d\xde\x27\x2a\xa7\xf7\xda\x63\x88\xc6\x2e\x68\xdd\xd2\x98\x09\xd0\xa6\x94\xe6\x35\xc0\x86\x85\xe2\x68\x88\xc7\x80\xdc\x3f\x2c\xc7\x6c\xc9\xc1\x59\x65\xef\x8b\x10\x8b\x85\xd4\x90\x8b\x6e\xa3\xd1\x7d\x76\x80\x6e\x47\x02\x44\x64\x92\x60\x41\x77\xa0\x06\x28\x4c\x8d\x0e\xb9\x24\x98\x87\xcf\x4c\x84\x95\x9b\x54\xb3\xcc\x73\x31\x40\x41\x36\xff\xab\xff\x27\x97\x71\xcc\x44\x1c\x2c\xa5\x4e\xb0\x9d\x5b\xf8\x66\xfd\xc7\x89\xa4\x30\xf7\xfa\x73\xf5\xa0\x58\xd8\x81\x3a\xff\xe9\xe7\xc5\xed\xf5\xed\xd3\xe5\xe3\xcd\xd3\xd5\xc3\xcd\x6f\xd7\x0f\x4f\x77\x17\xb7\xd7\x7f\xf2\xa5\xdd\x01\xf1\xb1\xcc\xa6\xf9\x7f\xce\xca\x53\xe4\x59\x74\xe6\x0c\x3d\xfb\xaf\x2f\x4a\xf0\x2f\x8c\xc3\x3c\x24\xa0\xad\x09\x09\x9e\x12\x5d\xb3\xc8\x8d\xfb\x12\x96\x9b\xa6\xc8\x1a\x36\x4d\x89\x35\x6c\x7c\x09\x43\x56\x40\x53\x0e\xfa\x73\x7e\x7a\x9f\x47\x1f\xb6\x6d\x7a\xeb\x77\xd1\x78\xb6\x8f\x77\x0d\xc5\x7d\x40\x64\x3e\xf6\x45\x64\x7d\xb9\x7e\xb8\xbd\xb9\xbb\xf8\x72\x73\x7f\xf7\xf4\xf9\xfe\xd3\xd3\xe2\xe2\xcb\x3f\x2a\x21\x84\x32\xcc\x53\x88\x50\x48\x21\x0b\x2d\xe8\xa4\xbc\xf2\x77\xf8\xb7\x96\xda\x07\x67\x7b\xa9\x8e\xe8\xaf\x16\xba\xbf\xca\x27\x3f\x2e\x2e\x2e\x5b\x2b\xfc\xa2\x65\x12\x79\x83\x08\x2d\x19\x70\x5a\x52\xc6\xd6\xf8\x02\xdb\x55\x54\xd5\x87\x69\x55\x45\x2b\x59\x96\xe0\x18\x22\x94\x5b\xd3\x0c\xc0\x88\x60\x81\xf5\xa6\x2e\xbb\x48\x39\x5f\x48\xce\xc8\x26\x42\x37\xcb\x3b\x69\x17\x1a\x0c\x88\xdd\x56\x7a\x29\xdb\x88\xe3\xaa\xe3\x6f\xfd\xad\xd2\x64\xd1\x20\x00\xfe\x4a\x3b\x36\x51\x7c\x0c\x90\x54\x33\xbb\xb9\x94\xc2\xc5\xbd\xef\xb6\x06\x4c\xef\x05\xdf\x3c\x48\x99\x47\x9b\xd9\x18\x0b\x49\x84\xac\x4e\x77\x1e\x7b\xfb\x78\x0b\xc6\x38\x97\x72\x90\x3a\x37\xb9\xb8\x28\xba\x75\x9c\xb8\xe6\x41\xe2\x46\xca\xf9\x79\xdc\xb6\xcc\x2f\xd3\x30\xcf\x80\xc9\x0b\x1e\xe8\x54\x5c\x98\x3b\x29\x9c\xfd\x0d\xab\xf3\x47\xbf\x1a\xd0\x05\x0f\xad\x56\xf2\xd9\xfa\xc1\xd6\xd1\x28\x05\x56\x72\xd0\xf5\xf6\x10\x20\x58\x2e\x81\x58\x47\x60\x1e\xcb\x34\xab\x74\xaf\x61\x13\xe5\x77\x70\xf9\x11\xa5\x51\x6d\x13\xec\x0e\x1e\x13\x1f\x26\x6f\xd5\x83\x18\xe4\x28\x68\xa8\x6d\x5f\x31\xd2\xa3\x01\x06\x85\xe4\x0b\x1c\xeb\x0a\x43\x22\xc5\x68\x86\xe5\x1c\x3e\x19\xb7\xea\x58\xfc\xc4\xac\xaa\xd4\xdc\xc5\xa7\xde\x88\x41\x55\x98\x8c\xe0\xde\xef\x88\x0e\x79\x6e\x9c\x8e\x08\xbd\xa0\xe4\x47\xa4\x40\x14\x5c\xa9\xbb\xc5\x02\xc7\xa0\xe7\x3c\x4b\x5e\xc9\x8f\x6a\x10\x06\x28\xd8\xbe\x33\x30\x4f\x05\xfb\x16\x85\x61\x48\x0c\x73\xff\xa6\x46\x92\x75\x93\x16\x31\x3a\xff\xe9\xe7\x7f\xfe\xfa\xf1\xfa\xe9\xee\xfe\xea\xba\x4d\x9d\x8c\x3b\xdf\xb9\xbe\x30\x0f\x33\xac\x43\xce\x9e\xc3\x1e\x84\x6b\x18\x3d\x73\x58\x2d\x40\x13\x10\x16\xc7\x30\x9f\xbd\x96\x0b\xd5\xdd\x19\x4f\x3c\x1a\x97\x0f\xf5\x87\x1e\x2b\x71\xd1\x32\x75\x60\xba\x8a\x7f\x3a\x3a\xd5\x93\xe2\xd9\x44\x1d\xec\xfe\x3f\x14\x49\x52\x9a\x65\x8c\x43\x0c\xb4\x41\x31\x6a\x24\xe3\xfc\x65\xbe\xd4\x85\x58\x1f\xbe\xb4\x4d\x0c\x57\xba\x38\xd8\x50\xf1\x34\x66\xc2\x84\x8d\xca\xea\x15\x2f\x54\xce\xd7\x52\xe1\x38\x57\x1a\xa1\x8f\x8c\x32\x0d\x79\xb9\xc2\xbc\x8d\x8b\x93\xcf\xd3\x3a\xa0\x4c\xf7\x35\xa4\x78\xe1\x62\xac\x4e\x37\xbd\x43\x1b\x85\xac\x35\x85\x42\xd6\x31\xc3\x6c\xda\xb4\xd2\x6c\xba\x74\xd4\x41\x2b\x67\x48\xb2\x86\x3e\x30\x74\xe4\xd4\x08\x30\xdc\x62\x79\x35\xf4\x34\x07\x08\xeb\xb8\x16\x17\xcd\x02\x1e\x94\x7b\x11\x68\x88\x99\xb1\x05\x5b\x0d\x54\xad\xa0\x36\xe3\xe6\x40\xc9\xdc\x5b\xc3\xf3\x04\xc6\x94\x6a\x30\x66\xbe\xbf\xd2\xef\xad\x91\xaf\xae\x4a\x65\x29\x59\x7f\x30\xd3\x98\x68\x17\xe0\x86\xc5\x41\x79\xb4\x0e\xb7\xad\xbd\x2c\x0a\x95\xf7\x3a\xca\x66\xd3\xbf\x4c\xcf\x47\x55\x99\xe6\x5a\x47\x3b\x49\xf5\x3a\x14\x8d\x89\x44\x7f\xd3\x5b\xb3\x6b\x11\x31\x28\xa6\x5e\xde\x73\x27\xb5\x04\x6c\x53\x0d\x41\x8c\x2d\x98\xf9\x17\xa9\x24\x97\xf1\x66\x5e\xf3\xdc\xc9\x15\xfb\x54\x71\xa7\xb6\x80\x71\x25\xd9\x06\xf6\xe0\x0a\x2c\x49\x80\x32\x97\x1a\x95\xcc\x12\x73\x53\x17\xb2\x2c\x01\x99\xda\xf9\xdf\x6b\x7c\x27\xa0\xb0\xc4\x29\xb7\xc1\xd2\xd8\x8d\x82\x39\x7c\xb3\x7f\xab\x3d\x2f\x1b\x42\xe5\x6f\x4e\x00\x66\xdd\xc1\xfd\xe6\xbd\xbf\x4c\x8e\x32\x31\xd6\x1f\x8c\xab\x1c\x39\x91\x6b\x26\x89\xf7\x5b\xe2\x6b\x1a\xf0\xbe\xdf\x26\x07\x75\xe2\xd9\xc9\xaf\x2b\x8e\x96\x64\xfe\x6d\xde\x6e\xa1\x12\xd4\xa2\x66\x0d\xbb\x55\x68\x5d\xc6\xb6\x4e\xff\x2b\x69\x0a\x13\x7d\xfa\xd1\xdd\xfc\x3b\x9b\x4f\xf1\x13\xcb\x55\xde\x6e\xa4\xde\xdc\xeb\xcb\xed\x2b\x6a\x9d\x9e\x8f\x31\xe4\xa9\x2c\x33\x9b\x70\xb0\xfe\x83\x05\x6a\x04\x1c\x5d\x5c\xa8\x9f\x35\x7b\x19\xd0\x30\x5b\xea\x74\xa8\x9f\xda\x06\x09\xea\xa5\xf0\x08\x7b\x7f\x80\x6e\x74\x28\xaf\xd3\xb1\x7e\x5a\xea\x74\xad\x73\xf9\x3a\x77\xeb\x19\xc0\x25\xb7\x6b\x5e\x6f\xd5\x5f\xf3\x0a\xb3\xd9\x33\x58\x5c\xbd\xba\xf0\x78\x73\x55\x9c\x24\x4e\xf3\xfb\xae\xb7\x29\xdb\x63\x3e\xb6\x16\x93\xd5\x03\xfc\x3b\x65\xda\x9d\x26\xb6\xcd\x4b\x49\x7a\x23\x96\xf2\x5e\xe4\x55\xab\xaa\x67\x45\x79\xf8\xcc\x96\x40\x36\x84\xc3\xad\xa4\xdb\xf7\xd3\x16\xd5\x0b\x79\xf9\x9f\xd7\x6a\x05\x09\x68\xcc\x27\xff\x0b\x00\x00\xff\xff\x2a\x5b\x31\x74\x3b\x32\x00\x00") func deployKubernetes118LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -160,12 +161,12 @@ func deployKubernetes118LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 10848, mode: os.FileMode(436), modTime: time.Unix(1610053002, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 12859, mode: os.FileMode(420), modTime: time.Unix(1611067498, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119AlphaDirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\x5b\x6f\xe3\xba\x11\x7e\xf7\xaf\x60\xd3\xf3\x70\x0e\x50\x59\xf1\xb6\x07\x68\x05\xf8\x21\x9b\xf8\xa4\x46\x37\x8e\x91\x64\xcf\x6b\xc0\x50\x63\x99\x35\x45\xaa\xe4\xc8\x1b\x6d\xd1\xff\x5e\x90\xba\x58\x37\x2b\xb6\x37\x17\x14\xf5\x3e\xac\x23\x91\x9c\x8f\x33\xc3\x6f\xe6\x93\xf5\x47\x72\x0d\x12\x34\x45\x08\xc9\x37\x8e\x6b\x72\x16\xd3\x0d\x90\x4d\x6a\x50\xc5\xfc\x3b\x9c\xfd\x89\x84\x8a\x48\x85\x04\x42\x8e\x7f\x18\x8d\x68\xc2\x7f\x07\x6d\xb8\x92\x01\xd9\x4e\x46\x1b\x2e\xc3\x80\xdc\x83\xde\x72\x06\x17\x8c\xa9\x54\xe2\x28\x06\xa4\x21\x45\x1a\x8c\x08\x11\xf4\x09\x84\xb1\xdf\x08\x49\x62\x88\x3d\x66\xf8\x98\x4b\x04\x31\x66\x2a\xf6\x43\x48\x84\xca\x62\x90\x18\x90\x90\x6b\x60\xe8\x25\x5a\x85\x29\x43\xae\xe4\x88\x10\x49\x63\x08\xaa\x89\x1e\x53\x12\xb5\x12\x02\x74\x71\xcf\x24\x94\xd5\x06\x8c\x3c\xcf\x6b\x40\xd4\x4f\x94\x8d\x69\x8a\x6b\xa5\xf9\x77\x6a\x17\x1d\x6f\xfe\x6a\xc6\x5c\xf9\x15\xf8\x3b\x25\xe0\x0d\x21\xc3\x33\x82\x96\x54\xd8\x41\x5b\x6e\x51\x81\xf6\xd8\x2a\xda\xb3\x01\x9d\x0a\x30\xc1\xc8\x23\x34\xe1\xd7\x5a\xa5\x89\x03\xe2\x91\xb3\xb3\x11\x21\x1a\x8c\x4a\x35\x83\xe2\x1a\xc8\x30\x51\x5c\xa2\x19\x11\xb2\x05\xfd\x54\x5c\x8e\x00\xdd\xff\xdf\x28\xb2\xb5\xfb\x26\xb8\xc9\x2f\x85\x20\x00\xc1\x7d\x4d\x93\x90\x16\x5f\x99\x06\xfb\xb5\x63\x93\x29\xa5\x43\x2e\xeb\x6e\xeb\x82\x10\x40\x0d\xbc\x15\x02\x83\x4a\xd3\x08\xf6\x1a\x67\x86\x17\x43\x18\x4d\x28\xe3\xc8\xf7\x41\xa9\x00\xec\x30\x15\x46\x5b\x50\x92\xea\x7e\x01\xf5\xa0\x50\x24\x2a\xec\x31\xdc\x99\x4a\x93\xc4\x74\x27\x6b\x48\x04\x67\xd4\x40\x5f\x24\x4f\xc8\xe8\x4b\x91\x1a\x04\xfd\x11\x89\xad\x53\x29\x41\x1f\x95\xc5\x89\xdd\x99\x41\x90\xb8\x55\x22\x8d\x4f\x0b\xe0\x51\xb1\x6a\x19\x64\x82\xf2\xf8\x70\xab\x45\xae\x1c\x9d\xab\x65\xa2\x0a\x6a\xf6\x1e\x98\x96\xb9\xc3\x58\x60\x0b\x1d\x0a\x38\x32\xdb\xbb\x7b\x91\x34\x31\x6b\x85\xe3\x97\x36\x55\x84\xac\x18\x3e\xb4\xab\xd7\xb2\x61\x4b\x40\x77\xc3\x2f\x98\x7a\x99\x46\xa4\x0a\x5f\x33\x28\x47\x2c\x77\x5a\xc5\xfa\xcc\x65\xc8\x65\xf4\x96\xb5\xd6\xf0\xe6\xd1\x56\x02\x86\x0a\x97\x12\x70\x07\x2b\x6b\xbd\xf4\xcf\xc0\x66\x46\x84\xd4\xaa\xef\xe1\x45\xd3\xa4\x4f\xff\x04\x86\x8e\x5b\x7a\x7b\x8f\x77\x69\x19\x6a\x04\xfb\x31\x71\x38\xd5\xdb\xf5\xca\x70\x14\xa1\xbf\x83\xdf\xdb\xdd\xe4\x47\xb6\x91\x26\x01\x66\x8d\x25\x4a\x63\x55\xe1\x35\x06\x64\x72\x7e\x7e\x7e\xee\x50\x20\xd5\x11\xe0\xb2\x71\xd5\x80\x00\x86\x4a\xe7\x38\x69\x92\xec\xb3\x79\xc2\x26\x3e\xd0\x5d\x31\xa0\xe6\xcc\x9c\xe0\xab\x49\xaf\xaf\x26\x6f\xed\x2b\x42\x30\x4b\x20\x20\x0b\x15\x82\xb5\xda\x71\x9e\xed\xc3\x76\xc7\xf9\x1e\x29\xc2\x2a\x15\xf7\xf0\xa1\xda\xa5\x74\x64\xd9\x0e\x06\x64\xd2\xf1\x53\x6c\x6b\xc6\x97\x1a\xae\x17\x5c\x77\x12\x70\x93\x67\xd4\x62\x00\x3f\x42\x9c\x08\x8a\x50\xa0\xaa\x39\xcd\x41\x92\x52\xa1\x23\xa0\x0a\x65\x2f\x10\xc3\x34\xb5\x71\xb2\x4b\x53\x2e\x41\x9b\x62\xb4\x68\xec\xf0\xc5\x3d\x9e\xb4\xcb\xbd\x13\xbf\xc1\xd3\x5a\xa9\x4d\x40\x78\x24\x95\x06\x37\xb4\x8c\x8d\x03\xb3\x5a\x71\xc9\x31\xdb\xc1\xb3\xd5\xfe\xa2\x73\xd5\x46\xf2\x5f\x29\xd7\x10\x5e\xa5\x9a\xcb\xe8\x9e\xad\x21\x4c\x05\x97\xd1\xdc\x2d\x5c\x5c\x9e\x3d\x03\x4b\x2d\xa6\xfa\xcc\x7c\xcd\xfb\x22\xf6\x0f\xa0\x63\xd3\xbc\xed\xe5\xa9\x30\x7b\x4e\x34\x18\xd3\xf4\x74\x39\x62\x03\x59\xd0\xb7\xc1\x1e\xff\x95\x1f\x95\x80\xa6\x36\xdb\xc8\x42\xe1\x5c\x76\xee\x6f\xa9\x48\xa1\x63\xca\x75\x41\x52\x9d\xf5\x5d\x5e\x51\x61\xa0\xbc\xb3\x0b\x74\xb9\x84\x55\x99\x71\x4c\x65\xb8\x5b\xd3\x23\x7e\x6a\xb4\x2f\x14\xa3\xc2\x7f\xe2\xd2\xaf\x02\x1f\x6a\xbe\xad\x81\xf6\x88\xb7\x9d\xfe\xb9\xfe\xa7\x50\x51\xc4\x65\xe4\xad\x94\x8e\x29\x4e\x11\x9e\xb1\x7e\x3b\x56\x21\x4c\x7b\x76\xef\x11\xaf\xd4\xd2\xd3\x54\xf2\xe7\xc0\xf7\x7d\x66\xb8\xdf\x4c\xb6\xb1\x51\x6c\x53\x9f\xa3\x21\xe2\x06\x75\x36\x2b\xe7\x22\x4b\x02\xdf\x3f\x1f\xbb\x7f\xc1\xae\x56\x14\xe3\x6d\x48\x79\x38\xfd\xe9\xe7\x7f\x7c\xfd\x3c\x7b\x5c\xdc\x5e\xcd\x1e\x17\x17\x37\xb3\x5f\xea\x63\x18\xfd\x8d\x0b\x98\xfa\x0c\x34\x1a\x9f\xd1\x31\xd3\x8d\x2d\xd8\xeb\xf5\x11\x28\x4c\x7b\xc8\x06\xb2\xf6\x88\x0d\x64\xf5\x11\xb9\x1f\x2d\x07\x4d\x7f\xfa\x79\x79\x33\xbb\x79\xbc\xbc\x9f\x3f\x5e\xdd\xcd\x7f\x9f\xdd\x75\x21\x15\xf4\xff\xc5\x69\xa6\x69\xb0\xa3\x75\xfb\x01\xb9\xad\x47\x2e\xe7\xbc\xe6\xfe\x46\xad\xf4\xf9\x4d\xab\xb8\x99\x41\x2b\x0e\x22\x2c\xda\x99\xfa\xa7\x55\xef\x9a\x37\xdd\xa4\x25\xc5\x75\xe0\x4e\xe7\xd8\x7a\xd7\x52\x56\x07\xcd\xc3\xec\xee\x66\xbe\xb8\x78\x98\xdf\x2e\x1e\xbf\xdc\x5e\x3f\x2e\x2f\x1e\xfe\xde\xc6\x14\x10\x1f\xe3\xc4\x47\xd0\x71\xf1\xc0\xc3\xe6\x52\x67\xa9\x3e\x5f\x75\x97\xea\x1e\xb9\xce\x42\xd7\xb7\x57\xb3\xcf\x5f\xaf\xbb\x73\x9f\x7f\x3d\xff\x5b\xce\x3b\x97\x8b\xe9\xce\xcd\x3c\xa6\x11\x04\xc4\x2d\xd8\x3e\x0f\x01\xa3\x92\xea\xac\x39\x76\x99\x0a\xb1\x54\x82\xb3\x2c\x20\xf3\xd5\x42\xe1\x52\x83\x01\xb9\x4b\x94\x5a\x75\x6a\x1d\xab\xaa\x90\x97\x90\xab\x53\xbb\x6c\xd5\xf5\xfa\x4a\xbb\x26\x21\xff\x18\x60\xa9\xe6\x98\x5d\x5a\xbd\xf6\x8c\x4d\x5a\xa4\xe1\xad\x14\xd9\x9d\x52\x2e\x97\x4d\x66\x10\xe2\x80\xa0\x4e\x77\xd1\xab\x85\xe2\x06\x8c\xb1\x5b\x72\xb1\x1e\x8c\x53\x2e\x15\x6f\x6c\x47\xda\xd8\x41\x6c\xaf\x14\xf3\xdd\xa9\xe8\xc0\x2f\xcf\xb2\x3b\x60\xfb\x66\x1a\xde\x99\x97\x88\x34\xe2\xd2\xb3\xd4\x00\xe8\x85\x5c\xef\x99\x8b\x71\xd2\x99\x8b\x71\x52\x9b\xe1\x11\xaa\xa3\x06\x6a\xcf\x2b\x9e\x6a\x65\x35\x16\xf2\x76\x75\xad\xe0\x33\x90\xa8\xa9\x18\x60\x45\x97\x2b\x34\x0c\x6d\xa5\x98\x1e\x42\x6c\xde\x0a\x28\xa6\x1a\xbc\x88\x22\x98\xe9\x83\x4a\x94\x50\x51\x36\x6d\x04\xc8\x8e\x33\x36\xe6\xe8\xe1\xde\x01\xc8\x63\x50\x29\x4e\x7f\x8d\x1b\x97\x43\x58\xd1\x54\xa0\xb7\x32\xb6\x51\x9b\xc2\x33\xfe\xa5\x71\xbf\x48\xa6\x0a\xb4\x23\x9d\xc9\x30\xe9\x2c\x6f\xaf\xdc\x79\xbc\x5f\x5e\x5c\xfe\x00\xe7\xd4\x68\xa5\x6c\x6b\xc6\x55\xb7\xb6\xd7\xe8\xeb\xdb\x6b\x1f\x7d\xab\xe4\x22\xa6\xad\x0e\x35\x3c\xf2\x8a\x47\x1a\x7e\x4b\x1d\x06\xdb\x4f\xe3\xf3\xf1\xa7\x93\xb8\xa0\x4f\xff\x1d\x45\x0a\x93\x37\x27\x85\x83\xce\xf7\x11\xa7\x74\x2f\x1e\x9d\xca\x0b\xb3\x50\xd2\xa2\x69\x61\x70\xb7\xbe\x1a\xd0\xb9\xee\xab\x56\xaa\xcb\xe1\xa1\xd6\xd9\x7e\x50\x09\xdb\x67\xd5\xdb\x36\x8f\xc0\x6a\x05\x0c\x6d\xe7\x55\x74\x8a\x3b\x9b\xae\x93\xb3\x05\xce\xa9\xff\xf1\x26\x7d\x02\x2d\x01\xc1\x3d\x96\x88\xa9\xd5\xf4\xa3\xba\x87\xea\xab\xc6\x09\x66\x57\x5c\x07\x44\xa6\x42\xb4\xe9\x7f\x8f\x5f\xbc\x41\x56\x34\xc0\x34\x34\x82\x97\x5f\x69\xed\xba\x9a\x9c\xdf\x35\x3d\x90\xfe\xfd\x9f\x16\xa0\x92\x12\x87\x05\xdb\x15\x85\x58\xc9\xb7\x95\x6b\xd6\xdb\x2f\x08\xb5\x23\xa5\x59\xb1\xe2\x89\xf0\x3e\x54\x71\xd5\xa0\xbf\xaf\xd6\x7a\x75\xc1\x10\x82\x3d\xa6\x37\x54\xd2\x08\xf4\x34\xc7\xfa\x83\x82\xa2\xe1\x9c\x01\x29\xd1\x29\xb3\x07\xe9\x81\x8a\x3a\x5a\x2a\xa3\x98\x65\x0b\xd0\x7c\xf9\x8b\x13\x1b\x93\x03\xc4\x49\x0f\x27\x75\x85\xca\xbb\x88\x10\x83\x14\x5d\x4f\x37\xf5\xb7\x54\xfb\x82\x3f\xf9\x07\x88\x91\xe3\xa4\x8b\xdd\xed\x12\xb4\x6d\x91\x68\x04\xd3\x49\x73\x9b\xff\xeb\xc2\xa6\x96\x00\xef\x00\x05\x29\xa6\x66\x9c\xa8\xb0\x66\xed\xd5\x75\xd1\x2b\x6a\xb5\xff\x6b\x89\x95\x68\xbe\xe5\x02\x22\x08\x5b\xdd\x4b\xa3\x7f\x39\x7f\x7b\xb5\x55\x1e\x6d\xdb\xb2\x08\x40\x3f\xef\x38\x8c\xdf\x6c\x61\x9a\x3d\x5b\x3e\x5f\xab\x84\x46\xce\x68\x40\x3e\xf3\x9c\xa9\xb9\x92\x35\xa5\x53\xf9\xc5\x8e\x77\x24\x37\x20\xc0\x3a\x40\xf2\x5f\xec\x4f\xb5\x69\xa7\x0f\x58\xeb\x17\x99\xae\x83\x1b\x10\x98\x21\x6c\x3b\x73\x42\xd8\x0e\xd8\x31\x59\xd7\x8a\xc9\x06\x91\xf5\xb4\xc7\x2f\xaa\xd7\xd2\x79\x03\x27\xf8\x04\x17\xda\xc5\x5c\x15\x18\x56\xc1\x6d\x35\x5b\x44\xb0\xac\x70\x79\x5e\x26\x8d\x42\xd2\xce\xb6\x3d\xa5\xa2\xb7\x2a\xf7\xaa\xe5\xe6\xa8\x7e\xfd\xf9\xa3\x1c\xf8\xb2\xd0\x73\xf9\x93\x53\x49\xb5\x7b\x1d\x6c\x27\xe3\x4f\xe3\xf3\x93\xb8\xa9\xbd\xd6\x87\x0b\xb5\x17\x33\xb1\x1e\xf4\xce\xec\x46\x46\xec\xd6\xa8\x3f\x4a\xdf\x81\x29\x1c\x9b\x87\x64\x9f\x7a\x5a\x2b\x93\x5b\xae\x73\xeb\x30\xb3\x0d\x9e\x91\xfc\x77\xa9\x2b\x77\x2a\x94\xce\x6e\xf5\x65\xf9\x5a\xc6\xa0\x17\x4e\x01\xf2\x58\x76\x80\xfe\xd1\xf6\xf7\xf8\xf1\x24\x77\x0c\x11\xfd\x61\x68\x7a\xe9\xfd\x38\x2c\x4d\xae\x3f\xcc\x6c\x8b\xe1\xbd\xbd\x1c\x7e\xa0\x1c\x76\x13\xdb\x52\xf8\x80\x4d\xbc\x42\x3e\xed\x61\xda\x01\xe3\xcd\x4a\x74\x98\x95\x66\xa5\x1a\x5c\xbe\x59\xb6\x0e\x3c\x14\x45\x59\x6b\x3f\x1a\x68\xbe\xdc\xe3\x6f\x27\x4f\x80\xb4\x7a\x53\xe3\x7e\x7e\x95\xb7\x5e\x6f\xf6\xa4\xa0\x16\x97\x52\xb9\x52\x44\xca\xd6\x77\xc5\xaf\x7f\x01\x71\xbf\x81\xb9\x9f\xc8\xc3\xb9\x5c\xa9\x5b\xe9\x88\xb1\xa2\xcc\x62\x07\x97\xc5\xc3\xde\xea\x7a\xce\x46\x5f\xf8\x0a\x58\xc6\x04\xdc\xa8\xb0\x7c\xbd\x68\x59\xbd\xc2\xe6\xfe\x9c\x25\x6b\x88\x41\x53\x31\xfa\x6f\x00\x00\x00\xff\xff\xdb\x89\xc9\x86\xd0\x2b\x00\x00") +var _deployKubernetes119AlphaDirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x02\x95\x15\xf7\x02\x0c\x04\xe8\x21\x9b\x64\xa7\x41\x27\x89\x91\xcc\xee\x6b\x40\x53\xc7\x32\x6b\x8a\x64\x49\x4a\x33\xda\xa2\xff\xbd\xa0\x44\xc9\xba\xd8\x8a\xac\xdc\xdc\x35\x30\x18\x87\x3a\xe4\x39\xe7\xe3\xb9\x7c\xa4\xf5\x47\xf4\x19\x38\x28\x6c\x20\x42\xdf\xa8\xd9\xa0\xb3\x04\x6f\x01\x6d\x53\x6d\x44\x42\x7f\x83\xb3\x3f\xa3\x48\x20\x2e\x0c\x82\x88\x9a\x3f\xcc\x66\x58\xd2\x5f\x41\x69\x2a\x78\x80\xb2\xc5\x6c\x4b\x79\x14\xa0\x47\x50\x19\x25\x70\x41\x88\x48\xb9\x99\x25\x60\x70\x84\x0d\x0e\x66\x08\x31\xbc\x02\xa6\xed\x37\x84\x64\x02\x89\x47\x34\x9d\x53\x6e\x80\xcd\x89\x48\xfc\x08\x24\x13\x79\x02\xdc\x04\x28\xa2\x0a\x88\xf1\xa4\x12\x51\x4a\x0c\x15\x7c\x86\x10\xc7\x09\x04\xf5\x44\xaf\x98\xe8\x11\x91\x78\x44\x70\xa3\x04\x63\xa0\x9c\x94\x96\x98\x34\x44\x67\x9e\xe7\x9d\x8c\xb1\xdf\x60\xb5\x11\x62\xab\x47\x9a\xaa\x56\x98\xcc\x71\x6a\x36\x42\xd1\xdf\xb0\x5d\x7c\xbe\xfd\xa4\xe7\x54\xf8\xb5\x13\x0f\x82\xc1\xbb\x98\x0e\xdf\x0d\x28\x8e\x99\x15\xcf\xa8\xb5\x0f\x94\x47\xd6\xf1\x01\x57\x54\xca\x40\x07\x33\x0f\x61\x49\x3f\x2b\x91\xca\xc2\x24\x0f\x9d\x9d\xcd\x10\x52\xa0\x45\xaa\x08\xb8\x31\xe0\x91\x14\x94\x1b\x0b\x4b\x06\x6a\xe5\x86\x63\x30\xc5\xff\xdf\xb0\x21\x9b\xe2\x1b\xa3\xba\x1c\x8a\x80\x81\x81\xe2\x6b\x2a\x23\xec\xbe\x12\x05\xf6\x6b\x4f\x27\x11\x42\x45\x94\x37\x01\xec\x1b\xc1\x00\x6b\x78\x2b\x0b\xb4\x11\x0a\xc7\x70\x50\x39\xd1\xd4\x89\x10\x2c\x31\xa1\x86\x1e\x32\xa5\x36\x60\x67\x93\x53\xda\x31\x45\xd6\xcf\x9d\xa9\xa3\xb6\x42\x8a\x68\x8f\xe2\xde\x54\x2c\xa5\xee\x4f\x56\x20\x19\x25\x58\xc3\xbe\x9d\x3c\xe5\xd8\xae\xd2\xf2\x95\xe2\x79\x3f\x88\x7b\x02\x69\x02\x26\x97\x2c\xd5\x06\xd4\xc7\xa6\xbd\x4a\x39\x07\x75\x1c\x26\xd6\x47\x6d\x80\x9b\x4c\xb0\x34\x99\x16\xde\x47\x45\x72\x47\x21\x61\x98\x26\xe3\xb5\xba\x4c\x3a\x3a\x93\xab\x34\x66\x58\x1f\x2c\x27\x1d\x75\xe3\x6a\x64\x06\xbd\x02\x79\x64\x2d\xe8\xfb\xc2\xb1\xd4\x1b\x61\xe6\xcf\x39\xe5\xb6\xcc\x89\x0f\x79\xf5\x5a\x3a\x6c\x57\xef\x3b\xfc\x8c\xaa\xe7\x8b\x2c\x17\xd1\x6b\x6e\xca\x11\xcb\x9d\x7e\xa6\xd7\x45\xf0\x95\xb2\xfb\x28\x78\xde\x21\xa3\xe5\xb3\xb9\x3d\x36\xef\x8e\x51\xf2\x12\x56\xd0\xab\x25\xc7\xc6\xf0\xb4\x96\xfb\x13\xe5\x11\xe5\xf1\xfb\xb0\x77\x4d\xdb\x9d\x45\x30\x18\xea\xc2\x82\xc1\x03\xac\xad\x1d\x15\x96\x03\x6e\xcd\x10\x6a\x10\x89\x29\xdc\x56\xa7\xab\x7f\x01\x31\x45\x1a\xec\x3d\x34\xbc\xe6\xc1\xe4\x04\xb7\x67\x57\x13\x3e\x64\x5f\x5a\xbc\xec\x25\x7b\xf1\xfa\xe7\xae\x46\x75\xfe\xe8\x7c\x99\x8a\x7e\xb3\xc1\x4c\x64\x80\xa7\x96\x1f\x1f\xb4\x2b\xad\x34\x79\xe3\xed\xe8\xb6\xe9\x77\x4c\x8b\xee\xcd\xc9\x69\xdc\xef\x68\x09\xc4\xaa\x95\x42\x99\xfa\x00\xa6\x4c\x80\x16\xe7\xe7\xe7\xe7\x85\x3d\x06\xab\x18\xcc\xb2\x35\xaa\x81\x01\x31\x42\x95\x16\x63\x29\xe7\xdb\x74\x05\x8a\x83\x81\x22\xa6\x28\xd7\x06\xf3\xa6\xa6\x9d\x37\x07\xa6\x74\x7c\x68\x59\x3e\x01\x94\x93\x80\x3f\x01\xa3\x28\x39\x14\x21\x43\xd8\x2f\xf6\x62\xbf\xf8\xff\xc0\x1e\x21\x93\x4b\x08\xd0\x9d\x88\xc0\xda\xde\xdb\x0c\x2c\xa5\xde\xd5\x9d\x47\x83\x0d\xac\x53\xf6\x08\x87\xef\x11\xfb\x36\x13\x91\x48\xc1\x0b\x0b\x3a\xf6\xbe\x03\x24\x7d\x79\x89\x95\xf1\xc4\xba\xb1\xbd\x1f\x96\xcb\xd5\x4d\x52\x80\x16\xbd\x70\x49\x2c\xb7\xfd\xd2\x00\x76\x12\x5c\xc7\x03\x36\x09\x0a\x5d\xa6\xea\xdd\x28\x44\x0c\x24\x92\x61\x03\xce\xcf\x46\x1c\x15\xf6\x72\x2e\x4c\xd1\x39\x6a\xbf\xf7\x9a\xa4\x89\xc2\x36\x74\xed\xd2\x98\x72\x50\xda\x49\xb3\x16\x66\xc7\x05\xe4\x64\x94\xa7\xe0\x3c\x3e\x38\x27\xee\xca\xc1\x89\xae\x21\x06\x88\xc6\x5c\x28\x28\x44\xab\x98\xb4\x9f\x1d\xa6\xd5\x88\x87\x88\x48\x12\xcc\xa3\x1d\xae\x1e\xf2\x53\xad\x7c\x26\x08\x66\xfe\x8a\x72\xbf\xf6\x34\x52\x34\x6b\x78\xe9\x21\x2f\x0b\xff\xda\xfc\x93\x89\x38\xa6\x3c\xf6\xd6\x42\x25\xd8\x84\x06\xbe\x9b\xe6\xe3\x44\x44\x10\x36\x9a\x76\xfd\xa0\x5c\xd8\xe2\x1a\xfe\xf0\xe3\xf2\xf6\xfa\xf6\xe9\xf2\xf1\xe6\xe9\xea\xe1\xe6\xd7\xeb\x87\xa7\xbb\x8b\xdb\xeb\x3f\x35\xa5\xed\x09\xf2\xd1\xe5\x54\xf8\x9f\x33\x77\xcc\x3c\x0b\xce\xac\xa1\x67\xff\x6d\x8a\x12\xfc\x33\x65\x10\xfa\x04\x94\xd1\x3e\xc1\x73\xa2\x5a\x16\xd9\xf1\xa6\x84\x61\xba\x2b\xb2\x85\xbc\x2b\xb1\x85\xbc\x29\xa1\xc9\x06\xa2\x94\x81\xfa\x52\x1c\xef\xc3\xe0\x53\xd5\xb8\x2b\xbf\xcb\x26\x54\x3d\xde\x35\x17\xfb\x01\x9e\x35\xb1\x2f\x83\xeb\xeb\xf5\xc3\xed\xcd\xdd\xc5\xd7\x9b\xfb\xbb\xa7\x2f\xf7\x9f\x9f\x96\x17\x5f\xff\x51\x0b\x21\x94\x61\x96\x42\x80\xfc\x08\x32\xdf\x80\x4a\xdc\x8f\x03\x16\xff\xde\x52\xfb\xe0\xec\x2f\x35\x90\x00\xf5\x42\xf7\x57\xc5\xe4\xc7\xe5\xc5\x65\x6f\x85\x9f\x95\x48\x82\xc6\x20\x42\x6b\x0a\x2c\x72\x3c\xb2\x37\xbe\xc4\x66\x13\xd4\x25\x62\x5e\xd7\xd2\x5a\x96\x26\x38\x86\x00\x15\xd6\x74\x03\x30\x20\x98\x63\x95\xb7\x65\x97\x29\x63\x4b\xc1\x28\xc9\x03\x74\xb3\xbe\x13\x66\xa9\x40\x03\xdf\x6d\x65\x23\x6b\x3b\x71\x5c\x77\xff\xca\xdf\x3a\x4d\x96\x1d\x32\xd0\x5c\x69\xc7\x2c\xca\x8f\x06\x92\x2a\x6a\xf2\x4b\xc1\x6d\xdc\x37\xdd\x56\x80\xa3\x7b\xce\xf2\x07\x21\x8a\x68\xd3\xb9\x36\x90\x04\xc8\xa8\x74\xe7\x71\x63\x1f\x6f\x41\x6b\xeb\x52\x01\xd2\xe0\x26\x97\x37\x49\xb7\x96\x28\xb7\x3c\x48\xec\x88\x9b\x5f\xc4\x6d\xcf\x7c\x97\x86\x45\x06\xcc\x9e\xf1\x40\xa5\xfc\x42\xdf\x09\x6e\xed\xef\x58\x5d\x3c\xfa\x45\x83\x2a\x99\x69\xbd\x52\x93\xc2\x1f\xec\x1e\x9d\x52\x60\x04\x03\xd5\xee\x10\x1e\x82\xf5\x1a\x88\xb1\x4c\xe6\xd1\xa5\x59\xad\x7b\x0b\x79\x50\x5c\xd2\x15\xe7\x96\x4e\xc1\x4d\xb0\x3d\x8d\xcc\x9a\x30\x35\x56\x3d\x88\x41\x81\x82\x82\xd6\xf6\x95\x23\x23\x7a\xa0\x57\x4a\x3e\x43\xb6\xae\x30\x24\x82\x4f\xa6\x5a\xd6\xe1\x37\x23\x59\x03\x8b\xbf\x3d\xbd\x72\xca\x87\x88\xd5\x07\x51\xa9\x1a\x96\x69\x54\xfc\x84\x78\x51\xc3\x93\xb7\x63\x44\xcf\x28\xf9\x9d\x72\xa1\x08\x6c\xcd\xbb\xc5\x1c\xc7\xa0\xc2\xd2\xd6\x17\x72\xa5\x16\x90\x1e\xf2\xaa\x37\x0d\xc2\x94\xd3\xef\x81\xef\xfb\x44\x53\xfb\x6f\xae\x05\xd9\x76\x29\x12\x8d\xc2\x1f\x7e\xfc\xe7\x2f\x3f\x5d\x3f\xdd\xdd\x5f\x5d\xf7\x69\x94\xb6\x87\x3e\xdb\x23\x42\x3f\xc3\xca\x67\x74\xe5\x8f\x20\x5f\xc7\x51\x35\x0b\xd7\x12\x14\x01\x6e\x70\x0c\xe1\xe2\xa5\xbc\xa8\xed\xce\x74\x12\xd2\xb9\x97\x68\x3f\x6c\x30\x14\x1b\x30\x73\x0b\xa6\xad\xfe\x6f\x47\xad\x46\xd2\x3d\x93\xc8\x83\x4c\xe0\x77\x45\x98\xa4\xa2\x19\x65\x10\x43\xd4\xa1\x1b\x2d\xc2\x71\xfe\x3c\x77\x1a\x42\x6c\x0c\x77\xaa\x12\xc3\x16\x30\x06\xc6\x97\x2c\x8d\x29\xd7\x7e\xa7\xbe\x36\x4a\x18\x72\xf3\x95\x90\x38\x2e\x94\x06\xe8\x27\x5a\x56\x03\x2a\x38\x66\x7d\x5c\xac\x7c\x91\xd6\x5e\x44\xd5\x58\x43\xca\x77\x34\xa6\xea\xb4\xd3\x07\xb4\x45\x90\xf5\xa6\x44\x90\x0d\xcc\xd0\x79\x9f\x62\xea\x7c\x48\x47\x1b\x34\x37\x43\x90\x2d\x8c\x81\x61\x20\xa7\x26\x80\x61\x17\x2b\xaa\x61\x43\xb3\x87\xb0\x8a\x5b\x71\xd1\x2d\xe0\x9e\xdb\x0b\x4f\x41\x4c\xb5\x29\x99\xab\x27\x5b\x05\xb5\x1b\x37\x07\x4a\xe6\xde\x1a\x5e\x24\x30\x8e\x22\x05\x5a\x87\xfb\x2b\xfd\xde\x1a\xf9\xe2\xaa\xe4\x4a\xc9\xf6\x93\x9e\xc7\x44\xd9\x00\xd7\x34\xf6\xdc\x31\xdb\xaf\x1a\xbc\x2b\x0a\xb5\xf7\x2a\xc8\x16\xf3\xbf\xcc\xcf\x27\x55\x99\xee\x5a\xaf\x76\xaa\x1a\x75\x40\x9a\x12\x89\xcd\x4d\xef\xcd\x6e\x45\xc4\x60\x4c\x79\xc0\xf1\x8a\x81\xe7\xde\x9b\xcb\xdb\x01\xe0\x06\x3d\xf1\x8d\x83\x52\xb0\xf6\x18\x64\xc0\xc2\xf3\xa1\xa8\x7c\x3e\x6a\xac\xd4\x1a\xb0\x49\x15\x78\x31\x36\xa0\xc3\xaf\x42\x0a\x26\xe2\x3c\x6c\x61\x67\xe5\xca\x9d\xae\x39\x58\x5f\x40\xdb\xa2\x6e\x3c\x73\x70\x05\x9a\x24\x10\x51\x9b\x5c\xb5\xcc\x1a\x33\xdd\x16\x32\x34\x01\x91\x9a\xf0\xef\x49\x6b\x38\x82\x35\x4e\x99\xf1\xd6\xda\xe4\x12\x42\xf8\x6e\xfe\xd6\x7a\xee\x5a\x4a\xed\x6f\x41\x21\x16\xc3\xe9\x71\x3a\xec\xe1\x3d\xef\x53\xba\x4a\x5f\x5f\x5f\xb7\x7c\xb8\xd2\xb1\xfd\xa4\x6d\x6d\x2d\xa8\x6e\xb7\x8c\x34\x7e\x91\x7d\x09\x45\xd9\xf7\x0b\xef\x51\x5c\x65\xf1\x12\xae\xf2\xbe\x65\xa8\x79\xf7\xb9\x5b\xc8\x81\x5a\x56\xf5\x59\x65\xf2\xa8\x3b\x98\xde\xed\x75\xef\xae\x64\x23\x74\x69\x62\x93\xa0\x0d\xd3\xa3\xc1\xf6\x5c\xfe\x32\x75\x55\x34\x64\xa1\xf2\x7b\x75\x59\xbd\xf4\x37\xe8\xf9\x14\x43\x9e\x5c\x21\xce\xfd\xa3\xf5\x1f\x2c\xe1\x13\xe0\x18\x62\x8b\xe3\xac\xd9\xcb\x11\x8f\xb3\xa5\x4d\x18\xc7\xa9\xed\xd0\xc4\x51\x0a\x5f\x61\xef\x0f\x10\xb2\x01\xe5\x6d\xc2\x3a\x4e\x4b\x9b\xd0\x0e\x2e\xdf\x66\xb7\x23\x03\xd8\xb1\xdf\xee\x65\x60\xfb\xad\x39\x3f\x5b\xac\xc0\xe0\xfa\x05\x90\xc7\x9b\xab\xf2\xac\xf5\x66\x3f\x8e\x37\xf6\xa5\xba\x0e\xc1\xc6\x60\xb2\x79\x80\x7f\xa7\x54\xd9\x23\x57\xd5\x9f\xa5\x88\x6e\xf8\x5a\xdc\xf3\xa2\x70\xd5\x25\xcd\x79\x70\xe9\xd8\x49\x3d\x5e\x56\x8e\x2f\x74\x0d\x24\x27\x0c\x6e\x45\x54\xbd\x09\xb8\xac\x5f\x7d\x2c\xfe\xbc\x96\x1b\x48\x40\x61\x36\xfb\x5f\x00\x00\x00\xff\xff\x67\xa8\xd0\x39\xb7\x33\x00\x00") func deployKubernetes119AlphaDirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -180,12 +181,12 @@ func deployKubernetes119AlphaDirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 11216, mode: os.FileMode(436), modTime: time.Unix(1610053023, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 13239, mode: os.FileMode(420), modTime: time.Unix(1611067507, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119AlphaLvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\xf8\xda\x03\x5a\x01\x7e\xc8\x26\xbe\xd4\xe8\x26\x31\x92\xec\xbd\x06\x0c\x35\x96\x59\x53\xa4\x4a\x8e\xb4\xd1\x15\xfd\xdf\x0b\x52\x1f\xd6\x97\x15\xdb\x9b\x0f\x14\xf5\x3e\xac\x23\x91\x9c\x1f\x67\x86\xbf\x99\x9f\xac\x3f\x92\x2b\x90\xa0\x29\x42\x48\xbe\x71\xdc\x90\x4f\x31\xdd\x02\xd9\xa6\x06\x55\xcc\x7f\x87\x4f\x7f\x22\xa1\x22\x52\x21\x81\x90\xe3\x1f\x26\x13\x9a\xf0\xdf\x40\x1b\xae\x64\x40\xb2\xd9\x64\xcb\x65\x18\x90\x7b\xd0\x19\x67\x70\xce\x98\x4a\x25\x4e\x62\x40\x1a\x52\xa4\xc1\x84\x10\x41\x9f\x40\x18\xfb\x8d\x90\x24\x86\xd8\x63\x86\x4f\xb9\x44\x10\x53\xa6\x62\x3f\x84\x44\xa8\x3c\x06\x89\x01\x11\x59\xec\x25\x5a\x85\x29\x43\xae\xe4\x84\x10\x49\x63\x08\xea\x59\x1e\x53\x12\xb5\x12\x02\x74\x79\xcf\x24\x94\x35\x06\x4c\x3c\xcf\x6b\xe1\xd3\x4f\x94\x4d\x69\x8a\x1b\xa5\xf9\xef\xd4\x2e\x3a\xdd\xfe\xd5\x4c\xb9\xf2\x6b\xe4\x77\x4a\xc0\x5b\xe1\x85\x67\x04\x2d\xa9\xb0\x83\x32\x6e\x21\x81\xf6\xd8\x3a\xda\x83\x5e\xa7\x02\x4c\x30\xf1\x08\x4d\xf8\x95\x56\x69\xe2\x50\x78\xe4\xd3\xa7\x09\x21\x1a\x8c\x4a\x35\x83\xf2\x1a\xc8\x30\x51\x5c\xa2\x99\x10\x92\x81\x7e\x2a\x2f\x47\x80\xee\xff\x6f\x14\xd9\xc6\x7d\x13\xdc\x14\x97\x42\x10\x80\xe0\xbe\xa6\x49\x48\xcb\xaf\x4c\x83\xfd\xda\xb3\xc9\x94\xd2\x21\x97\x4d\x9f\xf5\x41\x08\xa0\x06\xde\x0a\x81\x41\xa5\x69\x04\x7b\x8d\x33\xc3\xcb\x21\x8c\x26\x94\x71\xe4\xfb\xa0\xd4\x00\x76\x98\x4a\xa3\x1d\x28\x49\x7d\xbf\x84\x7a\x50\x28\x12\x15\x0e\x18\xee\x4d\xa5\x49\x62\xfa\x93\x35\x24\x82\x33\x6a\x60\x28\x92\x27\xa4\xf3\x85\x48\x0d\x82\x7e\xf7\xac\xd6\xa9\x94\xa0\x8f\x4a\xe1\xc4\x6e\xcb\x20\x48\xcc\x94\x48\xe3\xd3\xa2\x77\x54\xa0\x3a\x06\x99\xa0\x3c\x3e\xdc\x6a\x99\x28\x47\x27\x6a\x95\xa5\x82\x9a\xbd\xa7\xa5\x63\xee\x30\x0a\xc8\xa0\x77\xfe\x8f\x4c\xf5\xfe\x5e\x24\x4d\xcc\x46\xe1\xf4\xa5\x4d\x95\x21\x2b\x87\x8f\xed\xea\xb5\x6c\x58\xf2\xef\x6f\xf8\x05\x53\x2f\x73\x88\x54\xe1\x6b\x06\xe5\x88\xe5\x4e\xab\x55\x9f\xb9\x0c\xb9\x8c\xde\xac\xc4\x1a\xde\x3e\xd7\x4a\xc0\x58\xc9\x52\x02\xee\x60\x6d\x4d\x57\xce\x19\xd9\xc9\x84\x90\x46\xd1\x3d\xbc\x5c\x9a\xf4\xe9\x9f\xc0\xd0\x11\xcb\x60\xbf\xf1\x2e\x9d\x42\x83\x5a\x3f\x20\x08\xa7\xba\xba\x59\x10\x8e\xa2\xf2\x77\x70\x7a\xb7\x7d\xfc\xb0\xbe\xd1\x24\xc0\xac\xa5\x44\x69\xac\xab\xba\xc6\x80\xcc\xce\xce\xce\xce\x1c\x04\xa4\x3a\x02\x5c\xb5\xae\x1a\x10\xc0\x50\xe9\x02\x24\x4d\x92\x7d\x36\x8f\xdd\xc1\x47\x39\x2a\x06\xd4\x9c\x99\x13\xbc\x34\x1b\xf4\xd2\xec\x4d\xbd\x44\x08\xe6\x09\x04\xe4\x46\x85\x60\x4d\xf6\xdc\x66\x5b\xae\xdd\xf9\xbd\x47\x8a\xb0\x4e\xc5\x3d\x7c\x9c\x40\xa9\x5c\x58\xb5\x7d\x01\x99\xf5\x3c\x14\xdb\xf2\xf0\xa5\x01\xea\x05\xa7\x1d\x8f\xda\x14\x59\x74\x33\x02\x1e\x21\x4e\x04\x45\x28\x21\x35\xdc\xe5\xf0\x48\xa9\xd0\xd1\x4d\x0d\x71\x10\x85\x61\x9a\xda\x08\xd9\xa5\x29\x97\xa0\x4d\x39\x5a\xb4\xb6\xf7\xe2\x06\x8f\xdf\xe2\xde\x59\xdf\xe0\x69\xa3\xd4\x36\x20\x3c\x92\x4a\x83\x1b\x5a\x45\xc5\x21\x59\xaf\xb9\xe4\x98\xef\xb0\xd9\x92\x7e\xde\xbb\x6a\x63\xf8\xaf\x94\x6b\x08\x2f\x53\xcd\x65\x74\xcf\x36\x10\xa6\x82\xcb\x68\xe9\x16\x2e\x2f\x2f\x9e\x81\xa5\x16\x53\x73\x66\xb1\xe6\x7d\x19\xf5\x07\xd0\xb1\x69\xdf\xf6\x8a\x24\x58\x3c\x27\x1a\x8c\x69\xbb\xb9\x1a\xb1\x85\x3c\x18\xda\xe0\x80\xf3\xaa\x8f\x4a\xac\xd0\x57\xda\x1e\x19\x5c\xca\xde\xfd\x8c\x8a\x14\x7a\xa6\x5c\xab\x23\xd5\xa7\xa1\xcb\x6b\x2a\x0c\x54\x77\x76\x51\xae\x96\xb0\x3a\x32\x8e\xa9\x0c\x77\x6b\x7a\xc4\x4f\x8d\xf6\x85\x62\x54\xf8\x4f\x5c\xfa\x75\xd4\x43\xcd\xb3\x06\x68\x8f\x78\xd9\xfc\xcf\xcd\x3f\x85\x8a\x22\x2e\x23\x6f\xad\x74\x4c\x71\x8e\xf0\x8c\xcd\xdb\xb1\x0a\x61\x3e\xb0\x7b\x8f\x78\x95\x5a\x9e\xa7\x92\x3f\x07\xbe\xef\x33\xc3\xfd\x76\xa6\x4d\x8d\x62\xdb\xe6\x1c\x0d\x11\x37\xa8\xf3\x45\x35\x17\x59\x12\xf8\xfe\xd9\xd4\xfd\x0b\x76\x95\xa1\x1c\x6f\x43\xca\xc3\xf9\x0f\x3f\xfe\xe3\xeb\xe7\xc5\xe3\xcd\xed\xe5\xe2\xf1\xe6\xfc\x7a\xf1\x53\x73\x0c\xa3\xbf\x72\x01\x73\x9f\x81\x46\xe3\x33\x3a\x65\xba\xb5\x05\x7b\xbd\x39\x02\x85\xe9\x0e\xd9\x42\xde\x1d\xb1\x85\xbc\x39\xa2\xf0\xa3\x65\x9f\xf9\x0f\x3f\xae\xae\x17\xd7\x8f\x17\xf7\xcb\xc7\xcb\xbb\xe5\x6f\x8b\xbb\x3e\xa4\x92\xf2\xbf\x38\x61\x34\x0f\x76\x54\x6e\x3f\x20\xb3\x66\xe4\x0a\xb6\x6b\xef\x6f\xd2\x49\x9f\x5f\xb5\x8a\xdb\x19\xb4\xe6\x20\xc2\xb2\x73\x69\x7e\x3a\x05\xae\x7d\xd3\x4d\x5a\x51\xdc\x04\xee\x74\x4e\xad\x77\x2d\x5f\xf5\xd0\x3c\x2c\xee\xae\x97\x37\xe7\x0f\xcb\xdb\x9b\xc7\x2f\xb7\x57\x8f\xab\xf3\x87\xbf\x77\x31\x05\xc4\xc7\x38\xf1\x11\x74\x5c\x3e\xd2\xb0\xb9\xd4\x5b\x6a\xc8\x57\xfd\xa5\xfa\x47\xae\xb7\xd0\xd5\xed\xe5\xe2\xf3\xd7\xab\xfe\xdc\xe7\x5f\xce\xfe\x56\xf0\xce\xc5\xcd\x7c\xe7\x66\x1e\xd3\x08\x02\xe2\x16\xec\x9e\x87\x80\x51\x49\x75\xde\x1e\xbb\x4a\x85\x58\x29\xc1\x59\x1e\x90\xe5\xfa\x46\xe1\x4a\x83\x01\xb9\x4b\x94\x46\x5d\xea\x1c\xab\xba\x78\x57\x90\xeb\x53\xbb\xea\xd4\xf2\xe6\x4a\xbb\xc6\xa0\xf8\x18\x60\xa9\xe6\x98\x5f\x58\x51\xf6\x8c\x6d\x5a\xa4\xe1\xad\x14\xf9\x9d\x52\x2e\x97\x4d\x6e\x10\xe2\x80\xa0\x4e\x77\xd1\x6b\x84\xe2\x1a\x8c\xb1\x5b\x72\xb1\x1e\x8d\x53\xa1\x07\xaf\x6d\xf3\xd9\xda\x41\x6c\xaf\x94\xf3\xdd\xa9\xe8\xc1\xaf\xce\xb2\x3b\x60\xfb\x66\x1a\xde\x9b\x97\x88\x34\xe2\xd2\xb3\xd4\x00\xe8\x85\x5c\xef\x99\x8b\x71\xd2\x9b\x8b\x71\xd2\x98\xe1\x11\xaa\xa3\x16\x6a\xcf\x2b\x9f\x5b\xe5\x0d\x16\xf2\x76\x45\xad\xe4\x33\x90\xa8\xa9\x18\x61\x45\x97\x2b\x34\x0c\x6d\xa5\x98\x1f\x42\x6c\xde\x1a\x28\xa6\x1a\xbc\x88\x22\x98\xf9\x83\x4a\x94\x50\x51\x3e\x6f\x05\xc8\x8e\x33\x36\xe6\xe8\xe1\xde\x01\xc8\x63\x50\x29\xce\x7f\x89\x5b\x97\x43\x58\xd3\x54\xa0\xb7\x36\xb6\x3f\x9b\xc3\x33\xfe\xa5\x75\xbf\x4c\xa6\x1a\xb4\x23\x9d\xd9\x38\xe9\xac\x6e\x2f\xdd\x79\xbc\x5f\x9d\x5f\x7c\x07\xe7\x34\x68\xa5\xea\x69\xa6\x75\x9f\xb6\xd7\xe8\xeb\xdb\xeb\x1e\x7d\x2b\xda\x22\xa6\xad\xde\x34\x3c\xf2\xca\xe7\x16\x7e\x47\x08\x06\xd9\xcf\xd3\xb3\xe9\xcf\x27\x71\xc1\x90\xd4\x3b\x8a\x14\x66\x6f\x4e\x0a\x07\x9d\xef\x23\x4e\xe9\x5e\x3c\x3a\x95\xe7\xe6\x46\x49\x8b\xa6\x83\xc1\xdd\xfa\x6a\x40\x17\x2a\xaf\x5e\xa9\xa9\x7c\xc7\xfa\x66\xfb\x41\x25\x6c\x9f\xd5\x6c\xdb\x3c\x02\xeb\x35\x30\xb4\x9d\x57\xd9\x29\xee\x6c\xba\x4e\xce\x16\x38\x27\xf4\xa7\xdb\xf4\x09\xb4\x04\x04\xf7\xf8\x21\xa6\x56\xbe\x4f\x9a\x1e\x6a\xae\x1a\x27\x98\x5f\x72\x1d\x10\x99\x0a\xd1\xa5\xff\x3d\x7e\xf1\x46\x59\xd1\x00\xd3\xd0\x0a\x5e\x71\xa5\xb3\xeb\x7a\x72\x71\xd7\x0c\x40\xfa\xf7\x7f\x3a\x80\x2a\x4a\x1c\xd7\x69\x97\x14\x62\x25\xdf\x50\xa5\x59\x57\xbf\xa0\xcf\x8e\x54\x64\xe5\x8a\xa7\x60\xfb\x50\xa1\xd5\xc0\xfd\x8e\x12\xeb\xd5\x75\x42\x08\xf6\x74\x5e\x53\x49\x23\xd0\x73\x91\xc5\xdf\x29\x22\x5a\x6e\x19\x91\x0f\xbd\xd2\x7a\x90\x06\xa8\xe9\xa2\xa3\x2c\xca\x59\xb6\xe8\x2c\x57\x3f\x39\x81\x31\x3b\x40\x90\x0c\xf0\x50\x5f\x9c\xbc\x8b\xf0\x30\x48\xd1\xf5\x71\x73\x3f\xa3\xda\x17\xfc\xc9\x3f\x40\x80\x1c\x27\x57\xec\x6e\x57\xa0\x6d\x5b\x44\x23\x98\xcf\xda\xdb\xfc\x5f\x17\x33\x8d\x04\x78\x07\x28\x48\x31\x35\xd3\x44\x85\x0d\x6b\xaf\xae\x85\x5e\x51\x9f\xfd\x5f\xcb\xaa\x44\xf3\x8c\x0b\x88\x20\xec\x74\x2c\xad\x9e\xe5\xec\xed\x15\x56\x75\xb4\x6d\x9b\x22\x00\xfd\xa2\xcb\x30\x7e\xbb\x6d\x69\xf7\x69\xc5\x7c\xad\x12\x1a\x39\xa3\x01\xf9\xcc\x43\xae\xc1\x95\x92\x86\xba\xa9\xfd\x62\xc7\x3b\x92\x1b\x11\x5d\x3d\x20\xc5\xef\xf0\xa7\xda\xb4\xd3\x47\xac\x0d\x0b\x4b\xd7\xb5\x8d\x88\xca\x10\xb2\xde\x9c\x10\xb2\x11\x3b\x26\xef\x5b\x31\xf9\x28\xb2\x81\x96\xf8\x45\xc5\x5a\x39\x6f\xe4\x04\x9f\xe0\x42\xbb\x98\xab\x02\xe3\xca\xb7\xab\x60\xcb\x08\x56\x15\xae\xc8\xcb\xa4\x55\x48\xba\xd9\xb6\xa7\x54\x0c\x56\xe5\x41\x85\xdc\x1e\x35\xac\x39\xbf\x97\x03\x5f\x16\x77\x2e\x7f\x0a\x2a\xa9\x77\xaf\x83\x6c\x36\xfd\x79\x7a\x76\x12\x37\x75\xd7\xfa\x70\x71\xf6\x62\x26\x36\x83\xde\x9b\xdd\xca\x88\xdd\x1a\xcd\xc7\xe7\x3b\x30\xa5\x63\x8b\x90\xec\x53\x4c\x1b\x65\x0a\xcb\x4d\x6e\x1d\x67\xb6\xd1\x33\x52\xfc\x04\x75\xe9\x4e\x85\xd2\xf9\xad\xbe\xa8\xde\xb7\x18\xf5\xc2\x29\x40\x1e\xab\x0e\xd0\x3f\xda\xfe\x1e\x3f\x9e\xe4\x8e\x31\xa2\x3f\x0c\xcd\x20\xbd\x1f\x87\xa5\xcd\xf5\x87\x99\xed\x30\xbc\xb7\x97\xc3\x0f\x94\xc0\x6e\x62\x57\xfe\x1e\xb0\x89\x57\xc8\xa7\x3d\x4c\x3b\x62\xbc\x5d\x89\x0e\xb3\xd2\xae\x54\xa3\xcb\xb7\xcb\xd6\x81\x87\xa2\x2c\x6b\xdd\xc7\x01\xed\xb7\x76\xfc\x6c\xf6\x04\x48\xeb\xb7\x30\xee\x97\x97\x45\xeb\xf5\x36\x4f\x07\x1a\x41\xa9\x34\x2b\x45\xa4\x6c\x73\x57\xfe\xdc\x17\x10\xf7\xa3\x97\xfb\x1d\x3c\x5c\xca\xb5\xba\x95\x8e\x15\x6b\xbe\x2c\xe1\x5f\x94\x4f\x77\xeb\xeb\x05\x15\x7d\xe1\x6b\x60\x39\x13\x70\xad\xc2\xea\xa5\xa1\x55\xfd\x62\x9a\xfb\x73\x91\x6c\x20\x06\x4d\xc5\xe4\xbf\x01\x00\x00\xff\xff\xa5\xeb\x20\x7d\x9d\x2b\x00\x00") +var _deployKubernetes119AlphaLvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x02\x95\x15\xf7\x02\x0c\x04\xe8\x21\x9b\x64\xa7\x41\x27\x89\x91\xcc\xee\x6b\x40\x53\xc7\x32\x6b\x8a\x64\x49\x4a\x33\xda\xa2\xff\xbd\xa0\x44\xc9\xba\xd8\x8a\xac\x38\x97\xae\x81\xc1\x38\xd4\x21\xcf\x39\x1f\xcf\xe5\x23\xad\x3f\xa2\xcf\xc0\x41\x61\x03\x11\xfa\x46\xcd\x06\x9d\x25\x78\x0b\x68\x9b\x6a\x23\x12\xfa\x1b\x9c\xfd\x19\x45\x02\x71\x61\x10\x44\xd4\xfc\x61\x36\xc3\x92\xfe\x0a\x4a\x53\xc1\x03\x94\x2d\x66\x5b\xca\xa3\x00\x3d\x82\xca\x28\x81\x0b\x42\x44\xca\xcd\x2c\x01\x83\x23\x6c\x70\x30\x43\x88\xe1\x15\x30\x6d\xbf\x21\x24\x13\x48\x3c\xa2\xe9\x9c\x72\x03\x6c\x4e\x44\xe2\x47\x20\x99\xc8\x13\xe0\x26\x40\x2c\x4b\x3c\xa9\x44\x94\x12\x43\x05\x9f\x21\xc4\x71\x02\x41\x3d\xcb\x2b\x66\x79\x44\x24\x1e\x11\xdc\x28\xc1\x18\x28\x27\xa5\x25\x26\x0d\xd1\x99\xe7\x79\x1f\xc3\xd2\x6f\xb0\xda\x08\xb1\xd5\x23\xed\x54\x2b\x4c\xe6\x38\x35\x1b\xa1\xe8\x6f\xd8\x2e\x3e\xdf\x7e\xd2\x73\x2a\xfc\xda\x83\x07\xc1\xe0\xf5\xed\x86\xef\x06\x14\xc7\xcc\x8a\x67\xd4\x1a\x07\xca\x23\xeb\xf8\x80\x1f\x2a\x65\xa0\x83\x99\x87\xb0\xa4\x9f\x95\x48\x65\x61\x8f\x87\xce\xce\x66\x08\x29\xd0\x22\x55\x04\xdc\x18\xf0\x48\x0a\xca\x8d\xc5\x24\x03\xb5\x72\xc3\x31\x98\xe2\xff\x6f\xd8\x90\x4d\xf1\x8d\x51\x5d\x0e\x45\xc0\xc0\x40\xf1\x35\x95\x11\x76\x5f\x89\x02\xfb\xb5\xa7\x93\x08\xa1\x22\xca\x9b\xe8\xf5\x8d\x60\x80\x35\xbc\x96\x05\xda\x08\x85\x63\x38\xa8\x9c\x68\xea\x44\x08\x96\x98\x50\x43\x0f\x99\x52\x1b\xb0\xb3\xc9\x29\xed\x98\x22\xeb\xe7\xce\xd4\x51\x5b\x21\x45\xb4\x47\x71\x6f\x2a\x96\x52\xf7\x27\x2b\x90\x8c\x12\xac\x61\xdf\x4e\x7e\xd8\xc0\xae\x12\xf2\x44\xc1\xbc\x1f\xc1\x3d\x51\x34\x01\x90\x4b\x96\x6a\x03\xea\x1d\x13\x5e\xa5\x9c\x83\x3a\x0e\x10\xeb\xa0\x36\xc0\x4d\x26\x58\x9a\x4c\x0b\xec\xa3\x62\xb8\xa3\x90\x30\x4c\x93\xf1\x5a\x5d\x0e\x1d\x9d\xc3\x55\x02\x33\xac\x0f\x16\x92\x8e\xba\x71\xd5\x31\x83\x5e\x69\x3c\xb2\x0a\xf4\x7d\xe1\x58\xea\x8d\x30\xf3\xe7\x9c\x72\x5b\xe6\xc4\x87\xbc\x3a\x95\x0e\xdb\xc9\xfb\x0e\x3f\xa3\xea\xf9\xf2\xca\x45\x74\xca\x4d\x39\x62\xb9\x0f\x9e\xe6\x75\xf9\x3b\x51\x6a\x1f\x85\xcd\x1b\xa4\xb3\x7c\x36\xb1\xc7\x26\xdd\x31\x4a\x5e\x42\x06\x7a\x85\xe4\xd8\x00\x9e\xd6\x69\x7f\xa2\x3c\xa2\x3c\x7e\x03\xae\xae\x69\xbb\xa7\x08\x06\x43\xcd\x57\x30\x78\x80\xb5\x35\xa2\x02\x72\xc0\xa7\x19\x42\x0d\xf2\x30\x85\xcf\xea\x74\xf5\x2f\x20\xa6\xc8\x81\xbd\x47\x84\x53\x1e\x43\x3e\xda\xde\xec\xaa\xc1\xbb\x6c\x4a\x8b\x8b\xbd\x64\x23\x4e\x7f\xca\x6a\x14\xe5\x77\xcd\x94\xa9\xd0\x37\x9b\xca\x44\xd6\xf7\xd1\x32\xe3\x3d\xb6\xa4\x95\x20\xaf\xbc\x17\xdd\xd6\xfc\x86\x09\xd1\xbd\x1e\xf9\x00\x37\x38\x5a\x02\xb1\x3a\xa5\x50\xa6\x3e\x6b\x29\x13\xa0\xc5\xf9\xf9\xf9\x79\x61\x8c\xc1\x2a\x06\xb3\x6c\x8d\x6a\x60\x40\x8c\x50\xa5\xb9\x58\xca\xf9\x36\x5d\x81\xe2\x60\xa0\x88\x26\xca\xb5\xc1\xbc\xa9\x69\xe7\xca\x81\x29\x1d\x1f\x5a\x96\x1f\x8b\xc8\xfb\x03\x9f\x80\x51\x94\x1c\x0a\x8c\x21\xd4\x17\x7b\x51\x5f\xfc\x1f\xa0\x8e\x90\xc9\x25\x04\xe8\x4e\x44\x60\x0d\xef\x6d\x03\x96\x52\xef\x0a\xcd\xa3\xc1\x06\xd6\x29\x7b\x84\xc3\x17\x84\x7d\x83\x89\x48\xa4\xe0\x85\xfa\x8e\xb1\x6f\x80\x47\x5f\x5e\x62\x65\x3c\xb1\x6e\xec\xed\xfb\xe4\x6f\x75\x4b\x14\xa0\x45\x2f\x50\x12\x4b\x60\xbf\x34\x50\x9d\x84\xd5\xf1\x68\x1d\x8f\x83\x2e\xd3\xf3\x6e\x14\x1c\x06\x12\xc9\xb0\x01\xe7\x64\x23\x82\x0a\x63\x39\x17\xa6\xe8\x13\xb5\xd3\x7b\xed\xd1\x44\x61\x1b\xb4\x76\x69\x4c\x39\x28\xed\xa4\x59\x0b\xb0\xe3\x42\x71\x32\xc4\x53\x40\x1e\x1f\x96\x53\xb6\xe4\xe0\x2c\xd7\xfb\x02\x44\x63\x2e\x14\x14\xa2\x55\x34\xda\xcf\x0e\xd0\x6a\xc4\x43\x44\x24\x09\xe6\xd1\x0e\x54\x0f\xf9\xa9\x56\x3e\x13\x04\x33\x7f\x45\xb9\x5f\xbb\x19\x29\x9a\x35\x5c\xf4\x90\x97\x85\x7f\x6d\xfe\xc9\x44\x1c\x53\x1e\x7b\x6b\xa1\x12\x6c\x42\x03\xdf\x4d\xf3\x71\x22\x22\x08\x1b\xfd\xb9\x7e\x50\x2e\x6c\x41\x0d\x7f\xf8\x71\x79\x7b\x7d\xfb\x74\xf9\x78\xf3\x74\xf5\x70\xf3\xeb\xf5\xc3\xd3\xdd\xc5\xed\xf5\x9f\x9a\xd2\xf6\x80\xf8\xe8\xb2\x29\xfc\xcf\x99\x3b\x45\x9e\x05\x67\xd6\xd0\xb3\xff\x36\x45\x09\xfe\x99\x32\x08\x7d\x02\xca\x68\x9f\xe0\x39\x51\x2d\x8b\xec\x78\x53\xc2\x30\xdd\x15\xd9\x42\xde\x95\xd8\x42\xde\x94\xd0\x64\x03\x51\xca\x40\x7d\x29\x4e\xef\x61\xf0\xa9\x6a\xd3\x95\xdf\x65\xe3\xa9\x1e\xef\x1a\x8a\xfd\x00\xcf\x9a\xd8\x97\x91\xf5\xf5\xfa\xe1\xf6\xe6\xee\xe2\xeb\xcd\xfd\xdd\xd3\x97\xfb\xcf\x4f\xcb\x8b\xaf\xff\xa8\x85\x10\xca\x30\x4b\x21\x40\x7e\x04\x99\x6f\x40\x25\xee\xca\xdf\xe2\xdf\x5b\x6a\x1f\x9c\xfd\xa5\x06\xa2\xbf\x5e\xe8\xfe\xaa\x98\xfc\xb8\xbc\xb8\xec\xad\xf0\xb3\x12\x49\xd0\x18\x44\x68\x4d\x81\x45\x8e\x32\xf6\xc6\x97\xd8\x6c\x82\xba\x3e\xcc\xeb\x2a\x5a\xcb\xd2\x04\xc7\x10\xa0\xc2\x9a\x6e\x00\x06\x04\x73\xac\xf2\xb6\xec\x32\x65\x6c\x29\x18\x25\x79\x80\x6e\xd6\x77\xc2\x2c\x15\x68\xe0\xbb\xad\x6c\xa4\x6c\x27\x8e\xeb\x8e\x5f\xf9\x5b\xa7\xc9\xb2\x43\x00\x9a\x2b\xed\xd8\x44\xf9\xd1\x40\x52\x45\x4d\x7e\x29\xb8\x8d\xfb\xa6\xdb\x0a\x70\x74\xcf\x59\xfe\x20\x44\x11\x6d\x3a\xd7\x06\x92\x00\x19\x95\xee\x3c\x6e\xec\xe3\x2d\x68\x6d\x5d\x2a\x40\x1a\xdc\xe4\xf2\xa2\xe8\xd6\x72\xe2\x96\x07\x89\x1d\x71\xf3\x8b\xb8\xed\x99\xef\xd2\xb0\xc8\x80\xd9\x33\x1e\xa8\x94\x5f\xe8\x3b\xc1\xad\xfd\x1d\xab\x8b\x47\xbf\x68\x50\x25\x0f\xad\x57\x6a\xb2\xf5\x83\xad\xa3\x53\x0a\x8c\x60\xa0\xda\xed\xc1\x43\xb0\x5e\x03\x31\x96\xc0\x3c\xba\x34\xab\x75\x6f\x21\x0f\x8a\x3b\xb8\xe2\x88\xd2\xa9\xb6\x09\xb6\x07\x8f\x59\x13\xa6\xc6\xaa\x07\x31\x28\x50\x50\xd0\xda\xbe\x72\x64\x44\x03\xf4\x4a\xc9\x67\x38\xd6\x15\x86\x44\xf0\xc9\x0c\xcb\x3a\xfc\x6a\xdc\x6a\x60\xf1\x57\x66\x55\x4e\xf3\x10\x9f\x7a\x27\x06\x55\x63\x32\x81\x7b\x7f\x20\x3a\xd4\x70\xe3\xf5\x88\xd0\x33\x4a\x7e\x8f\x14\x28\x02\x5b\xea\x6e\x31\xc7\x31\xa8\x90\x65\xc9\x0b\xf9\x51\x0b\x42\x0f\x79\xd5\x3b\x03\x61\xca\xe9\xf7\xc0\xf7\x7d\xa2\xa9\xfd\x37\xd7\x82\x6c\xbb\xb4\x88\x46\xe1\x0f\x3f\xfe\xf3\x97\x9f\xae\x9f\xee\xee\xaf\xae\xfb\xd4\x49\xdb\xf3\x9d\xed\x0b\xa1\x9f\x61\xe5\x33\xba\xf2\x47\x10\xae\xe3\xe8\x99\xc5\x6a\x09\x8a\x00\x37\x38\x86\x70\xf1\x52\x2e\xd4\x76\x67\x3a\xf1\xe8\x5c\x3e\xb4\x1f\x36\x58\x89\x8d\x96\xb9\x05\xd3\x56\xfc\xd7\xa3\x53\x23\x29\x9e\x49\xe4\xc1\xee\xff\xbb\x22\x49\x52\xd1\x8c\x32\x88\x21\xea\x50\x8c\x16\xc9\x38\x7f\x9e\x2f\x0d\x21\x36\x86\x2f\x55\x89\x61\x4b\x17\x03\xe3\x4b\x96\xc6\x94\x6b\xbf\x53\x59\x1b\xc5\x0b\xb9\xf9\x4a\x48\x1c\x17\x4a\x03\xf4\x13\x8d\xa8\x82\xa2\x5c\x61\xd6\xc7\xc5\xca\x17\x69\xed\x45\x54\x8d\x35\xa4\x7c\xe1\x62\xaa\x4e\x3b\x7d\x40\x5b\x04\x59\x6f\x4a\x04\xd9\xc0\x0c\x9d\xf7\x69\xa5\xce\x87\x74\xb4\x41\x73\x33\x04\xd9\xc2\x18\x18\x06\x72\x6a\x02\x18\x76\xb1\xa2\x1a\x36\x34\x7b\x08\xab\xb8\x15\x17\xdd\x02\xee\xb9\xbd\xf0\x14\xc4\x54\x9b\x92\xad\x7a\xb2\x55\x50\xbb\x71\x73\xa0\x64\xee\xad\xe1\x45\x02\xe3\x28\x52\xa0\x75\xb8\xbf\xd2\xef\xad\x91\x2f\xae\x4a\xae\x94\x6c\x3f\xe9\x79\x4c\x94\x0d\x70\x4d\x63\xcf\x1d\xad\xfd\xaa\xb5\xbb\xa2\x50\x7b\xaf\x82\x6c\x31\xff\xcb\xfc\x7c\x52\x95\xe9\xae\x75\xb2\x93\xd4\xa8\x43\xd1\x94\x48\x6c\x6e\x7a\x6f\x76\x2b\x22\x06\x63\xca\x03\x8e\x57\x0c\x3c\xf7\x06\x5c\xde\x0e\x00\x37\xe8\x89\x6f\x1c\x94\x82\xb5\xc7\x20\x03\x16\x9e\x0f\x45\xe5\xf3\x51\x63\xa5\xd6\x80\x4d\xaa\xc0\x8b\xb1\x01\x1d\x7e\x15\x52\x30\x11\xe7\x61\x0b\x3b\x2b\x57\xee\x74\xcd\xbe\xfa\x02\xda\x16\x75\xe3\x99\x83\x2b\xd0\x24\x81\x88\xda\xe4\xaa\x65\xd6\x98\xe9\xb6\x90\xa1\x09\x88\xd4\x84\x7f\x6f\x31\x26\x2f\x82\x35\x4e\x99\xf1\xd6\xda\xe4\x12\x42\xf8\x6e\xfe\xd6\x7a\xee\x5a\x4a\xed\x6f\x41\x21\x16\xc3\xe9\xf1\x71\xd8\xc3\x5b\xde\xa1\x74\x95\x9e\x5e\x5f\xb7\x7c\xb8\xd2\xb1\xfd\xa4\x6d\x6d\x2d\xa8\x6e\xb7\x8c\x34\x7e\x6d\x7d\x09\x45\xd9\xf7\xeb\xed\x51\x5c\x65\xf1\x12\xae\xf2\xb6\x65\xa8\x79\xdf\xb9\x5b\xc8\x81\x5a\x56\xf5\x59\x65\xf2\xa8\x7b\x97\xde\x75\x75\xef\x7e\x64\x23\x74\x69\x62\x93\xa0\x0d\xd3\xa3\xc1\xf6\x5c\xfe\x08\x75\x55\x34\x64\xa1\xf2\x7b\x75\x59\xbd\xc4\x37\xe8\xf9\x14\x43\x9e\x5c\x21\xce\xfd\xa3\xf5\x1f\x2c\xe1\x13\xe0\x18\x62\x8b\xe3\xac\xd9\xcb\x11\x8f\xb3\xa5\x4d\x18\xc7\xa9\xed\xd0\xc4\x51\x0a\x4f\xb0\xf7\x07\x08\xd9\x80\xf2\x36\x61\x1d\xa7\xa5\x4d\x68\x07\x97\x6f\xb3\xdb\x91\x01\xec\xd8\x6f\xf7\x02\xb0\xfd\x22\x9c\x9f\x2d\x56\x60\x70\xfd\x72\xc7\xe3\xcd\x55\x79\xd6\x7a\x9d\x5f\xc0\x1b\x9b\x52\x5d\x84\x60\x63\x30\xd9\x3c\xc0\xbf\x53\xaa\xec\x79\xab\x6a\xce\x52\x44\x37\x7c\x2d\xee\x79\x51\xb5\xea\x7a\xe6\xcc\xbf\x74\xd4\xa4\x1e\x2f\xcb\xc6\x17\xba\x06\x92\x13\x06\xb7\x22\xaa\xde\xec\x5b\xd6\xaf\x32\x16\x7f\x5e\xcb\x0d\x24\xa0\x30\x9b\xfd\x2f\x00\x00\xff\xff\x6f\x94\x5a\xfc\x75\x33\x00\x00") func deployKubernetes119AlphaLvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -200,12 +201,12 @@ func deployKubernetes119AlphaLvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 11165, mode: os.FileMode(436), modTime: time.Unix(1610053026, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 13173, mode: os.FileMode(420), modTime: time.Unix(1611067508, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\x78\xdb\x03\x5a\x01\x7e\xc8\x26\xbe\xd4\xe8\xc6\x31\x92\xec\xbd\x06\x0c\x35\x96\x59\x53\x24\x4b\x8e\xbc\xf1\x15\xfd\xdf\x0b\x52\x92\xad\x2f\xcb\x1f\x97\x0f\x14\xf5\x3e\xac\x43\x71\x38\xbf\xf9\xe0\xcc\x6f\x6c\xff\x91\xdc\x80\x04\x43\x11\x62\xf2\x9d\xe3\x92\x7c\x4a\xe9\x0a\xc8\x2a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x18\x50\xcd\x7f\x05\x63\xb9\x92\x11\x59\x8f\x06\x2b\x2e\xe3\x88\x3c\x80\x59\x73\x06\x97\x8c\xa9\x4c\xe2\x20\x05\xa4\x31\x45\x1a\x0d\x08\x11\xf4\x19\x84\x75\xef\x08\xd1\x29\xa4\x01\xb3\x7c\xc8\x25\x82\x18\x32\x95\x86\x31\x68\xa1\x36\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x20\x44\xd2\x14\xa2\xad\x60\xc0\x94\x44\xa3\x84\x00\x53\x3c\xb3\x9a\xb2\xca\x86\x41\x10\x04\x35\x88\xe6\x99\xb2\x21\xcd\x70\xa9\x0c\xff\x8d\xba\x43\x87\xab\xbf\xda\x21\x57\xe1\x16\xfc\xbd\x12\xf0\x86\x90\xe1\x05\xc1\x48\x2a\xdc\xa6\x35\x77\xa8\xc0\x04\x6c\x91\xec\x31\xc0\x64\x02\x6c\x34\x08\x08\xd5\xfc\xc6\xa8\x4c\x7b\x20\x01\xf9\xf4\x69\x40\x88\x01\xab\x32\xc3\xa0\x58\x03\x19\x6b\xc5\x25\xda\x01\x21\x6b\x30\xcf\xc5\x72\x02\xe8\xff\xff\x4e\x91\x2d\xfd\x3b\xc1\x6d\xbe\x14\x83\x00\x04\xff\x36\xd3\x31\x2d\xde\x32\x03\xee\x6d\x4b\x27\x53\xca\xc4\x5c\x56\xdd\xd6\x06\x21\x80\x5a\x78\x2b\x04\x16\x95\xa1\x09\xec\x55\xce\x2c\x2f\xb6\x30\xaa\x29\xe3\xc8\xf7\x41\xd9\x02\xd8\x61\x2a\x94\x36\xa0\xe8\xed\xf3\x02\xea\x51\xa1\xd0\x2a\xee\x50\xdc\x12\xa5\x5a\xdb\xb6\xb0\x01\x2d\x38\xa3\x16\xba\x22\x79\x46\x46\x5f\x89\xcc\x22\x98\x8f\x48\x6c\x93\x49\x09\xe6\xa4\x2c\xd6\xce\x32\x8b\x20\x71\xad\x44\x96\x9e\x17\xc0\x93\x62\xd5\x50\xc8\x04\xe5\xe9\xf1\x5a\x8b\x5c\x39\x39\x57\xcb\x44\x15\xd4\xee\xbd\x30\x0d\x75\xc7\x55\x81\x35\xb4\x4a\xc0\x89\xd9\xde\xb6\x45\x52\x6d\x97\x0a\x87\x87\x8c\x2a\x42\x56\x6c\xef\xb3\xea\xb5\x74\xb8\x16\xd0\x36\xf8\x80\xaa\xc3\x65\x44\xaa\xf8\x35\x83\x72\xc2\x71\xe7\x75\xac\x2f\x5c\xc6\x5c\x26\x6f\xd9\x6b\x2d\xaf\x5f\x6d\x25\xa0\xaf\x71\x29\x01\xf7\xb0\x70\xda\x4b\xff\xf4\x18\x33\x20\xa4\xd2\x7d\x8f\x6f\x9a\x36\x7b\xfe\x27\x30\xf4\xb5\xa5\x93\x7b\xbc\x0b\x65\xa8\x14\xd8\x8f\x89\xc3\xb9\xde\xae\x76\x86\x93\x0a\xfa\x3b\xf8\xbd\xc9\x26\x3f\x92\x46\x5a\x0d\xcc\x29\xd3\xca\xe0\xb6\xc3\x1b\x8c\xc8\xe8\xe2\xe2\xe2\xc2\xa3\x40\x6a\x12\xc0\x79\x6d\xd5\x82\x00\x86\xca\xe4\x38\xa9\xd6\xfb\x74\x9e\x61\xc4\x07\xba\x2b\x05\x34\x9c\xd9\x33\x7c\x35\xea\xf4\xd5\xe8\xad\x7d\x45\x08\x6e\x34\x44\x64\xa6\x62\x70\x5a\x5b\xce\x73\x3c\x6c\x77\x9d\x1f\x90\x22\x2c\x32\xf1\x00\x1f\x3a\xbb\x94\x8e\x2c\xe9\x60\x44\x46\x2d\x3f\xa5\xae\x67\x7c\xad\xe0\x3a\xe0\xba\xb3\x80\xdb\x3c\xa3\x66\x3d\xf8\x11\x52\x2d\x28\x42\x81\xaa\xe2\x34\x0f\x49\x4a\x85\xbe\x00\x6d\x51\x76\x02\xb1\xcc\x50\x17\x27\x77\x34\xe5\x12\x8c\x2d\x76\x8b\x9a\x85\x07\x6d\x3c\xcb\xca\xbd\x82\xdf\xe1\x79\xa9\xd4\x2a\x22\x3c\x91\xca\x80\xdf\x5a\xc6\xc6\x83\x59\x2c\xb8\xe4\xb8\xd9\xc1\x73\xdd\xfe\xb2\xb5\xea\x22\xf9\xaf\x8c\x1b\x88\xaf\x33\xc3\x65\xf2\xc0\x96\x10\x67\x82\xcb\x64\xea\x0f\x2e\x96\x27\x2f\xc0\x32\x87\xa9\x2a\x99\x9f\xf9\x50\xc4\xfe\x11\x4c\x6a\xeb\x8f\x83\x3c\x15\x26\x2f\xda\x80\xb5\x75\x4f\x97\x3b\x56\xb0\x89\xba\x0c\xec\xf0\x5f\xf9\x52\x1a\x0c\x75\xd9\x46\x66\x0a\xa7\xb2\xf5\x7c\x4d\x45\x06\x2d\x55\x9e\x05\x49\xf5\xa9\x6b\x79\x41\x85\x85\xf2\xc9\x2e\xd0\xe5\x11\x6e\xca\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x33\x97\xe1\x36\xf0\xb1\xe1\xeb\x0a\xe8\x80\x04\xeb\xf1\x9f\xab\x7f\x0a\x95\x24\x5c\x26\xc1\x42\x99\x94\xe2\x18\xe1\x05\xab\x8f\x53\x15\xc3\xb8\xc3\xfa\x80\x04\xe5\x2c\x3d\xce\x24\x7f\x89\xc2\x30\x64\x96\x87\xf5\x64\x1b\x5a\xc5\x56\x55\x19\x03\x09\xb7\x68\x36\x93\x52\x16\x99\x8e\xc2\xf0\x62\xe8\xff\x45\xbb\x5e\x51\xec\x77\x21\xe5\xf1\xf8\x87\x1f\xff\xf1\xed\xcb\xe4\x69\x76\x77\x3d\x79\x9a\x5d\xde\x4e\x7e\xaa\xee\x61\xf4\x17\x2e\x60\x1c\x32\x30\x68\x43\x46\x87\xcc\xd4\x4c\x70\xeb\xd5\x1d\x28\x6c\x73\xcb\x0a\x36\xcd\x1d\x2b\xd8\x54\x77\xe4\x7e\x74\x35\x68\xfc\xc3\x8f\xf3\xdb\xc9\xed\xd3\xd5\xc3\xf4\xe9\xfa\x7e\xfa\xeb\xe4\xbe\x0d\xa9\x28\xff\x5f\xfd\xcc\x34\x8e\x76\x65\xdd\xbd\x40\xae\xab\x91\xcb\x6b\x5e\xdd\xbe\x41\x23\x7d\x7e\x31\x2a\xad\x67\xd0\x82\x83\x88\x0b\x3a\x53\x7d\x35\xfa\x5d\xfd\xa1\x17\x9a\x53\x5c\x46\xfe\x76\x0e\x9d\x77\x5d\xc9\x6a\xa1\x79\x9c\xdc\xdf\x4e\x67\x97\x8f\xd3\xbb\xd9\xd3\xd7\xbb\x9b\xa7\xf9\xe5\xe3\xdf\x9b\x98\x22\x12\x62\xaa\x43\x04\x93\x16\x1f\x78\xb8\x5c\x6a\x1d\xd5\xe5\xab\xf6\x51\xed\x2b\xd7\x3a\xe8\xe6\xee\x7a\xf2\xe5\xdb\x4d\x5b\xf6\xe5\xe7\x8b\xbf\xe5\x75\xe7\x6a\x36\xde\xb9\x99\xa7\x34\x81\x88\xf8\x03\x9b\xf7\x21\x62\x54\x52\xb3\xa9\xef\x9d\x67\x42\xcc\x95\xe0\x6c\x13\x91\xe9\x62\xa6\x70\x6e\xc0\x82\xdc\x25\x4a\xa5\x3b\x35\xae\xd5\xb6\x91\x97\x90\xb7\xb7\x76\xde\xe8\xeb\xd5\x93\x76\x24\x21\x7f\x59\x60\x99\xe1\xb8\xb9\x72\xf3\xda\x0b\xd6\xcb\x22\x8d\xef\xa4\xd8\xdc\x2b\xe5\x73\xd9\x6e\x2c\x42\x1a\x11\x34\xd9\x2e\x7a\x95\x50\xdc\x82\xb5\xce\x24\x1f\xeb\xde\x38\xe5\xa3\xe2\xad\x63\xa4\x35\x0b\x52\xb7\x52\xc8\xfb\x5b\xd1\x82\x5f\xde\x65\x7f\xc1\xf6\x49\x5a\xde\x92\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x5b\xb2\x98\xea\x8a\x44\x40\xa8\x49\x6a\xa8\x9b\xd5\xcd\xc7\x9c\xc6\xb1\xab\xf8\xe3\x63\x0a\x54\xb0\x00\x8a\x99\x81\x20\xa1\x08\x76\xfc\xa8\xb4\x12\x2a\xd9\x8c\x6b\x8e\x76\xfb\xac\x8b\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xe3\x9f\xd3\xda\x72\x0c\x0b\x9a\x09\x0c\x16\xd6\x11\xae\x31\xbc\xe0\x5f\x6a\xcf\x8b\xa4\xd8\x82\xf6\xc5\x63\xd4\xcc\x6a\x37\xa4\x24\xcc\xb8\x11\xcb\xf2\x24\x28\xa6\xf5\xb0\x31\xf8\x44\xeb\xcf\xc3\x8b\xe1\xe7\xb3\xd2\xbc\x6b\xb4\x39\x29\xdf\x47\x6f\x9e\xef\x47\xa5\xee\x09\x09\xb8\x17\x8f\xc9\xe4\xa5\x9d\x29\xe9\xd0\x34\x30\xf8\x47\xdf\x2c\x98\x7c\xa4\xd9\x9e\x54\x9d\xf4\xfa\x58\xa1\x7b\xa1\x12\x8e\x42\x54\x19\x49\x40\x60\xb1\x00\x86\x8e\x54\x14\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x60\x3b\x5c\x65\xcf\x60\x24\x20\xf8\x89\x3b\xa5\x6e\x5c\x1d\x54\x3d\x54\x3d\x35\xd5\xb8\xb9\xe6\x26\x22\x32\x13\xa2\x59\xd9\xf6\xf8\x25\xe8\xbd\xf0\x16\x98\x81\x5a\xf0\xf2\x95\x86\xd5\x5b\xe1\xfc\xa9\xed\x80\xf4\xef\xff\x34\x00\x95\xb7\xbd\x7f\x16\xb9\xa6\x90\x2a\xf9\xb6\x93\x88\xf3\xf6\x81\x19\xe4\xc4\xa9\xa3\x38\xf1\x4c\x78\x1f\x3a\x4c\x54\xa0\xbf\xef\x18\xf1\xea\x5c\x38\x06\x77\x4d\x6f\xa9\xa4\x09\x98\x71\x8e\xf5\x77\x72\xe5\x9a\x73\x7a\x58\x72\xab\xf3\x1c\x45\x75\xb7\xa5\xa3\x41\xa0\x0b\xa9\xf9\xdd\xf5\xd3\x74\xfe\x93\xe7\xd1\xa3\x23\x78\x77\x47\x4d\x6a\x73\xf0\x77\xe1\xd7\x16\x29\x7a\xba\x32\x0e\xd7\xd4\x84\x82\x3f\x87\x47\xf0\xec\xd3\x58\xb9\xb3\x76\x0e\x86\x81\x44\x9a\xc0\x78\x54\x37\xf3\x7f\x9d\xb3\x57\x12\xe0\x1d\xa0\x20\xc5\xcc\x0e\xb5\x8a\x2b\xda\x5e\x9d\xf2\xbf\xe2\x18\xf2\x7f\x3d\x3d\x68\xc3\xd7\x5c\x40\x02\x71\x83\xbd\xd4\xf8\xcb\xc5\xdb\x0f\x12\xe5\xd5\x76\x94\x45\x00\x86\x39\xe3\xb0\x61\x9d\xc2\xd4\x39\x5b\x2e\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xaf\xd4\x5c\x49\x2a\xda\x7e\x71\xfb\x7d\x91\xeb\x99\x2d\x5a\x40\xf2\x2f\xa3\xcf\xd5\xe9\xc4\x7b\xb4\x75\xcf\x4f\x9e\xc1\xf5\xcc\x4e\x31\xac\x5b\x32\x31\xac\x7b\xf4\xd8\x4d\x5b\x8b\xdd\xf4\x22\xeb\xa0\xc7\x07\x07\xb3\xd2\x79\x3d\x37\xf8\x0c\x17\xba\xc3\x7c\x17\x38\x6d\xc0\x2b\x22\x58\x76\xb8\x3c\x2f\x75\xad\x91\x34\xb3\x6d\x4f\xab\xe8\xec\xca\x9d\x03\x64\x7d\x57\x67\x6f\xf8\xdd\x35\xf0\xf0\xa0\xe7\xf3\x27\x2f\x25\x5b\xeb\x4d\xb4\x1e\x0d\x3f\x0f\x2f\xce\xaa\x4d\xcd\xb3\x3e\x7c\x50\x3b\x98\x89\xd5\xa0\xb7\xa4\x6b\x19\xb1\x3b\xa3\xfa\x29\xf1\x0e\x4c\xe1\xd8\x3c\x24\xfb\xa6\xa7\xa5\xb2\xb9\xe6\x6a\x6d\xed\xaf\x6c\xbd\x77\x24\xff\xca\xe5\xda\xdf\x0a\x65\x36\x77\xe6\xaa\xfc\xc5\x41\xaf\x17\xce\x01\xf2\x54\x32\xc0\xf0\x64\xfd\x7b\xfc\x78\x96\x3b\xfa\x0a\xfd\x71\x68\x3a\xcb\xfb\x69\x58\xea\xb5\xfe\x38\xb5\x8d\x0a\x1f\xec\xad\xe1\x47\x8e\xc3\x5e\xb0\x39\x0a\x1f\x61\xc4\x2b\xe4\xd3\x9e\x4a\xdb\xa3\xbc\xde\x89\x8e\xd3\x52\xef\x54\xbd\xc7\xd7\xdb\xd6\x91\x97\xa2\x68\x6b\xcd\x8f\x06\xea\xbf\x5b\x09\xd7\xa3\x67\x40\xba\xfd\x11\xc2\xc3\xf4\x3a\xa7\x5e\x6f\xf6\x49\x41\x25\x2e\xe5\xe4\x4a\x11\x29\x5b\xde\x17\x5f\x6c\x45\xc4\x7f\xbd\xe3\xbf\xfd\x8d\xa7\x72\xa1\xee\xa4\x2f\x8c\xdb\x92\x99\x57\x9d\xaf\x7c\x01\x6c\xc3\x04\xdc\xaa\xb8\xfc\x85\xcc\x7c\xfb\x2b\x2c\xff\xe7\x44\x2f\x21\x05\x43\xc5\xe0\xbf\x01\x00\x00\xff\xff\x99\xe3\x8f\xb5\x93\x2a\x00\x00") +var _deployKubernetes119DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x05\x2a\x2b\xee\x07\xb0\x10\xe0\x87\x5c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8d\x65\xd6\x14\xc9\x92\x23\xef\xfa\x8a\xfe\xef\x05\x25\x59\xa6\x24\x5b\x96\x15\xe7\xa3\x67\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x1f\x3f\xd2\xfa\x23\xf9\x0c\x12\x0c\x45\x88\xc9\x37\x8e\x0b\x72\x96\xd2\x25\x90\x65\x66\x51\xa5\xfc\x37\x38\xfb\x33\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\x59\x4d\x46\x4b\x2e\xe3\x88\x3c\x82\x59\x71\x06\x17\x8c\xa9\x4c\xe2\x28\x05\xa4\x31\x45\x1a\x8d\x08\x11\xf4\x19\x84\x75\xdf\x08\xd1\x29\xa4\x01\xb3\x7c\xcc\x25\x82\x18\x33\x95\x86\x31\x68\xa1\xd6\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x6a\x62\x90\x4f\x0c\x98\x4a\x03\xa6\x24\x1a\x25\x04\x98\x52\xca\x6a\xca\x3c\xd1\x51\x10\x04\x1f\xc6\xd8\x6f\xf0\xbc\x50\x6a\x69\x7b\x9a\x6a\x9e\x29\x1b\xd3\x0c\x17\xca\xf0\xdf\xa8\x5b\x7c\xbc\xfc\x64\xc7\x5c\x85\x95\x13\x0f\x4a\xc0\x9b\x98\x0e\xdf\x11\x8c\xa4\xc2\x89\xaf\xb8\xb3\x0f\x4c\xc0\xe6\xc9\x1e\x57\x4c\x26\xc0\x46\xa3\x80\x50\xcd\x3f\x1b\x95\xe9\xdc\xa4\x80\x9c\x9d\x8d\x08\x31\x60\x55\x66\x18\x94\x63\x20\x63\xad\xb8\x44\x07\xcb\x0a\xcc\x73\x39\x9c\x00\xe6\xff\x7f\xa3\xc8\x16\xf9\x37\xc1\x6d\x31\x14\x83\x00\x84\xfc\x6b\xa6\x63\x5a\x7e\x65\x06\xdc\xd7\x96\x4e\xa6\x94\x89\xb9\xf4\x01\x6c\x1b\x21\x80\x5a\x78\x2d\x0b\x2c\x2a\x43\x13\xd8\xab\x9c\x59\x5e\x8a\x30\xaa\x29\xe3\xc8\xf7\x99\x52\x19\xb0\xb5\xa9\x54\xda\x30\x45\x57\xcf\x4b\x53\x7b\x6d\x85\x56\xf1\x0e\xc5\xad\xa9\x54\x6b\xdb\x9e\x6c\x40\x0b\xce\xa8\x85\x5d\x3b\xf9\x91\x63\x7b\x93\x96\x27\x8a\xe7\xdd\x20\xee\x08\xa4\x01\x98\x5c\x8a\xcc\x22\x98\xf7\x4d\x7b\x93\x49\x09\xe6\x38\x4c\x9c\x8f\x16\x41\xe2\x4a\x89\x2c\x1d\x16\xde\x47\x45\x72\x43\x21\x13\x94\xa7\xfd\xb5\x96\x99\x74\x74\x26\x6f\xd2\x58\x50\xbb\xb7\x9c\x34\xd4\xf5\xab\x91\x2b\x68\x15\xc8\x23\x6b\x41\xdb\x17\x49\xb5\x5d\x28\x1c\x1f\x72\xaa\xdc\xb2\x52\xbc\xcb\xab\x53\xe9\x70\x5d\xbd\xed\xf0\x01\x55\x87\x8b\xac\x54\xf1\x29\x37\xe5\x88\xe5\x3e\x7e\xa6\x57\x45\xf0\x44\xd9\x7d\x14\x3c\x6f\x90\xd1\xfa\x60\x6e\xf7\xcd\xbb\x63\x94\xbc\x84\x15\xb4\x6a\xc9\xb1\x31\x3c\xac\xe5\xfe\xc4\x65\xcc\x65\xf2\x36\xec\xdd\xf2\x7a\x67\x51\x02\xba\xba\xb0\x12\xf0\x00\x73\x67\xc7\x06\xcb\x0e\xb7\x46\x84\x78\x44\x62\x08\xb7\xb5\xd9\xf3\xbf\x80\x61\x9e\x06\x3b\x0f\x0d\xa7\x3c\x98\x7c\xc0\xed\xd9\xd6\x84\x77\xd9\x97\x1a\x2f\x7b\xc9\x5e\x9c\xfe\xdc\xe5\x55\xe7\xf7\xce\x97\xa1\xe8\xfb\x0d\x66\x20\x03\xfc\x68\xf9\xf1\x4e\xbb\x52\x4b\x93\x57\xde\x8e\x66\x9b\x7e\xc3\xb4\x68\xde\x9c\x7c\x8c\xfb\x1d\xab\x81\x39\xb5\x5a\x19\xac\x0e\x60\x06\x23\x32\x39\x3f\x3f\x3f\xcf\xed\x41\x6a\x12\xc0\x59\x6d\xd4\x82\x00\x86\xca\x14\x16\x53\xad\xc7\xcb\xec\x19\x8c\x04\x84\x3c\xa6\xb8\xb4\x48\xa5\xaf\x69\xeb\xcd\x9e\x29\x0d\x1f\x6a\x96\x0f\x00\xe5\x43\xc0\x9f\x02\x1a\xce\xf6\x45\x48\x17\xf6\x93\x9d\xd8\x4f\xfe\x3f\xb0\x27\x04\xd7\x1a\x22\x72\xa7\x62\x70\xb6\xb7\x36\x83\x6a\x6d\xb7\x75\xe7\x11\x29\xc2\x3c\x13\x8f\xb0\xff\x1e\xb1\x6d\x33\x53\xa9\x56\x32\xb7\xa0\x61\xef\x1b\x40\xd2\x96\xd7\xd4\x60\xa0\xe6\xde\xf6\xbe\x5b\x2e\x6f\x6e\x92\x22\x32\x69\x85\x4b\xea\xb8\xed\x17\x0f\xd8\x41\x70\x1d\x0f\xd8\x20\x28\x6c\x91\xaa\x77\xbd\x10\x41\x48\xb5\xa0\x08\xa5\x9f\x5e\x1c\xe5\xf6\x4a\xa9\x30\xef\x1c\x95\xdf\x3b\x4d\xb2\xcc\x50\x17\xba\x6e\x69\xca\x25\x18\x5b\x4a\x8b\x1a\x66\xc7\x05\xe4\x60\x94\x87\xe0\xdc\x3f\x38\x07\xee\xca\xde\x89\x65\x43\x8c\x08\x4f\xa4\x32\x90\x8b\x6e\x62\xd2\x7d\xb6\x98\x6e\x46\x02\xc2\x54\x9a\x52\x19\x6f\x71\x0d\x48\x98\x59\x13\x0a\xc5\xa8\x08\x9f\xb9\x0c\x2b\x4f\x63\xc3\x57\x9e\x97\x01\x09\x56\xd3\xbf\xfa\x7f\x0a\x95\x24\x5c\x26\xc1\x5c\x99\x94\xe2\x14\xe1\x3b\xfa\x8f\x53\x15\xc3\xd4\x6b\xda\xd5\x83\x62\x61\x87\xeb\xf4\x87\x1f\x67\xb7\xd7\xb7\x4f\x97\x8f\x37\x4f\x57\x0f\x37\xbf\x5e\x3f\x3c\xdd\x5d\xdc\x5e\xff\xc9\x97\x76\x27\xc8\xc7\x32\xa7\xa6\xff\x39\x2b\x8f\x99\x67\xd1\x99\x33\xf4\xec\xbf\xbe\x28\xa3\x3f\x73\x01\xd3\x90\x81\x41\x1b\x32\x3a\x66\xa6\x66\x91\x1b\xf7\x25\x50\xd8\xa6\xc8\x12\xd6\x4d\x89\x25\xac\x7d\x09\xcb\x16\x10\x67\x02\xcc\x97\xfc\x78\x3f\x8d\x3e\x6d\x1a\xf7\xc6\xef\xa2\x09\x6d\x1e\x6f\x9b\x8b\xfb\x80\x5c\xf9\xd8\x17\xc1\xf5\xf5\xfa\xe1\xf6\xe6\xee\xe2\xeb\xcd\xfd\xdd\xd3\x97\xfb\xcf\x4f\xb3\x8b\xaf\xff\xa8\x84\x08\x59\x51\x91\x41\x44\xc2\x18\x56\x21\x82\x49\xcb\x1f\x07\x1c\xfe\xad\xa5\x76\xc1\xd9\x5e\xaa\x23\x01\xaa\x85\xee\xaf\xf2\xc9\x8f\xb3\x8b\xcb\xd6\x0a\x3f\x1b\x95\x46\xde\x20\x21\x73\x0e\x22\x2e\x79\x64\x6b\x7c\x46\x71\x11\x55\x25\x62\x5c\xd5\xd2\x4a\x96\xa7\x34\x81\x88\xe4\xd6\x34\x03\x30\x62\x54\x52\xb3\xae\xcb\xce\x32\x21\x66\x4a\x70\xb6\x8e\xc8\xcd\xfc\x4e\xe1\xcc\x80\x05\xb9\xdd\x4a\x2f\x6b\x1b\x71\x5c\x75\xff\x8d\xbf\x55\x9a\xcc\x1a\x64\xc0\x5f\x69\xcb\x2c\x8a\x8f\x05\x96\x19\x8e\xeb\x4b\x25\x5d\xdc\xfb\x6e\x1b\xa0\xf1\xbd\x14\xeb\x07\xa5\xf2\x68\xb3\x6b\x8b\x90\x46\x04\x4d\xb6\xf5\xd8\xdb\xc7\x5b\xb0\xd6\xb9\x94\x83\xd4\xb9\xc9\xc5\x4d\xd2\xad\x23\xca\x35\x0f\x52\x37\x52\xce\xcf\xe3\xb6\x65\x7e\x99\x86\x79\x06\x8c\x0e\x78\x60\x32\x79\x61\xef\x94\x74\xf6\x37\xac\xce\x1f\xfd\x62\xc1\x14\xcc\xb4\x5a\xc9\xa7\xf0\x7b\xbb\x47\xa3\x14\xa0\x12\x60\xea\x1d\x22\x20\x30\x9f\x03\x43\xc7\x64\x1e\xcb\x34\xab\x74\x2f\x61\x1d\xe5\x97\x74\xf9\xb9\xa5\x51\x70\x53\xea\x4e\x23\x23\x1f\x26\x6f\xd5\xbd\x18\xe4\x28\x18\xa8\x6d\x5f\x31\xd2\xa3\x07\x06\x85\xe4\x01\xb2\x75\x45\x21\x55\x72\x30\xd5\x72\x0e\xbf\x1a\xc9\xea\x58\xfc\xf5\xe9\x55\xa9\xbc\x8b\x58\xbd\x13\x95\xaa\x60\x19\x46\xc5\x3f\x10\x2f\xf2\x3c\x79\x3d\x46\x74\x40\xc9\xef\x94\x0b\xc5\xe0\x6a\xde\x2d\x95\x34\x01\x33\x2d\x6c\x7d\x21\x57\xaa\x01\x19\x90\x60\xf3\xa6\xc1\x34\x93\xfc\x7b\x14\x86\x21\xb3\xdc\xfd\x1b\x5b\xc5\x96\x4d\x8a\xc4\xe3\xe9\x0f\x3f\xfe\xf3\x97\x9f\xae\x9f\xee\xee\xaf\xae\xdb\x34\xca\xba\x43\x9f\xeb\x11\xd3\x70\x45\x4d\x28\xf8\x73\xd8\x83\x7c\x1d\x47\xd5\x1c\x5c\x33\x30\x0c\x24\xd2\x04\xa6\x93\x97\xf2\xa2\xba\x3b\xc3\x49\x48\xe3\x5e\xa2\xfe\xd0\x63\x28\x2e\x60\xc6\x0e\x4c\x57\xfd\x5f\x8f\x5a\xf5\xa4\x7b\x98\xea\xbd\x4c\xe0\x77\x45\x98\xb4\xe1\x2b\x2e\x20\x81\xb8\x41\x37\x6a\x84\xe3\xfc\x30\x77\xea\x42\xac\x0f\x77\xda\x24\x86\x2b\x60\x02\x30\xd4\x22\x4b\xb8\xb4\x61\xa3\xbe\x7a\x25\x8c\x94\xf3\x8d\xd2\x34\xc9\x95\x46\xe4\x27\x5e\x54\x03\xae\x24\x15\x6d\x5c\x9c\x7c\x9e\xd6\x41\xcc\x4d\x5f\x43\x8a\x77\x34\x86\xea\x74\xd3\x3b\xb4\xc5\xb0\x6a\x4d\x89\x61\xd5\x31\xc3\xae\xdb\x14\xd3\xae\xbb\x74\xd4\x41\x2b\x67\x28\xb6\x84\x3e\x30\x74\xe4\xd4\x00\x30\xdc\x62\x79\x35\xf4\x34\x07\x84\x9a\xa4\x16\x17\xcd\x02\x1e\x94\x7b\x11\x18\x48\xb8\xc5\x82\xb9\x06\xba\x56\x50\x9b\x71\xb3\xa7\x64\xee\xac\xe1\x79\x02\xd3\x38\x36\x60\xed\x74\x77\xa5\xdf\x59\x23\x5f\x5c\x95\xca\x52\xb2\xfc\x64\xc7\x09\x33\x2e\xc0\x2d\x4f\x82\xf2\x98\x1d\x6e\x1a\x7c\x59\x14\x2a\xef\x4d\xb4\x9a\x8c\xff\x32\x3e\x1f\x54\x65\x9a\x6b\x9d\xec\x54\xd5\xeb\x80\x34\x24\x12\xfd\x4d\x6f\xcd\xae\x45\xc4\x51\x31\x75\x78\xcf\x9d\xd4\x1c\x28\x66\x06\x82\x84\x22\xd8\xe9\x57\xa5\x95\x50\xc9\x7a\x5a\xf3\xdc\xc9\x15\xfb\x54\x31\xa8\xb6\x80\x75\x25\x19\x03\xdc\xbb\x02\x4f\x53\x88\xb9\x4b\x8d\x4a\x66\x4e\x85\xad\x0b\x21\x4f\x41\x65\x38\xfd\x7b\x5a\x1b\x8e\x61\x4e\x33\x81\xc1\xdc\xe2\x5a\xc3\x14\xbe\xe3\xdf\x6a\xcf\xcb\x86\x50\xf9\x9b\x13\x80\x49\x77\x70\xbf\x7b\xef\x2f\x93\xa3\x4c\x8c\xe5\x27\xeb\x2a\x47\x4e\xe4\x9a\x49\xe2\xfd\xde\xf8\x92\x06\xbc\xeb\xf7\xcb\xa3\x3a\xf1\xe4\xd5\xaf\x2e\x4e\x96\x64\xfe\xcd\xde\x76\xa1\x12\xd4\xa2\x66\x1d\x77\xc3\xd0\xba\x9b\x6d\xdd\x04\x2c\x94\x2d\x4c\xf4\xe9\x47\x77\xf3\xef\x6c\x3e\xc5\xef\x2e\x57\x79\xbb\x51\x66\x7d\x6f\x2e\x37\xaf\xb4\x75\x7a\x3e\xc4\x90\xa7\xb2\xcc\xac\xc3\xa3\xf5\xef\x2d\x50\x03\xe0\xe8\xe2\x42\xfd\xac\xd9\xc9\x80\x8e\xb3\xa5\x4e\x87\xfa\xa9\x6d\x90\xa0\x5e\x0a\x4f\xb0\xf7\x7b\xe8\x46\x87\xf2\x3a\x1d\xeb\xa7\xa5\x4e\xd7\x3a\x97\xaf\x73\xb7\x9e\x01\x5c\x72\xbb\xe6\x55\x57\xfd\x9d\xb0\x70\x35\x79\x06\xa4\xd5\xeb\x0d\x8f\x37\x57\xc5\x49\xe2\xd5\x7e\xfa\xf5\xf6\x65\x73\xd8\xa7\x88\x94\x2d\x1e\xe0\xdf\x19\x37\xee\x40\xb1\xe9\x5f\x5a\xc5\x37\x72\xae\xee\x65\x5e\xb8\xaa\x92\x56\x54\x88\x2f\x7c\x0e\x6c\xcd\x04\xdc\xaa\x78\xf3\x3e\xdb\xac\x7a\x81\x2f\xff\xf3\x5a\x2f\x20\x05\x43\xc5\xe8\x7f\x01\x00\x00\xff\xff\x5b\xc9\x12\x9a\x7d\x32\x00\x00") func deployKubernetes119DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -220,47 +221,67 @@ func deployKubernetes119DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 10899, mode: os.FileMode(436), modTime: time.Unix(1610053011, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 12925, mode: os.FileMode(420), modTime: time.Unix(1611067502, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119FakePmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\x78\xdb\x03\x5a\x01\x7e\xc8\x26\xbe\xd4\xe8\x26\x31\x92\xec\xbd\x06\x0c\x35\x96\x59\x53\x24\x4b\x8e\xbc\xd1\x15\xfd\xdf\x0b\x52\x92\xad\x2f\x2b\xb6\x2f\x1f\x28\xea\x7d\x58\x87\xe2\x70\x7e\xf3\xc1\x99\xdf\xd8\xfe\x23\xb9\x02\x09\x86\x22\xc4\xe4\x3b\xc7\x15\xf9\x94\xd2\x35\x90\x75\x66\x51\xa5\xfc\x37\xf8\xf4\x27\x12\x2b\x22\x15\x12\x88\x39\xfe\x61\x34\xa2\x9a\xff\x0a\xc6\x72\x25\x23\xb2\x99\x8c\xd6\x5c\xc6\x11\xb9\x07\xb3\xe1\x0c\xce\x19\x53\x99\xc4\x51\x0a\x48\x63\x8a\x34\x1a\x11\x22\xe8\x13\x08\xeb\xde\x11\xa2\x53\x48\x03\x66\xf9\x98\x4b\x04\x31\x66\x2a\x0d\x63\xd0\x42\xe5\x29\x48\x8c\xc8\x92\xae\x21\xd0\x46\xc5\x19\x43\xae\xe4\x88\x10\x49\x53\x88\xb6\x62\x01\x53\x12\x8d\x12\x02\x4c\xf9\xcc\x6a\xca\x6a\x1b\x46\x41\x10\x34\x00\x9a\x27\xca\xc6\x34\xc3\x95\x32\xfc\x37\xea\x0e\x1d\xaf\xff\x6a\xc7\x5c\x85\x5b\xe8\x77\x4a\xc0\x9b\x01\x86\x67\x04\x23\xa9\x70\x9b\x36\xdc\x61\x02\x13\xb0\x65\xb2\x07\xbe\xc9\x04\xd8\x68\x14\x10\xaa\xf9\x95\x51\x99\xf6\x30\x02\xf2\xe9\xd3\x88\x10\x03\x56\x65\x86\x41\xb9\x06\x32\xd6\x8a\x4b\xb4\x23\x42\x36\x60\x9e\xca\xe5\x04\xd0\xff\xff\x9d\x22\x5b\xf9\x77\x82\xdb\x62\x29\x06\x01\x08\xfe\x6d\xa6\x63\x5a\xbe\x65\x06\xdc\xdb\x8e\x4e\xa6\x94\x89\xb9\xac\x3b\xad\x0b\x42\x00\xb5\xf0\x56\x08\x2c\x2a\x43\x13\xd8\xab\x9c\x59\x5e\x6e\x61\x54\x53\xc6\x91\xef\x83\xb2\x05\xb0\xc3\x54\x2a\x6d\x41\xd1\xdb\xe7\x25\xd4\x83\x42\xa1\x55\xdc\xa3\xb8\x23\x4a\xb5\xb6\x5d\x61\x03\x5a\x70\x46\x2d\xf4\x45\xf2\x84\x7c\xbe\x10\x99\x45\x30\xef\x9f\xd6\x26\x93\x12\xcc\x51\x39\xac\x9d\x5d\x16\x41\xe2\x46\x89\x2c\x3d\x2d\x7c\x47\x45\xaa\xa5\x90\x09\xca\xd3\xc3\xb5\x96\x99\x72\x74\xa6\x56\x69\x2a\xa8\xdd\x7b\x5d\x5a\xea\x0e\xab\x01\x1b\xe8\x14\x80\x23\x73\xbd\x6b\x8b\xa4\xda\xae\x14\x8e\x5f\x32\xaa\x0c\x59\xb9\x7d\xc8\xaa\xd7\xd2\xe1\xca\x7f\xd7\xe0\x17\x54\xbd\x5c\x44\xa4\x8a\x5f\x33\x28\x47\x1c\x77\x5a\xb7\xfa\xc2\x65\xcc\x65\xf2\x76\x5d\xd6\xf2\xe6\xc5\x56\x02\x86\x9a\x96\x12\x70\x07\x4b\xa7\xbb\xf2\xce\x80\x29\x23\x42\x6a\x7d\xf7\xf0\x86\x69\xb3\xa7\x7f\x02\x43\x5f\x59\x7a\x39\xc7\xbb\x90\x85\x5a\x71\xfd\x88\x28\x9c\xea\xeb\x7a\x4f\x38\xaa\x98\xbf\x83\xd7\xdb\x1c\xf2\xe3\xc8\xa3\xd5\xc0\x9c\x2a\xad\x0c\x6e\x3b\xbb\xc1\x88\x4c\xce\xce\xce\xce\x3c\x06\xa4\x26\x01\x5c\x34\x56\x2d\x08\x60\xa8\x4c\x81\x92\x6a\xbd\x4f\xe7\xd1\x26\x7c\x98\xab\x52\x40\xc3\x99\x3d\xc1\x4f\x93\x5e\x3f\x4d\xde\xd6\x4f\x84\x60\xae\x21\x22\x37\x2a\x06\xa7\xb3\xe3\x38\xc7\xbc\x76\x97\xf8\x1e\x29\xc2\x32\x13\xf7\xf0\x81\x93\x4a\xe5\xc4\x8a\xfe\x45\x64\xd2\xf1\x51\xea\xba\xc4\xd7\x1a\xaa\x17\xdc\x76\x02\x6c\x5b\x64\xd2\xcd\x00\x7a\x84\x54\x0b\x8a\x50\x62\xaa\x39\xcc\x03\x92\x52\xa1\x2f\x3a\x5b\x8c\xbd\x30\x2c\x33\xd4\xc5\xc8\x1d\x4d\xb9\x04\x63\xcb\xdd\xa2\x61\xdf\x8b\x16\x9e\x60\xe3\x5e\xb1\xef\xf0\xb4\x52\x6a\x1d\x11\x9e\x48\x65\xc0\x6f\xad\xe2\xe2\xa1\x2c\x97\x5c\x72\xcc\x77\xe0\x5c\x6f\x3f\xef\xac\xba\x28\xfe\x2b\xe3\x06\xe2\xcb\xcc\x70\x99\xdc\xb3\x15\xc4\x99\xe0\x32\x99\xfb\x83\xcb\xe5\xd9\x33\xb0\xcc\x61\xaa\x4b\x16\x67\xde\x97\x71\x7f\x00\x93\xda\xe6\xe3\xa0\x48\x83\xd9\xb3\x36\x60\x6d\xd3\xcf\xd5\x8e\x35\xe4\x51\x9f\x81\x3d\xde\xab\x5e\x4a\xbb\xa1\x5f\x19\x77\x6b\x70\x2e\x3b\xcf\x37\x54\x64\xd0\x51\xe5\x39\x8f\x54\x9f\xfa\x96\x97\x54\x58\xa8\x9e\xec\xc2\x5c\x1d\xe1\x26\xca\x34\xa5\x32\xde\x9d\x19\x90\x30\xb3\x26\x14\x8a\x51\x11\x3e\x71\x19\x6e\xc3\x1e\x1b\xbe\xa9\x81\x0e\x48\xb0\x99\xfe\xb9\xfe\xa7\x50\x49\xc2\x65\x12\x2c\x95\x49\x29\x4e\x11\x9e\xb1\xfe\x38\x55\x31\x4c\x7b\xac\x0f\x48\x50\xcd\xcd\xd3\x4c\xf2\xe7\x28\x0c\x43\x66\x79\xd8\x4c\xb5\xb1\x55\x6c\x5d\x97\x31\x90\x70\x8b\x26\x9f\x55\xb2\xc8\x74\x14\x86\x67\x63\xff\x2f\xda\xf5\x87\x72\xbf\x0b\x29\x8f\xa7\x3f\xfc\xf8\x8f\x6f\x5f\x66\x8f\x37\xb7\x97\xb3\xc7\x9b\xf3\xeb\xd9\x4f\xf5\x3d\x8c\xfe\xc2\x05\x4c\x43\x06\x06\x6d\xc8\xe8\x98\x99\x86\x09\x6e\xbd\xbe\x03\x85\x6d\x6f\x59\x43\xde\xde\xb1\x86\xbc\xbe\xa3\xf0\xa3\xab\x3f\xd3\x1f\x7e\x5c\x5c\xcf\xae\x1f\x2f\xee\xe7\x8f\x97\x77\xf3\x5f\x67\x77\x5d\x48\x65\xd9\xff\xea\x27\xa4\x69\xb4\x2b\xe7\xee\x05\x72\x53\x8f\x5c\x51\xef\x9a\xf6\x8d\x5a\xe9\xf3\x8b\x51\x69\x33\x83\x96\x1c\x44\x5c\x12\x98\xfa\xab\xd5\xe5\x9a\x0f\xbd\xd0\x82\xe2\x2a\xf2\xb7\x73\xec\xbc\xeb\x0a\x56\x07\xcd\xc3\xec\xee\x7a\x7e\x73\xfe\x30\xbf\xbd\x79\xfc\x7a\x7b\xf5\xb8\x38\x7f\xf8\x7b\x1b\x53\x44\x42\x4c\x75\x88\x60\xd2\xf2\xc3\x0d\x97\x4b\x9d\xa3\xfa\x7c\xd5\x3d\xaa\x7b\xe5\x3a\x07\x5d\xdd\x5e\xce\xbe\x7c\xbb\xea\xca\x3e\xff\x7c\xf6\xb7\xa2\xee\x5c\xdc\x4c\x77\x6e\xe6\x29\x4d\x20\x22\xfe\xc0\xf6\x7d\x88\x18\x95\xd4\xe4\xcd\xbd\x8b\x4c\x88\x85\x12\x9c\xe5\x11\x99\x2f\x6f\x14\x2e\x0c\x58\x90\xbb\x44\xa9\x75\xa6\xd6\xb5\xda\x36\xf0\x0a\xf2\xf6\xd6\x2e\x5a\xfd\xbc\x7e\xd2\x8e\x1c\x14\x2f\x0b\x2c\x33\x1c\xf3\x0b\x37\x9d\x3d\x63\xb3\x2c\xd2\xf8\x56\x8a\xfc\x4e\x29\x9f\xcb\x36\xb7\x08\x69\x44\xd0\x64\xbb\xe8\xd5\x42\x71\x0d\xd6\x3a\x93\x7c\xac\x07\xe3\x54\x0c\x86\xd7\x8e\x83\x36\x2c\x48\xdd\x4a\x29\xef\x6f\x45\x07\x7e\x75\x97\xfd\x05\xdb\x27\x69\x79\x47\x4e\x8b\x2c\xe1\x32\x70\xa5\x01\x30\x88\xb9\xd9\x23\x8b\xa9\xee\xc8\x62\xaa\x6b\x12\x01\xa1\x26\x69\xa0\x6e\x57\x37\x1f\x73\x1a\xc7\xae\xe2\x4f\x0f\x29\x50\xc1\x12\x28\x66\x06\x82\x84\x22\xd8\xe9\x83\xd2\x4a\xa8\x24\x9f\x36\x1c\xed\xf6\x59\x17\x3b\x0c\x70\xef\x06\xe4\x29\xa8\x0c\xa7\x3f\xa7\x8d\xe5\x18\x96\x34\x13\x18\x2c\xad\xa3\x5a\x53\x78\xc6\xbf\x34\x9e\x97\x49\xb1\x05\xed\x8b\xc7\xa4\x9d\xd5\x6e\x2c\x49\x98\x71\x23\x95\xe5\x49\x50\xce\xe6\x61\x6b\xd4\x89\x36\x9f\xc7\x67\xe3\xcf\x27\xa5\x79\xdf\x30\x73\x54\xbe\x4f\xde\x3c\xdf\x0f\x4a\xdd\x23\x12\x70\x2f\x1e\x93\xc9\x73\x7b\xa3\xa4\x43\xd3\xc2\xe0\x1f\x7d\xb3\x60\x8a\x31\x66\x7b\x52\x7d\xb6\x1b\xe2\x84\xee\x85\x4a\x38\x0a\x51\x67\x24\x01\x81\xe5\x12\x18\x3a\x52\x51\x92\xa0\x9d\x4e\x4f\x52\x5c\xed\xf6\xa3\xec\x78\x9d\x3d\x81\x91\x80\xe0\x27\xec\x94\xba\x01\x75\x54\xf7\x50\xfd\xd4\x54\x63\x7e\xc9\x4d\x44\x64\x26\x44\xbb\xb2\xed\xf1\x4b\x30\x78\xe1\x2d\x30\x03\x8d\xe0\x15\x2b\x2d\xab\xb7\xc2\xc5\x53\xdb\x03\xe9\xdf\xff\x69\x01\xaa\x6e\xfb\xf0\x14\x72\x49\x21\x55\xf2\x2d\x67\x10\xe7\xeb\x17\xa6\x8f\x23\xe7\x8d\xf2\xc4\x93\xc0\x7d\xe8\x18\x51\x03\xfe\x9e\x03\xc4\xab\xb3\xe0\x18\xdc\x05\xbd\xa6\x92\x26\x60\xa6\x31\x37\xc0\xf0\x77\xb2\xe4\x86\x6b\x06\xf8\x71\xa7\xe7\x1c\x44\x72\xb7\x45\xa3\x45\x9d\x4b\xa9\xc5\xed\xe5\xe3\x7c\xf1\x93\x67\xd0\x93\x03\x18\x77\x4f\x35\xea\xb2\xef\x77\x61\xd6\x16\x29\x7a\xa2\x32\x0d\x37\xd4\x84\x82\x3f\x85\x07\x30\xec\xe3\xf8\xb8\xb3\x76\x01\x86\x81\x44\x9a\xc0\x74\xd2\x34\x73\x90\xad\x77\x72\xc5\x65\xf5\xff\x12\x97\xaf\xa5\xc7\x3b\x40\x41\x8a\x99\x1d\x6b\x15\xd7\xb4\xbd\xfa\x28\xf0\x8a\xe3\xc9\xff\xf5\x54\xa1\x0d\xdf\x70\x01\x09\xc4\x2d\x56\xd3\xe0\x35\x67\x6f\x3f\x60\x54\x17\xdf\x51\x19\x01\x18\x16\x4c\xc4\x86\x4d\x6a\xd3\xe4\x72\x85\xbc\x51\x9a\x26\x5e\x69\x44\xbe\xf0\xa2\x8e\x73\x25\xa9\xe8\xfa\xc5\xed\xf7\x25\x70\x60\xe6\xe8\x00\x29\xbe\x90\x3e\x55\xa7\x13\x1f\xd0\xd6\x3f\x57\x79\x66\x37\x30\x53\xc5\xb0\xe9\xc8\xc4\xb0\x19\xd0\x63\xf3\xae\x16\x9b\x0f\x22\xeb\xa1\xcd\x2f\x0e\x6c\x95\xf3\x06\x6e\xf0\x09\x2e\x74\x87\xf9\x1e\x71\xdc\xe0\x57\x46\xb0\xea\x7f\x45\x5e\xea\x46\x9b\x69\x67\xdb\x9e\x46\xd2\xdb\xb3\x7b\x07\xcb\xe6\xae\xde\xde\xf0\xbb\x6b\xe0\xcb\x03\xa0\xcf\x9f\xa2\x94\x6c\xad\x37\xd1\x66\x32\xfe\x3c\x3e\x3b\xa9\x36\xb5\xcf\xfa\xf0\x01\xee\xc5\x4c\xac\x07\xbd\x23\xdd\xc8\x88\xdd\x19\xf5\x4f\x8f\x77\x60\x4a\xc7\x16\x21\xd9\x37\x55\xad\x94\x2d\x34\xd7\x6b\xeb\x70\x65\x1b\xbc\x23\xc5\x97\x30\x97\xfe\x56\x28\x93\xdf\x9a\x8b\xea\x77\x07\x83\x5e\x38\x05\xc8\x63\xc5\x0f\xc3\xa3\xf5\xef\xf1\xe3\x49\xee\x18\x2a\xf4\x87\xa1\xe9\x2d\xef\xc7\x61\x69\xd6\xfa\xc3\xd4\xb6\x2a\x7c\xb0\xb7\x86\x1f\x38\x26\x7b\xc1\xf6\x88\x7c\x80\x11\xaf\x90\x4f\x7b\x2a\xed\x80\xf2\x66\x27\x3a\x4c\x4b\xb3\x53\x0d\x1e\xdf\x6c\x5b\x07\x5e\x8a\xb2\xad\xb5\x3f\x32\x68\xfe\x7a\x25\xdc\x4c\x9e\x00\xe9\xf6\xc7\x08\xf7\xf3\xcb\x82\x7a\xbd\xd1\x27\x08\xb5\xa8\x54\x53\x2d\x45\xa4\x6c\x75\x57\x7e\xdd\xe5\x8e\x10\x16\xfc\x77\xc1\xf1\x5c\x2e\xd5\xad\xf4\x65\x71\x5b\x30\x8b\x9a\xf3\x95\x2f\x81\xe5\x4c\xc0\xb5\x8a\xab\x5f\xc9\x2c\xb6\xbf\xc4\xf2\x7f\xce\xf4\x0a\x52\x30\x54\x8c\xfe\x1b\x00\x00\xff\xff\xc7\x92\xe8\x5c\x91\x2a\x00\x00") +var _deployKubernetes119LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xd3\x7b\xb8\x02\x95\x15\xf7\x0f\xb0\x10\xe0\x87\x6c\x92\xdb\x06\xdd\x24\x46\xb2\x77\xaf\x01\x43\x8e\x65\xd6\x14\xc9\x92\x94\x76\x7d\x45\xbf\x7b\x41\x49\x96\x29\xc9\x56\x24\xc5\xde\xa4\x6b\x60\xb1\x0e\x35\xe4\xcc\xfc\x38\x7f\x7e\xa4\xf5\x47\xf4\x09\x04\x68\x6c\x81\xa2\xaf\xcc\xae\xd0\x59\x82\xd7\x80\xd6\xa9\xb1\x32\x61\xbf\xc3\xd9\x9f\x11\x95\x48\x48\x8b\x80\x32\xfb\x87\xc9\x04\x2b\xf6\x1b\x68\xc3\xa4\x88\x50\x36\x9b\xac\x99\xa0\x11\x7a\x04\x9d\x31\x02\x17\x84\xc8\x54\xd8\x49\x02\x16\x53\x6c\x71\x34\x41\x88\xe3\x67\xe0\xc6\x7d\x43\x48\x25\x90\x04\xc4\xb0\x29\x13\x16\xf8\x94\xc8\x24\xa4\xa0\xb8\xdc\x24\x20\x6c\x84\x78\x96\x04\x4a\x4b\x9a\x12\xcb\xa4\x98\x20\x24\x70\x02\x51\x35\x2b\xc8\x67\x05\x44\x26\x01\x91\xc2\x6a\xc9\x39\xe8\x52\xca\x28\x4c\x3c\xd1\x49\x10\x04\xef\xc3\xd2\xaf\xf0\xbc\x92\x72\x6d\x7a\xda\xa9\x9f\x31\x99\xe2\xd4\xae\xa4\x66\xbf\x63\xb7\xf8\x74\xfd\xc1\x4c\x99\x0c\x2b\x0f\x1e\x24\x87\xd3\xdb\x0d\xdf\x2c\x68\x81\xb9\x13\xcf\x98\x33\x0e\x74\x40\x96\xf1\x01\x3f\x74\xca\xc1\x44\x93\x00\x61\xc5\x3e\x69\x99\xaa\xdc\x9e\x00\x9d\x9d\x4d\x10\xd2\x60\x64\xaa\x09\x94\x63\x20\xa8\x92\x4c\x58\x87\x49\x06\xfa\xb9\x1c\x8e\xc1\xe6\xff\x7f\xc5\x96\xac\xf2\x6f\x9c\x99\x62\x88\x02\x07\x0b\xf9\xd7\x54\x51\x5c\x7e\x25\x1a\xdc\xd7\x96\x4e\x22\xa5\xa6\x4c\xf8\xe8\xb5\x8d\xe0\x80\x0d\x9c\xca\x02\x63\xa5\xc6\x31\x1c\x54\x4e\x0c\x2b\x45\x08\x56\x98\x30\xcb\x0e\x99\x52\x19\xb0\xb3\xa9\x54\xda\x30\x45\x55\xcf\x4b\x53\x7b\x6d\x85\x92\x74\x8f\xe2\xd6\x54\xac\x94\x69\x4f\xd6\xa0\x38\x23\xd8\xc0\xbe\x9d\x7c\xb7\x81\xbd\x4d\xc8\x23\x05\xf3\x7e\x04\xf7\x44\xd1\x08\x40\x2e\x79\x6a\x2c\xe8\x37\x4c\x78\x9d\x0a\x01\x7a\x18\x20\xce\x41\x63\x41\xd8\x4c\xf2\x34\x19\x17\xd8\x83\x62\xb8\xa1\x90\x70\xcc\x92\xfe\x5a\xcb\x1c\x1a\x9c\xc3\xdb\x04\xe6\xd8\x1c\x2c\x24\x0d\x75\xfd\xaa\x63\x06\xad\xd2\x38\xb0\x0a\xb4\x7d\x11\x58\x99\x95\xb4\xd3\x97\x9c\x2a\xb7\xac\x14\xef\xf2\xea\x58\x3a\x5c\x27\x6f\x3b\xfc\x82\xaa\x97\xcb\xab\x90\xf4\x98\x9b\x32\x60\xb9\x77\x9e\xe6\x55\xf9\x3b\x52\x6a\x0f\xc2\xe6\x3b\xa4\xb3\x7a\x31\xb1\xfb\x26\xdd\x10\x25\xaf\x21\x03\xad\x42\x32\x34\x80\xc7\x75\xda\x8f\x4c\x50\x26\xe2\xef\xc0\xd5\x0d\xab\xf7\x14\xc9\xa1\xab\xf9\x4a\x0e\x0f\xb0\x74\x46\x6c\x81\xec\xf0\x69\x82\x90\x47\x1e\xc6\xf0\x59\x93\x3e\xff\x0b\x88\xcd\x73\x60\xef\x11\xe1\x98\xc7\x90\xf7\xb6\x37\xbb\x6a\xf0\x26\x9b\x52\xe3\x62\xaf\xd9\x88\xe3\x9f\xb2\xbc\xa2\xfc\xa6\x99\x32\x16\x7a\xbf\xa9\x8c\x64\x7d\xef\x2d\x33\xde\x62\x4b\x6a\x09\x72\xe2\xbd\x68\xb6\xe6\xef\x98\x10\xcd\xeb\x91\x77\x70\x83\x63\x14\x10\xa7\x53\x49\x6d\xab\xb3\x96\xb6\x11\x9a\x9d\x9f\x9f\x9f\xe7\xc6\x58\xac\x63\xb0\x8b\xda\xa8\x01\x0e\xc4\x4a\x5d\x98\x8b\x95\x9a\xae\xd3\x67\xd0\x02\x2c\xe4\xd1\xc4\x84\xb1\x58\xf8\x9a\x76\xae\x1c\x98\xd2\xf0\xa1\x66\xf9\x50\x44\xde\x1e\xf8\x04\xac\x66\xe4\x50\x60\x74\xa1\x3e\xdb\x8b\xfa\xec\xff\x00\x75\x84\xec\x46\x41\x84\xee\x24\x05\x67\x78\x6b\x1b\xb0\x52\x66\x57\x68\x1e\x2d\xb6\xb0\x4c\xf9\x23\x1c\xbe\x20\x6c\x1b\x4c\x64\xa2\xa4\xc8\xd5\x37\x8c\xfd\x0e\x78\xb4\xe5\x15\xd6\x36\x90\x4b\x6f\x6f\xdf\x26\x7f\xb7\xb7\x44\x11\x9a\xb5\x02\x25\x71\x04\xf6\xb3\x87\xea\x28\xac\x86\xa3\x35\x1c\x07\x53\xa4\xe7\x5d\x2f\x38\x2c\x24\x8a\x63\x0b\xa5\x93\x5e\x04\xe5\xc6\x0a\x21\x6d\xde\x27\x2a\xa7\xf7\xda\x63\x88\xc6\x2e\x68\xdd\xd2\x98\x09\xd0\xa6\x94\xe6\x35\xc0\x86\x85\xe2\x68\x88\xc7\x80\xdc\x3f\x2c\xc7\x6c\xc9\xc1\x59\x65\xef\x8b\x10\x8b\x85\xd4\x90\x8b\x6e\xa3\xd1\x7d\x76\x80\x6e\x47\x02\x44\x64\x92\x60\x41\x77\xa0\x06\x28\x4c\x8d\x0e\xb9\x24\x98\x87\xcf\x4c\x84\x95\x9b\x54\xb3\xcc\x73\x31\x40\x41\x36\xff\xab\xff\x27\x97\x71\xcc\x44\x1c\x2c\xa5\x4e\xb0\x9d\x5b\xf8\x66\xfd\xc7\x89\xa4\x30\xf7\xfa\x73\xf5\xa0\x58\xd8\x81\x3a\xff\xe9\xe7\xc5\xed\xf5\xed\xd3\xe5\xe3\xcd\xd3\xd5\xc3\xcd\x6f\xd7\x0f\x4f\x77\x17\xb7\xd7\x7f\xf2\xa5\xdd\x01\xf1\xb1\xcc\xa6\xf9\x7f\xce\xca\x53\xe4\x59\x74\xe6\x0c\x3d\xfb\xaf\x2f\x4a\xf0\x2f\x8c\xc3\x3c\x24\xa0\xad\x09\x09\x9e\x12\x5d\xb3\xc8\x8d\xfb\x12\x96\x9b\xa6\xc8\x1a\x36\x4d\x89\x35\x6c\x7c\x09\x43\x56\x40\x53\x0e\xfa\x73\x7e\x7a\x9f\x47\x1f\xb6\x6d\x7a\xeb\x77\xd1\x78\xb6\x8f\x77\x0d\xc5\x7d\x40\x64\x3e\xf6\x45\x64\x7d\xb9\x7e\xb8\xbd\xb9\xbb\xf8\x72\x73\x7f\xf7\xf4\xf9\xfe\xd3\xd3\xe2\xe2\xcb\x3f\x2a\x21\x84\x32\xcc\x53\x88\x50\x48\x21\x0b\x2d\xe8\xa4\xbc\xf2\x77\xf8\xb7\x96\xda\x07\x67\x7b\xa9\x8e\xe8\xaf\x16\xba\xbf\xca\x27\x3f\x2e\x2e\x2e\x5b\x2b\xfc\xa2\x65\x12\x79\x83\x08\x2d\x19\x70\x5a\x52\xc6\xd6\xf8\x02\xdb\x55\x54\xd5\x87\x69\x55\x45\x2b\x59\x96\xe0\x18\x22\x94\x5b\xd3\x0c\xc0\x88\x60\x81\xf5\xa6\x2e\xbb\x48\x39\x5f\x48\xce\xc8\x26\x42\x37\xcb\x3b\x69\x17\x1a\x0c\x88\xdd\x56\x7a\x29\xdb\x88\xe3\xaa\xe3\x6f\xfd\xad\xd2\x64\xd1\x20\x00\xfe\x4a\x3b\x36\x51\x7c\x0c\x90\x54\x33\xbb\xb9\x94\xc2\xc5\xbd\xef\xb6\x06\x4c\xef\x05\xdf\x3c\x48\x99\x47\x9b\xd9\x18\x0b\x49\x84\xac\x4e\x77\x1e\x7b\xfb\x78\x0b\xc6\x38\x97\x72\x90\x3a\x37\xb9\xb8\x28\xba\x75\x9c\xb8\xe6\x41\xe2\x46\xca\xf9\x79\xdc\xb6\xcc\x2f\xd3\x30\xcf\x80\xc9\x0b\x1e\xe8\x54\x5c\x98\x3b\x29\x9c\xfd\x0d\xab\xf3\x47\xbf\x1a\xd0\x05\x0f\xad\x56\xf2\xd9\xfa\xc1\xd6\xd1\x28\x05\x56\x72\xd0\xf5\xf6\x10\x20\x58\x2e\x81\x58\x47\x60\x1e\xcb\x34\xab\x74\xaf\x61\x13\xe5\x77\x70\xf9\x11\xa5\x51\x6d\x13\xec\x0e\x1e\x13\x1f\x26\x6f\xd5\x83\x18\xe4\x28\x68\xa8\x6d\x5f\x31\xd2\xa3\x01\x06\x85\xe4\x0b\x1c\xeb\x0a\x43\x22\xc5\x68\x86\xe5\x1c\x3e\x19\xb7\xea\x58\xfc\xc4\xac\xaa\xd4\xdc\xc5\xa7\xde\x88\x41\x55\x98\x8c\xe0\xde\xef\x88\x0e\x79\x6e\x9c\x8e\x08\xbd\xa0\xe4\x47\xa4\x40\x14\x5c\xa9\xbb\xc5\x02\xc7\xa0\xe7\x3c\x4b\x5e\xc9\x8f\x6a\x10\x06\x28\xd8\xbe\x33\x30\x4f\x05\xfb\x16\x85\x61\x48\x0c\x73\xff\xa6\x46\x92\x75\x93\x16\x31\x3a\xff\xe9\xe7\x7f\xfe\xfa\xf1\xfa\xe9\xee\xfe\xea\xba\x4d\x9d\x8c\x3b\xdf\xb9\xbe\x30\x0f\x33\xac\x43\xce\x9e\xc3\x1e\x84\x6b\x18\x3d\x73\x58\x2d\x40\x13\x10\x16\xc7\x30\x9f\xbd\x96\x0b\xd5\xdd\x19\x4f\x3c\x1a\x97\x0f\xf5\x87\x1e\x2b\x71\xd1\x32\x75\x60\xba\x8a\x7f\x3a\x3a\xd5\x93\xe2\xd9\x44\x1d\xec\xfe\x3f\x14\x49\x52\x9a\x65\x8c\x43\x0c\xb4\x41\x31\x6a\x24\xe3\xfc\x65\xbe\xd4\x85\x58\x1f\xbe\xb4\x4d\x0c\x57\xba\x38\xd8\x50\xf1\x34\x66\xc2\x84\x8d\xca\xea\x15\x2f\x54\xce\xd7\x52\xe1\x38\x57\x1a\xa1\x8f\x8c\x32\x0d\x79\xb9\xc2\xbc\x8d\x8b\x93\xcf\xd3\x3a\xa0\x4c\xf7\x35\xa4\x78\xe1\x62\xac\x4e\x37\xbd\x43\x1b\x85\xac\x35\x85\x42\xd6\x31\xc3\x6c\xda\xb4\xd2\x6c\xba\x74\xd4\x41\x2b\x67\x48\xb2\x86\x3e\x30\x74\xe4\xd4\x08\x30\xdc\x62\x79\x35\xf4\x34\x07\x08\xeb\xb8\x16\x17\xcd\x02\x1e\x94\x7b\x11\x68\x88\x99\xb1\x05\x5b\x0d\x54\xad\xa0\x36\xe3\xe6\x40\xc9\xdc\x5b\xc3\xf3\x04\xc6\x94\x6a\x30\x66\xbe\xbf\xd2\xef\xad\x91\xaf\xae\x4a\x65\x29\x59\x7f\x30\xd3\x98\x68\x17\xe0\x86\xc5\x41\x79\xb4\x0e\xb7\xad\xbd\x2c\x0a\x95\xf7\x3a\xca\x66\xd3\xbf\x4c\xcf\x47\x55\x99\xe6\x5a\x47\x3b\x49\xf5\x3a\x14\x8d\x89\x44\x7f\xd3\x5b\xb3\x6b\x11\x31\x28\xa6\x5e\xde\x73\x27\xb5\x04\x6c\x53\x0d\x41\x8c\x2d\x98\xf9\x17\xa9\x24\x97\xf1\x66\x5e\xf3\xdc\xc9\x15\xfb\x54\x71\xa7\xb6\x80\x71\x25\xd9\x06\xf6\xe0\x0a\x2c\x49\x80\x32\x97\x1a\x95\xcc\x12\x73\x53\x17\xb2\x2c\x01\x99\xda\xf9\xdf\x6b\x7c\x27\xa0\xb0\xc4\x29\xb7\xc1\xd2\xd8\x8d\x82\x39\x7c\xb3\x7f\xab\x3d\x2f\x1b\x42\xe5\x6f\x4e\x00\x66\xdd\xc1\xfd\xe6\xbd\xbf\x4c\x8e\x32\x31\xd6\x1f\x8c\xab\x1c\x39\x91\x6b\x26\x89\xf7\x5b\xe2\x6b\x1a\xf0\xbe\xdf\x26\x07\x75\xe2\xd9\xc9\xaf\x2b\x8e\x96\x64\xfe\x6d\xde\x6e\xa1\x12\xd4\xa2\x66\x0d\xbb\x55\x68\x5d\xc6\xb6\x4e\xff\x2b\x69\x0a\x13\x7d\xfa\xd1\xdd\xfc\x3b\x9b\x4f\xf1\x13\xcb\x55\xde\x6e\xa4\xde\xdc\xeb\xcb\xed\x2b\x6a\x9d\x9e\x8f\x31\xe4\xa9\x2c\x33\x9b\x70\xb0\xfe\x83\x05\x6a\x04\x1c\x5d\x5c\xa8\x9f\x35\x7b\x19\xd0\x30\x5b\xea\x74\xa8\x9f\xda\x06\x09\xea\xa5\xf0\x08\x7b\x7f\x80\x6e\x74\x28\xaf\xd3\xb1\x7e\x5a\xea\x74\xad\x73\xf9\x3a\x77\xeb\x19\xc0\x25\xb7\x6b\x5e\x6f\xd5\x5f\xf3\x0a\xb3\xd9\x33\x58\x5c\xbd\xba\xf0\x78\x73\x55\x9c\x24\x4e\xf3\xfb\xae\xb7\x29\xdb\x63\x3e\xb6\x16\x93\xd5\x03\xfc\x3b\x65\xda\x9d\x26\xb6\xcd\x4b\x49\x7a\x23\x96\xf2\x5e\xe4\x55\xab\xaa\x67\x45\x79\xf8\xcc\x96\x40\x36\x84\xc3\xad\xa4\xdb\xf7\xd3\x16\xd5\x0b\x79\xf9\x9f\xd7\x6a\x05\x09\x68\xcc\x27\xff\x0b\x00\x00\xff\xff\x2a\x5b\x31\x74\x3b\x32\x00\x00") -func deployKubernetes119FakePmemCsiYamlBytes() ([]byte, error) { +func deployKubernetes119LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( - _deployKubernetes119FakePmemCsiYaml, - "deploy/kubernetes-1.19/fake/pmem-csi.yaml", + _deployKubernetes119LvmPmemCsiYaml, + "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", ) } -func deployKubernetes119FakePmemCsiYaml() (*asset, error) { - bytes, err := deployKubernetes119FakePmemCsiYamlBytes() +func deployKubernetes119LvmPmemCsiYaml() (*asset, error) { + bytes, err := deployKubernetes119LvmPmemCsiYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/fake/pmem-csi.yaml", size: 10897, mode: os.FileMode(436), modTime: time.Unix(1610053046, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 12859, mode: os.FileMode(420), modTime: time.Unix(1611067503, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\x3c\xed\x02\xad\x00\x3f\x64\x12\x6f\x6a\x74\x92\x18\x49\x66\x5f\x03\x86\x3a\x96\x59\x53\x24\x4b\x1e\x69\xe2\x2d\xfa\xdf\x0b\x52\x92\xad\x9b\x15\xdb\x9b\x0b\x8a\x7a\x1e\xc6\xa1\x78\x78\x3e\x9e\xeb\x77\x6c\xff\x91\x5c\x81\x04\x43\x11\x62\xf2\x9d\xe3\x8a\x7c\x4a\xe9\x1a\xc8\x3a\xb3\xa8\x52\xfe\x1b\x7c\xfa\x13\x89\x15\x91\x0a\x09\xc4\x1c\xff\x30\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x11\xc9\x27\xa3\x35\x97\x71\x44\xee\xc1\xe4\x9c\xc1\x39\x63\x2a\x93\x38\x4a\x01\x69\x4c\x91\x46\x23\x42\x04\x7d\x02\x61\xdd\x3b\x42\x74\x0a\x69\xc0\x2c\x1f\x73\x89\x20\xc6\x4c\xa5\x61\x0c\x5a\xa8\x4d\x0a\x12\x23\x22\xf2\x34\xd0\x46\xc5\x19\x43\xae\xe4\x88\x10\x49\x53\x88\xb6\x52\x01\x53\x12\x8d\x12\x02\x4c\xf9\xcc\x6a\xca\x6a\x1b\x46\x41\x10\x34\xf0\x99\x27\xca\xc6\x34\xc3\x95\x32\xfc\x37\xea\x0e\x1d\xaf\xff\x6a\xc7\x5c\x85\x5b\xe4\x77\x4a\xc0\x5b\xe1\x85\x67\x04\x23\xa9\x70\x9b\x72\xee\x20\x81\x09\xd8\x32\xd9\x83\xde\x64\x02\x6c\x34\x0a\x08\xd5\xfc\xca\xa8\x4c\x7b\x14\x01\xf9\xf4\x69\x44\x88\x01\xab\x32\xc3\xa0\x5c\x03\x19\x6b\xc5\x25\xda\x11\x21\x39\x98\xa7\x72\x39\x01\xf4\xff\x7f\xa7\xc8\x56\xfe\x9d\xe0\xb6\x58\x8a\x41\x00\x82\x7f\x9b\xe9\x98\x96\x6f\x99\x01\xf7\xb6\xa3\x93\x29\x65\x62\x2e\xeb\x36\xeb\x82\x10\x40\x2d\xbc\x15\x02\x8b\xca\xd0\x04\xf6\x2a\x67\x96\x97\x5b\x18\xd5\x94\x71\xe4\xfb\xa0\x6c\x01\xec\x30\x95\x4a\x5b\x50\xf4\xf6\x79\x09\xf5\x20\x57\x68\x15\xf7\x28\xee\x88\x52\xad\x6d\x57\xd8\x80\x16\x9c\x51\x0b\x7d\x9e\x3c\x21\x9c\x2f\x44\x66\x11\xcc\xbb\x47\xb5\xc9\xa4\x04\x73\x54\x08\x6b\x77\x2d\x8b\x20\x31\x57\x22\x4b\x4f\xf3\xde\x51\x8e\x6a\x29\x64\x82\xf2\xf4\x70\xad\x65\xa0\x1c\x1d\xa8\x55\x94\x0a\x6a\xf7\x66\x4b\x4b\xdd\x61\x25\x20\x87\x4e\xfe\x1f\x19\xea\xdd\xbb\x48\xaa\xed\x4a\xe1\xf8\xa5\x4b\x95\x2e\x2b\xb7\x0f\xdd\xea\xb5\x74\xb8\xe2\xdf\xbd\xf0\x0b\xaa\x5e\xae\x21\x52\xc5\xaf\xe9\x94\x23\x8e\x3b\xad\x57\x7d\xe1\x32\xe6\x32\x79\xb3\x16\x6b\x79\x33\xaf\x95\x80\xa1\x96\xa5\x04\xdc\xc1\xd2\xa9\xae\x8c\x33\x70\x93\x11\x21\xb5\xa6\x7b\x78\xbb\xb4\xd9\xd3\x3f\x81\xa1\x2f\x2c\xbd\x7c\xe3\x5d\x98\x42\xad\xb4\x7e\x80\x13\x4e\x35\x75\xbd\x21\x1c\x55\xca\xdf\xc1\xe8\x6d\xfa\xf8\x61\xbc\xd1\x6a\x60\x4e\x93\x56\x06\xb7\x5d\xdd\x60\x44\x26\x67\x67\x67\x67\x1e\x02\x52\x93\x00\x2e\x1a\xab\x16\x04\x30\x54\xa6\x00\x49\xb5\xde\xa7\xf3\xd8\x1b\x7c\x94\xa1\x52\x40\xc3\x99\x3d\xc1\x4a\x93\x5e\x2b\x4d\xde\xd4\x4a\x84\xe0\x46\x43\x44\x6e\x54\x0c\x4e\x65\xc7\x6c\x8e\x72\xed\xf2\xf7\x1e\x29\xc2\x32\x13\xf7\xf0\x71\x03\x4a\x65\xc2\x8a\xf6\x45\x64\xd2\xb1\x50\xea\xda\xc3\xd7\x1a\xa8\x17\x8c\x76\x3c\x6a\x5b\x44\xd1\xcd\x00\x78\x84\x54\x0b\x8a\x50\x42\xaa\x99\xcb\xe3\x91\x52\xa1\x2f\x37\x5b\x88\xbd\x28\x2c\x33\xd4\x79\xc8\x1d\x4d\xb9\x04\x63\xcb\xdd\xa2\x71\xbd\x17\x2f\x78\xfc\x15\xf7\x4a\x7d\x87\xa7\x95\x52\xeb\x88\xf0\x44\x2a\x03\x7e\x6b\xe5\x15\x8f\x64\xb9\xe4\x92\xe3\x66\x87\xcd\xb5\xf4\xf3\xce\xaa\xf3\xe1\xbf\x32\x6e\x20\xbe\xcc\x0c\x97\xc9\x3d\x5b\x41\x9c\x09\x2e\x93\xb9\x3f\xb8\x5c\x9e\x3d\x03\xcb\x1c\xa6\xba\x64\x71\xe6\x7d\xe9\xf5\x07\x30\xa9\x6d\x3e\x0e\x8a\x20\x98\x3d\x6b\x03\xd6\x36\xcd\x5c\xed\x58\xc3\x26\xea\xbb\x60\x8f\xf1\xaa\x97\xd2\x6e\xd0\x57\xc6\xa5\x0c\xce\x65\xe7\x79\x4e\x45\x06\x1d\x55\x9e\xea\x48\xf5\xa9\x6f\x79\x49\x85\x85\xea\xc9\xce\xcb\xd5\x11\x6e\x8e\x4c\x53\x2a\xe3\xdd\x99\x01\x09\x33\x6b\x42\xa1\x18\x15\xe1\x13\x97\xe1\xd6\xeb\xb1\xe1\x79\x0d\x74\x40\x82\x7c\xfa\xe7\xfa\x9f\x42\x25\x09\x97\x49\xb0\x54\x26\xa5\x38\x45\x78\xc6\xfa\xe3\x54\xc5\x30\xed\xb9\x7d\x40\x82\x6a\x5a\x9e\x66\x92\x3f\x47\x61\x18\x32\xcb\xc3\x66\xa4\x8d\xad\x62\xeb\xba\x8c\x81\x84\x5b\x34\x9b\x59\x25\x8b\x4c\x47\x61\x78\x36\xf6\xff\xa2\x5d\x67\x28\xf7\x3b\x97\xf2\x78\xfa\xc3\x8f\xff\xf8\xf6\x65\xf6\x78\x73\x7b\x39\x7b\xbc\x39\xbf\x9e\xfd\x54\xdf\xc3\xe8\x2f\x5c\xc0\x34\x64\x60\xd0\x86\x8c\x8e\x99\x69\x5c\xc1\xad\xd7\x77\xa0\xb0\xed\x2d\x6b\xd8\xb4\x77\xac\x61\x53\xdf\x51\xd8\xd1\x55\x9f\xe9\x0f\x3f\x2e\xae\x67\xd7\x8f\x17\xf7\xf3\xc7\xcb\xbb\xf9\xaf\xb3\xbb\x2e\xa4\xb2\xe4\x7f\xf5\x83\xd1\x34\xda\x95\x72\xf7\x02\x99\xd7\x3d\x57\x54\xbb\xe6\xfd\x46\xad\xf0\xf9\xc5\xa8\xb4\x19\x41\x4b\x0e\x22\x2e\x99\x4b\xfd\xd5\x6a\x70\xcd\x87\x5e\x68\x41\x71\x15\xf9\xec\x1c\x3b\xeb\xba\x7a\xd5\x41\xf3\x30\xbb\xbb\x9e\xdf\x9c\x3f\xcc\x6f\x6f\x1e\xbf\xde\x5e\x3d\x2e\xce\x1f\xfe\xde\xc6\x14\x91\x10\x53\x1d\x22\x98\xb4\xfc\x48\xc3\xc5\x52\xe7\xa8\x3e\x5b\x75\x8f\xea\xa6\x5c\xe7\xa0\xab\xdb\xcb\xd9\x97\x6f\x57\x5d\xd9\xe7\x9f\xcf\xfe\x56\xd4\x9d\x8b\x9b\xe9\xce\xcc\x3c\xa5\x09\x44\xc4\x1f\xd8\xce\x87\x88\x51\x49\xcd\xa6\xb9\x77\x91\x09\xb1\x50\x82\xb3\x4d\x44\xe6\xcb\x1b\x85\x0b\x03\x16\xe4\x2e\x50\x6a\x7d\xa9\x95\x56\xdb\xe6\x5d\x41\xde\x66\xed\xa2\xd5\xcb\xeb\x27\xed\x88\x41\xf1\xb2\xc0\x32\xc3\x71\x73\xe1\x86\xb2\x67\x6c\x96\x45\x1a\xdf\x4a\xb1\xb9\x53\xca\xc7\xb2\xdd\x58\x84\x34\x22\x68\xb2\x9d\xf7\x6a\xae\xb8\x06\x6b\xdd\x95\xbc\xaf\x07\xfd\x54\xcc\x83\xd7\x8e\x7c\x36\x6e\x90\xba\x95\x52\xde\x67\x45\x07\x7e\x95\xcb\x3e\xc1\xf6\x49\x5a\xde\x91\xd3\x22\x4b\xb8\x0c\x5c\x69\x00\x0c\x62\x6e\xf6\xc8\x62\xaa\x3b\xb2\x98\xea\x9a\x44\x40\xa8\x49\x1a\xa8\xdb\xd5\xcd\xfb\x9c\xc6\xb1\xab\xf8\xd3\x43\x0a\x54\xb0\x04\x8a\x99\x81\x20\xa1\x08\x76\xfa\xa0\xb4\x12\x2a\xd9\x4c\x1b\x86\x76\xfb\xac\xf3\x1d\x06\xb8\x77\x03\xf2\x14\x54\x86\xd3\x9f\xd3\xc6\x72\x0c\x4b\x9a\x09\x0c\x96\xd6\xf1\xac\x29\x3c\xe3\x5f\x1a\xcf\xcb\xa0\xd8\x82\xf6\xc5\x63\xd2\x8e\x6a\x37\x8f\x24\xcc\xb8\x51\xca\xf2\x24\x28\x47\xf2\xb0\x35\xe3\x44\xf9\xe7\xf1\xd9\xf8\xf3\x49\x61\xde\x37\xc5\x1c\x15\xef\x93\x37\x8f\xf7\x83\x42\xf7\x88\x00\xdc\x8b\xc7\x64\xf2\xdc\xde\x28\xe9\xd0\xb4\x30\xf8\x47\xdf\x2c\x98\x62\x80\xd9\x9e\x54\x1f\xea\x86\x28\xa1\x7b\xa1\x12\x8e\x42\xd4\x19\x49\x40\x60\xb9\x04\x86\x8e\x54\x94\x24\x68\xa7\xd3\x93\x14\x57\xbb\xfd\x0c\x3b\x5e\x67\x4f\x60\x24\x20\xf8\xc9\x3a\xa5\x6e\x32\x1d\xd5\x2d\x54\x3f\x35\xd5\xb8\xb9\xe4\x26\x22\x32\x13\xa2\x5d\xd9\xf6\xd8\x25\x18\x4c\x78\x0b\xcc\x40\xc3\x79\xc5\x4a\xeb\xd6\x5b\xe1\xe2\xa9\xed\x81\xf4\xef\xff\xb4\x00\x55\xd9\x3e\x3c\x82\x5c\x52\x48\x95\x7c\xc3\x01\xc4\x99\xfa\x85\xd1\xe3\xc8\x61\xa3\x3c\xf1\x14\x6c\x1f\x3a\x43\xd4\x70\xbf\xe3\xf4\xf0\xea\x14\x38\x06\x97\x9d\xd7\x54\xd2\x04\xcc\x54\xe4\xe9\xef\xe4\xc7\x0d\xb3\x0c\x30\xe3\x4e\xb7\x39\x88\xde\x6e\xcb\x45\x8b\x34\x97\x52\x8b\xdb\xcb\xc7\xf9\xe2\x27\xcf\x9d\x27\x07\x70\xed\x9e\x3a\xd4\xe5\xdd\xef\xc2\xa9\x2d\x52\xf4\x14\x65\x1a\xe6\xd4\x84\x82\x3f\x85\x07\x70\xeb\xe3\x98\xb8\xbb\xed\x02\x0c\x03\x89\x34\x81\xe9\xa4\x79\xcd\xff\x75\x9e\x5e\x0b\x80\x77\x80\x82\x14\x33\x3b\xd6\x2a\xae\x69\x7b\x75\x9a\xff\x8a\xa3\xc7\xff\xf5\xc4\xa0\x0d\xcf\xb9\x80\x04\xe2\x16\x63\x69\x70\x96\xb3\xb7\x1f\x1e\xaa\xd4\x76\x34\x45\x00\x86\x05\xcb\xb0\x61\x93\xb6\x34\x79\x5a\x21\x6f\x94\xa6\x89\x57\x1a\x91\x2f\x3c\xe6\x06\x7c\x2b\xa1\xa2\x6b\x17\xb7\xdf\x17\xb9\x81\x79\xa2\x03\xa4\xf8\x8a\xf9\x54\x9d\x4e\x7c\x40\x5b\xff\xcc\xe4\x59\xdb\xc0\xbc\x14\x43\xde\x91\x89\x21\x1f\xd0\x63\x37\x5d\x2d\x76\x33\x88\xac\x87\x12\xbf\x38\x8c\x55\xc6\x1b\xc8\xe0\x13\x4c\xe8\x0e\xf3\x5d\xe0\xb8\xa1\xae\xf4\x60\xd5\xe1\x8a\xb8\xd4\x8d\x46\xd2\x8e\xb6\x3d\xad\xa2\xb7\x2b\xf7\x0e\x8d\xcd\x5d\xbd\xbd\xe1\x77\xd7\xc0\x97\x87\x3b\x1f\x3f\x45\x29\xd9\xde\xde\x44\xf9\x64\xfc\x79\x7c\x76\x52\x6d\x6a\x9f\xf5\xe1\xc3\xd9\x8b\x91\x58\x77\x7a\x47\xba\x11\x11\xbb\x33\xea\x9f\x0c\xef\xc0\x94\x86\x2d\x5c\xb2\x6f\x62\x5a\x29\x5b\x68\xae\xd7\xd6\xe1\xca\x36\x98\x23\xc5\xb7\x2b\x97\x3e\x2b\x94\xd9\xdc\x9a\x8b\xea\xa7\x04\x83\x56\x38\x05\xc8\x63\xc5\x00\xc3\xa3\xf5\xef\xb1\xe3\x49\xe6\x18\x2a\xf4\x87\xa1\xe9\x2d\xef\xc7\x61\x69\xd6\xfa\xc3\xd4\xb6\x2a\x7c\xb0\xb7\x86\x1f\x38\x02\x7b\xc1\xf6\xf8\x7b\xc0\x25\x5e\x21\x9e\xf6\x54\xda\x01\xe5\xcd\x4e\x74\x98\x96\x66\xa7\x1a\x3c\xbe\xd9\xb6\x0e\x4c\x8a\xb2\xad\xb5\x3f\x0e\x68\xfe\x20\x25\xcc\x27\x4f\x80\x74\xfb\x03\x83\xfb\xf9\x65\x41\xbd\xde\xe6\xd3\x81\x9a\x53\xaa\x99\x95\x22\x52\xb6\xba\x2b\xbf\xc9\x8a\x88\xff\x3e\xc7\x7f\xc5\x1b\xcf\xe5\x52\xdd\x4a\x5f\x15\xb7\xf5\xb2\x28\x39\x5f\xf9\x12\xd8\x86\x09\xb8\x56\x71\xf5\xbb\x97\xc5\xf6\xb7\x55\xfe\xcf\x99\x5e\x41\x0a\x86\x8a\xd1\x7f\x03\x00\x00\xff\xff\x3a\x78\x18\xc6\x60\x2a\x00\x00") +var _deployKustomizeWebhookWebhookYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x54\x3f\x6f\xfb\x46\x0c\xdd\xfd\x29\x08\x77\x68\x0b\x44\x16\x02\xa4\x40\xa1\xad\x0d\xdc\xc2\x43\x82\xa0\x29\xda\x21\xc8\x70\x3a\xbd\x58\xac\x4f\x47\xe1\xc8\x93\xe3\x6f\x5f\x9c\x2c\x3b\xc9\xf0\x9b\x7f\x9e\x7c\x7c\xfc\xf3\x48\x3e\xca\x8d\xfc\x0f\x92\xb2\xc4\x86\x5c\x37\xb0\x96\xbf\x09\x7b\x56\x4b\xce\x58\xe2\xe6\xf0\xab\x6e\x58\xea\xe9\xb6\x85\xb9\xdb\xd5\x81\x63\xd7\xd0\x43\x36\x67\x1c\xf7\xff\xa2\xed\x45\x0e\xf7\x12\xdf\x78\x9f\xcf\x11\xab\x01\xe6\x3a\x67\xae\x59\x11\x45\x37\xa0\xa1\x71\xc0\x50\x79\xe5\x8a\xa3\x21\x54\x5e\x86\xaa\x84\xad\x8e\xe7\x70\x2d\x9e\xd5\xc5\x57\xba\x19\xdc\x5c\x82\x36\x73\xd0\xc6\xcb\xb0\x22\x3a\x67\xd4\xd1\x79\x3c\x23\xc0\x9b\xa4\x66\x36\x13\x0d\xce\x7c\xbf\x7d\x1f\x13\xe6\x26\xf4\x62\xaf\xe8\x80\xd3\x07\x87\x8f\x74\xf5\x52\x7e\xf1\x23\x92\x11\xc9\x95\x8c\xf4\x28\xb6\x8b\x57\xfb\xe4\x42\x86\x36\xf4\xb2\xe6\x7d\x94\x84\xf5\xeb\x0c\x49\xfb\x1f\xbc\x7d\x77\x1a\x3f\xd0\x1f\x92\xca\xd8\x94\x8e\x6c\x3d\x3d\x3d\x6c\x1f\x68\x92\x90\x07\x28\x65\xe5\xb8\xa7\xe0\x0c\xd4\x72\xec\x38\xee\x6f\x28\x8a\x91\xf3\xc6\xd3\xbc\x42\xb2\x1e\x4b\x1e\xf5\x3d\xba\x1c\x90\x08\xef\x86\x58\xe8\x13\x2b\xc9\xc1\x9d\x58\x7b\xfa\xa9\xcd\x46\x0a\x50\x6f\x36\x6a\x53\xd7\x7b\xb6\x3e\xb7\x73\x0f\x87\xdc\x22\x45\x18\xb4\x34\x57\x97\xf8\x14\x5d\xa8\xc6\x24\x13\x97\x44\x48\x35\xab\x66\x68\xfd\xcb\xdd\xdd\xcf\x9b\xa5\xe2\xce\x7e\x54\x3a\x4a\x52\xd0\x9b\x24\xba\x7f\xde\x11\xc6\x1e\x03\x92\x0b\xd7\x1e\x5a\x78\x97\x15\x85\x68\x42\x21\x14\x85\x12\xbc\x4c\x48\xa7\x4b\xa2\x47\x89\xb0\x1e\x01\xaa\x37\xf4\xe6\x38\x94\xce\x46\xe9\x2e\x3d\x95\x27\xa2\x71\x42\x38\xd1\xb1\x47\x9c\xc7\x54\x95\x82\xac\xd4\xc9\x31\x5e\x66\x00\x0c\x0b\xa5\x1b\x52\xa1\x23\x28\xc0\x4a\xed\x4f\xe3\xf1\x12\x8d\x63\x06\x75\xd0\x91\x0d\x73\xc5\x9c\xa0\x67\x3a\xcb\xeb\x49\x02\xfb\x53\x43\xbb\x79\x5b\x33\xe2\x03\x23\xda\xf9\x5c\x2e\xc2\x50\xa4\x89\x3d\x9a\xeb\x9e\xbf\x79\x33\x57\x02\x5f\x5c\xe7\x63\xf8\xf0\xbf\x62\xa3\xb3\xbe\xa1\x7a\x94\xae\x1e\xca\xb9\x62\x41\xbc\xfb\x3d\xc7\x2e\x2c\x05\x53\x0e\xf8\xa4\xd1\xb3\xf6\x66\xdd\xd2\xcb\xfa\xfe\xaf\xed\x6f\x7f\x6f\x17\x9d\x95\x9f\x1b\xf9\xcf\x24\x79\x9c\xd1\xaf\xf6\xe5\x33\x32\x23\xd3\xed\x27\x2c\x41\x25\x27\x7f\x56\x6e\x91\xe9\xfa\x75\xf5\x7f\x00\x00\x00\xff\xff\xcd\x46\xf3\x6c\x77\x04\x00\x00") -func deployKubernetes119LvmPmemCsiYamlBytes() ([]byte, error) { +func deployKustomizeWebhookWebhookYamlBytes() ([]byte, error) { return bindataRead( - _deployKubernetes119LvmPmemCsiYaml, - "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", + _deployKustomizeWebhookWebhookYaml, + "deploy/kustomize/webhook/webhook.yaml", ) } -func deployKubernetes119LvmPmemCsiYaml() (*asset, error) { - bytes, err := deployKubernetes119LvmPmemCsiYamlBytes() +func deployKustomizeWebhookWebhookYaml() (*asset, error) { + bytes, err := deployKustomizeWebhookWebhookYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 10848, mode: os.FileMode(436), modTime: time.Unix(1610053014, 0)} + info := bindataFileInfo{name: "deploy/kustomize/webhook/webhook.yaml", size: 1143, mode: os.FileMode(420), modTime: time.Unix(1610912692, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deployKustomizeSchedulerSchedulerServiceYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\x8e\xb1\x4a\x04\x41\x0c\x86\xfb\x79\x8a\xbc\xc0\xae\x2b\x5e\x21\xd3\x5a\xd9\x1d\x08\xf6\x63\xf6\x47\x87\x9b\x99\x84\x24\x7b\xe0\xdb\xcb\xae\x16\x87\x5c\x97\xc0\xc7\xf7\xfd\x45\xeb\x3b\xcc\xab\x8c\x4c\xd7\xc7\x74\xa9\x63\xcd\xf4\x06\xbb\x56\x46\xea\x88\xb2\x96\x28\x39\x11\x8d\xd2\x91\x49\x3b\xfa\xc4\x5e\xa7\x3a\x02\x6d\x62\xe9\x93\xf3\x17\xd6\xad\xc1\xfe\x20\xd7\xc2\x37\x64\x72\x05\xef\x02\x47\x03\x87\xd8\x7e\x13\x15\xd5\xf9\xb2\x7d\xc0\x06\x02\x3e\x57\x79\xf8\x17\x60\x19\x61\xd2\x7e\xbd\xf7\xf8\x3a\x3c\xca\xb8\x4d\xcd\xc7\xa8\x99\xa5\x27\xa2\xf8\x56\x64\x7a\x69\x9b\x07\xec\xf5\x9c\x88\x54\x2c\x7c\xaf\x4f\x14\xc5\x3e\x11\x67\xb1\xc8\xf4\xbc\x2c\xcb\x91\xd0\xe3\x3d\x9d\x9e\xd2\x4f\x00\x00\x00\xff\xff\x05\xad\x4e\x1c\x15\x01\x00\x00") + +func deployKustomizeSchedulerSchedulerServiceYamlBytes() ([]byte, error) { + return bindataRead( + _deployKustomizeSchedulerSchedulerServiceYaml, + "deploy/kustomize/scheduler/scheduler-service.yaml", + ) +} + +func deployKustomizeSchedulerSchedulerServiceYaml() (*asset, error) { + bytes, err := deployKustomizeSchedulerSchedulerServiceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/kustomize/scheduler/scheduler-service.yaml", size: 277, mode: os.FileMode(420), modTime: time.Unix(1610912692, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -324,8 +345,9 @@ var _bindata = map[string]func() (*asset, error){ "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml": deployKubernetes119AlphaDirectPmemCsiYaml, "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml": deployKubernetes119AlphaLvmPmemCsiYaml, "deploy/kubernetes-1.19/direct/pmem-csi.yaml": deployKubernetes119DirectPmemCsiYaml, - "deploy/kubernetes-1.19/fake/pmem-csi.yaml": deployKubernetes119FakePmemCsiYaml, "deploy/kubernetes-1.19/lvm/pmem-csi.yaml": deployKubernetes119LvmPmemCsiYaml, + "deploy/kustomize/webhook/webhook.yaml": deployKustomizeWebhookWebhookYaml, + "deploy/kustomize/scheduler/scheduler-service.yaml": deployKustomizeSchedulerSchedulerServiceYaml, } // AssetDir returns the file names below a certain @@ -390,9 +412,6 @@ var _bintree = &bintree{nil, map[string]*bintree{ "direct": &bintree{nil, map[string]*bintree{ "pmem-csi.yaml": &bintree{deployKubernetes119DirectPmemCsiYaml, map[string]*bintree{}}, }}, - "fake": &bintree{nil, map[string]*bintree{ - "pmem-csi.yaml": &bintree{deployKubernetes119FakePmemCsiYaml, map[string]*bintree{}}, - }}, "lvm": &bintree{nil, map[string]*bintree{ "pmem-csi.yaml": &bintree{deployKubernetes119LvmPmemCsiYaml, map[string]*bintree{}}, }}, @@ -405,6 +424,14 @@ var _bintree = &bintree{nil, map[string]*bintree{ "pmem-csi.yaml": &bintree{deployKubernetes119AlphaLvmPmemCsiYaml, map[string]*bintree{}}, }}, }}, + "kustomize": &bintree{nil, map[string]*bintree{ + "scheduler": &bintree{nil, map[string]*bintree{ + "scheduler-service.yaml": &bintree{deployKustomizeSchedulerSchedulerServiceYaml, map[string]*bintree{}}, + }}, + "webhook": &bintree{nil, map[string]*bintree{ + "webhook.yaml": &bintree{deployKustomizeWebhookWebhookYaml, map[string]*bintree{}}, + }}, + }}, }}, }} diff --git a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml index c78dcdb934..efe7d0cc8c 100644 --- a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml +++ b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml @@ -55,15 +55,9 @@ spec: spec: description: DeploymentSpec defines the desired state of Deployment properties: - caCert: - description: CACert encoded root certificate of the CA by which the - registry and node controller certificates are signed If not provided - operator uses a self-signed CA certificate - format: byte - type: string controllerDriverResources: description: ControllerDriverResources Compute resources required - by driver container running on master node + by central driver container properties: limits: additionalProperties: @@ -88,6 +82,12 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object + controllerTLSSecret: + description: ControllerTLSSecret is the name of a secret which contains + ca.crt, tls.crt and tls.key data for the scheduler extender and + pod mutation webhook. A controller is started if (and only if) this + secret is specified. + type: string deviceMode: description: DeviceMode to use to manage PMEM devices. enum: @@ -118,17 +118,14 @@ spec: logLevel: description: LogLevel number for the log verbosity type: integer - nodeControllerCert: - description: NodeControllerCert encoded certificate signed by a CA - for node controller server authentication If not provided, provisioned - one by the operator using self-signed CA - format: byte - type: string - nodeControllerKey: - description: NodeControllerPrivateKey encoded private key used for - node controller server certificate If not provided, provisioned - one by the operator - format: byte + mutatePods: + description: MutatePod defines how a mutating pod webhook is configured + if a controller is started. The field is ignored if the controller + is not enabled. The default is "Try". + enum: + - Always + - Try + - Never type: string nodeDriverResources: description: NodeDriverResources Compute resources required by driver @@ -232,17 +229,15 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object - registryCert: - description: RegistryCert encoded certificate signed by a CA for registry - server authentication If not provided, provisioned one by the operator - using self-signed CA - format: byte - type: string - registryKey: - description: RegistryPrivateKey encoded private key used for registry - server certificate If not provided, provisioned one by the operator - format: byte - type: string + schedulerNodePort: + description: SchedulerNodePort, if non-zero, ensures that the "scheduler" + service is created as a NodeService with that fixed port number. + Otherwise that service is created as a cluster service. The number + must be from the range reserved by Kubernetes for node ports. This + is useful if the kube-scheduler cannot reach the scheduler extender + via a cluster service. + format: int32 + type: integer type: object status: description: DeploymentStatus defines the observed state of Deployment diff --git a/deploy/kubernetes-1.17/direct/pmem-csi.yaml b/deploy/kubernetes-1.17/direct/pmem-csi.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.17/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/direct/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.17/lvm/pmem-csi.yaml b/deploy/kubernetes-1.17/lvm/pmem-csi.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.17/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/lvm/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.17/pmem-csi-direct.yaml b/deploy/kubernetes-1.17/pmem-csi-direct.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.17/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-direct.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.17/pmem-csi-lvm.yaml b/deploy/kubernetes-1.17/pmem-csi-lvm.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.17/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-lvm.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.18/direct/pmem-csi.yaml b/deploy/kubernetes-1.18/direct/pmem-csi.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.18/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/direct/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.18/lvm/pmem-csi.yaml b/deploy/kubernetes-1.18/lvm/pmem-csi.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.18/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/lvm/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.18/pmem-csi-direct.yaml b/deploy/kubernetes-1.18/pmem-csi-direct.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.18/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-direct.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.18/pmem-csi-lvm.yaml b/deploy/kubernetes-1.18/pmem-csi-lvm.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.18/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-lvm.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml index f6029c785f..09ac81b9ab 100644 --- a/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,140 +309,105 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 + image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - name: external-provisioner + name: pmem-driver ports: - - containerPort: 10011 + - containerPort: 10010 name: metrics securityContext: readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + - mountPath: /certs + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -350,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -365,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -393,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -421,8 +477,46 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -440,9 +534,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml index d810dc53c7..a5654e8925 100644 --- a/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,55 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -351,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -372,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -398,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -416,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -444,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -475,8 +520,47 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -494,9 +578,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -518,18 +599,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml index d145fa4ed5..72e1978ba3 100644 --- a/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,140 +309,105 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 + image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - name: external-provisioner + name: pmem-driver ports: - - containerPort: 10011 + - containerPort: 10010 name: metrics securityContext: readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + - mountPath: /certs + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -350,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -365,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -393,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -421,8 +477,46 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -440,9 +534,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml index 01b1295cca..0d1833b85e 100644 --- a/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,55 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -351,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -372,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -398,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -416,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -444,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -475,8 +520,47 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -494,9 +578,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -518,18 +599,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml index d810dc53c7..a5654e8925 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,55 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -351,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -372,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -398,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -416,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -444,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -475,8 +520,47 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -494,9 +578,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -518,18 +599,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml index f6029c785f..09ac81b9ab 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,140 +309,105 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 + image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - name: external-provisioner + name: pmem-driver ports: - - containerPort: 10011 + - containerPort: 10010 name: metrics securityContext: readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + - mountPath: /certs + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -350,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -365,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -393,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -421,8 +477,46 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -440,9 +534,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml index 01b1295cca..0d1833b85e 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,55 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -351,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -372,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -398,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -416,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -444,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -475,8 +520,47 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -494,9 +578,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -518,18 +599,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml index d145fa4ed5..72e1978ba3 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-csi-provisioner-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,140 +309,105 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - --capacity-controller-deployment-mode=central - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 + image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - name: external-provisioner + name: pmem-driver ports: - - containerPort: 10011 + - containerPort: 10010 name: metrics securityContext: readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + - mountPath: /certs + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -350,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -365,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -393,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -421,8 +477,46 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - --enable-capacity + - --capacity-ownerref-level=0 + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -440,9 +534,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19/direct/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/pmem-csi.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.19/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19/fake/kustomization.yaml b/deploy/kubernetes-1.19/fake/kustomization.yaml deleted file mode 100644 index b4145a62f7..0000000000 --- a/deploy/kubernetes-1.19/fake/kustomization.yaml +++ /dev/null @@ -1 +0,0 @@ -resources: [ pmem-csi.yaml ] diff --git a/deploy/kubernetes-1.19/fake/pmem-csi.yaml b/deploy/kubernetes-1.19/fake/pmem-csi.yaml deleted file mode 100644 index a9378c2d94..0000000000 --- a/deploy/kubernetes-1.19/fake/pmem-csi.yaml +++ /dev/null @@ -1,461 +0,0 @@ -# Generated with "make kustomize", do not edit! - -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-external-provisioner-cfg - namespace: pmem-csi -rules: -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - watch - - list - - delete - - update - - create -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - delete - - update - - create -- apiGroups: - - storage.k8s.io - resources: - - csistoragecapacities - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-external-provisioner-runner -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-csi-provisioner-role-cfg - namespace: pmem-csi -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pmem-csi-external-provisioner-cfg -subjects: -- kind: ServiceAccount - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-csi-provisioner-role -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pmem-csi-external-provisioner-runner -subjects: -- kind: ServiceAccount - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: v1 -kind: Service -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi -spec: - ports: - - port: 10000 - targetPort: 10000 - selector: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production ---- -apiVersion: v1 -kind: Service -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-metrics - namespace: pmem-csi -spec: - ports: - - port: 10010 - targetPort: 10010 - selector: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - type: NodePort ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi -spec: - replicas: 1 - selector: - matchLabels: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - serviceName: pmem-csi-controller - template: - metadata: - annotations: - pmem-csi.intel.com/scrape: containers - labels: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - pmem-csi.intel.com/webhook: ignore - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" - containers: - - command: - - /usr/local/bin/pmem-csi-driver - - -v=3 - - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) - - -metricsListen=:10010 - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: TERMINATION_LOG_PATH - value: /tmp/termination-log - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - securityContext: - runAsNonRoot: true - runAsUser: 1000 - serviceAccountName: pmem-csi-controller - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert - secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-node - namespace: pmem-csi -spec: - selector: - matchLabels: - app: pmem-csi-node - pmem-csi.intel.com/deployment: fake-production - template: - metadata: - annotations: - pmem-csi.intel.com/scrape: containers - labels: - app: pmem-csi-node - pmem-csi.intel.com/deployment: fake-production - pmem-csi.intel.com/webhook: ignore - spec: - containers: - - command: - - /usr/local/bin/pmem-csi-driver - - -deviceManager=direct - - -v=3 - - -logging-format=text - - -mode=node - - -endpoint=unix:///csi/csi.sock - - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - - -drivername=$(PMEM_CSI_DRIVER_NAME) - - -pmemPercentage=100 - - -metricsListen=:10010 - - -deviceManager=fake - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - - name: TERMINATION_LOG_PATH - value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - privileged: true - runAsUser: 0 - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi - mountPropagation: Bidirectional - name: mountpoint-dir - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: pods-dir - - mountPath: /certs - name: node-cert - - mountPath: /dev - name: dev-dir - - mountPath: /sys - name: sys-dir - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/pmem-csi.intel.com - mountPropagation: Bidirectional - name: pmem-state-dir - - args: - - -v=3 - - --kubelet-registration-path=/var/lib/kubelet/plugins/$(PMEM_CSI_DRIVER_NAME)/csi.sock - - --csi-address=/csi/csi.sock - env: - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.2.0 - imagePullPolicy: IfNotPresent - name: driver-registrar - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /registration - name: registration-dir - nodeSelector: - storage: pmem - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/pmem-csi.intel.com - type: DirectoryOrCreate - name: socket-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/kubernetes.io/csi - type: DirectoryOrCreate - name: mountpoint-dir - - hostPath: - path: /var/lib/kubelet/pods - type: DirectoryOrCreate - name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - - hostPath: - path: /var/lib/pmem-csi.intel.com - type: DirectoryOrCreate - name: pmem-state-dir - - hostPath: - path: /dev - type: DirectoryOrCreate - name: dev-dir - - hostPath: - path: /sys - type: DirectoryOrCreate - name: sys-dir ---- -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi.intel.com -spec: - attachRequired: false - podInfoOnMount: true - volumeLifecycleModes: - - Persistent - - Ephemeral diff --git a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml index 2fd604ff23..221747bc4a 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: direct-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19/pmem-csi-direct.yaml b/deploy/kubernetes-1.19/pmem-csi-direct.yaml index 8aa197f272..2bd8956bf9 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: direct-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: direct-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kubernetes-1.19/pmem-csi-fake.yaml b/deploy/kubernetes-1.19/pmem-csi-fake.yaml deleted file mode 100644 index a9378c2d94..0000000000 --- a/deploy/kubernetes-1.19/pmem-csi-fake.yaml +++ /dev/null @@ -1,461 +0,0 @@ -# Generated with "make kustomize", do not edit! - -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-external-provisioner-cfg - namespace: pmem-csi -rules: -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - watch - - list - - delete - - update - - create -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - delete - - update - - create -- apiGroups: - - storage.k8s.io - resources: - - csistoragecapacities - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-external-provisioner-runner -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-csi-provisioner-role-cfg - namespace: pmem-csi -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pmem-csi-external-provisioner-cfg -subjects: -- kind: ServiceAccount - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-csi-provisioner-role -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pmem-csi-external-provisioner-runner -subjects: -- kind: ServiceAccount - name: pmem-csi-controller - namespace: pmem-csi ---- -apiVersion: v1 -kind: Service -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi -spec: - ports: - - port: 10000 - targetPort: 10000 - selector: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production ---- -apiVersion: v1 -kind: Service -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-metrics - namespace: pmem-csi -spec: - ports: - - port: 10010 - targetPort: 10010 - selector: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - type: NodePort ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-controller - namespace: pmem-csi -spec: - replicas: 1 - selector: - matchLabels: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - serviceName: pmem-csi-controller - template: - metadata: - annotations: - pmem-csi.intel.com/scrape: containers - labels: - app: pmem-csi-controller - pmem-csi.intel.com/deployment: fake-production - pmem-csi.intel.com/webhook: ignore - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" - containers: - - command: - - /usr/local/bin/pmem-csi-driver - - -v=3 - - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) - - -metricsListen=:10010 - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: TERMINATION_LOG_PATH - value: /tmp/termination-log - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - securityContext: - runAsNonRoot: true - runAsUser: 1000 - serviceAccountName: pmem-csi-controller - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert - secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi-node - namespace: pmem-csi -spec: - selector: - matchLabels: - app: pmem-csi-node - pmem-csi.intel.com/deployment: fake-production - template: - metadata: - annotations: - pmem-csi.intel.com/scrape: containers - labels: - app: pmem-csi-node - pmem-csi.intel.com/deployment: fake-production - pmem-csi.intel.com/webhook: ignore - spec: - containers: - - command: - - /usr/local/bin/pmem-csi-driver - - -deviceManager=direct - - -v=3 - - -logging-format=text - - -mode=node - - -endpoint=unix:///csi/csi.sock - - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - - -drivername=$(PMEM_CSI_DRIVER_NAME) - - -pmemPercentage=100 - - -metricsListen=:10010 - - -deviceManager=fake - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - - name: TERMINATION_LOG_PATH - value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 - image: intel/pmem-csi-driver:canary - imagePullPolicy: IfNotPresent - name: pmem-driver - ports: - - containerPort: 10010 - name: metrics - securityContext: - privileged: true - runAsUser: 0 - terminationMessagePath: /tmp/termination-log - volumeMounts: - - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi - mountPropagation: Bidirectional - name: mountpoint-dir - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: pods-dir - - mountPath: /certs - name: node-cert - - mountPath: /dev - name: dev-dir - - mountPath: /sys - name: sys-dir - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/pmem-csi.intel.com - mountPropagation: Bidirectional - name: pmem-state-dir - - args: - - -v=3 - - --kubelet-registration-path=/var/lib/kubelet/plugins/$(PMEM_CSI_DRIVER_NAME)/csi.sock - - --csi-address=/csi/csi.sock - env: - - name: PMEM_CSI_DRIVER_NAME - value: pmem-csi.intel.com - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.2.0 - imagePullPolicy: IfNotPresent - name: driver-registrar - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /registration - name: registration-dir - nodeSelector: - storage: pmem - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/pmem-csi.intel.com - type: DirectoryOrCreate - name: socket-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/kubernetes.io/csi - type: DirectoryOrCreate - name: mountpoint-dir - - hostPath: - path: /var/lib/kubelet/pods - type: DirectoryOrCreate - name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - - hostPath: - path: /var/lib/pmem-csi.intel.com - type: DirectoryOrCreate - name: pmem-state-dir - - hostPath: - path: /dev - type: DirectoryOrCreate - name: dev-dir - - hostPath: - path: /sys - type: DirectoryOrCreate - name: sys-dir ---- -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - labels: - pmem-csi.intel.com/deployment: fake-production - name: pmem-csi.intel.com -spec: - attachRequired: false - podInfoOnMount: true - volumeLifecycleModes: - - Persistent - - Ephemeral diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml index cd1028d7af..3bccd98fb3 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,82 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +247,29 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-csi-provisioner-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing --- apiVersion: v1 @@ -189,13 +293,14 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: pmem-csi spec: ports: - port: 10002 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -204,14 +309,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing type: NodePort --- @@ -219,64 +325,57 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -285,45 +384,12 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir + name: webhook-cert - mountPath: /var/lib/pmem-csi-coverage name: coverage-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - - -v=5 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir - - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - image: alpine/socat:1.0.3 - name: socat - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir initContainers: - command: - chown @@ -341,18 +407,14 @@ spec: name: coverage-dir securityContext: runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret - hostPath: path: /var/lib/pmem-csi-coverage type: DirectoryOrCreate @@ -362,20 +424,28 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: @@ -388,17 +458,11 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 - -metricsListen=:10010 - -v=5 - - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out env: - name: KUBE_NODE_NAME @@ -406,17 +470,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -434,8 +491,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -465,8 +520,37 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + - -v=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -484,9 +568,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate @@ -508,18 +589,26 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing pmem-csi.intel.com/deployment: lvm-testing template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-testing pmem-csi.intel.com/webhook: ignore spec: diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml index 23c2720183..a0c9c0e6cb 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml @@ -5,7 +5,15 @@ kind: ServiceAccount metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +21,7 @@ kind: Role metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-external-provisioner-cfg namespace: pmem-csi rules: - apiGroups: @@ -64,11 +72,28 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-cfg + namespace: pmem-csi +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-external-provisioner-runner rules: - apiGroups: - "" @@ -139,19 +164,97 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role-cfg + namespace: pmem-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-controller + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role-cfg + name: pmem-csi-intel-com-webhooks-role-cfg namespace: pmem-csi roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: pmem-csi-external-provisioner-cfg + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: pmem-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-intel-com-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-external-provisioner-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi --- apiVersion: rbac.authorization.k8s.io/v1 @@ -159,14 +262,14 @@ kind: ClusterRoleBinding metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-csi-provisioner-role + name: pmem-csi-intel-com-webhooks-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pmem-csi-external-provisioner-runner + name: pmem-csi-intel-com-webhooks-runner subjects: - kind: ServiceAccount - name: pmem-csi-controller + name: pmem-csi-intel-com-webhooks namespace: pmem-csi --- apiVersion: v1 @@ -174,14 +277,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: ports: - port: 10000 targetPort: 10000 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production --- apiVersion: v1 @@ -189,14 +293,15 @@ kind: Service metadata: labels: pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-metrics + name: pmem-csi-intel-com-metrics namespace: pmem-csi spec: ports: - port: 10010 targetPort: 10010 selector: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production type: NodePort --- @@ -204,61 +309,55 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: pmem-csi spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production - serviceName: pmem-csi-controller + serviceName: pmem-csi-intel-com-controller template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-controller + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" containers: - command: - /usr/local/bin/pmem-csi-driver - -v=3 - -logging-format=text - - -mode=controller - - -endpoint=unix:///csi/csi-controller.sock - - -registryEndpoint=tcp://0.0.0.0:10000 - - -nodeid=$(KUBE_NODE_NAME) + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -schedulerListen=:8000 - -metricsListen=:10010 env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -267,67 +366,48 @@ spec: name: metrics securityContext: readOnlyRootFilesystem: true - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - mountPath: /certs - name: registry-cert - - mountPath: /csi - name: plugin-socket-dir - - mountPath: /tmp - name: tmp-dir - - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 - - --metrics-address=:10011 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.2 - imagePullPolicy: IfNotPresent - name: external-provisioner - ports: - - containerPort: 10011 - name: metrics - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir + name: webhook-cert securityContext: runAsNonRoot: true runAsUser: 1000 - serviceAccountName: pmem-csi-controller + serviceAccountName: pmem-csi-intel-com-webhooks tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - emptyDir: null - name: plugin-socket-dir - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - emptyDir: {} - name: tmp-dir + secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: pmem-csi spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production template: metadata: annotations: pmem-csi.intel.com/scrape: containers labels: - app: pmem-csi-node + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi pmem-csi.intel.com/deployment: lvm-production pmem-csi.intel.com/webhook: ignore spec: @@ -340,11 +420,6 @@ spec: - -mode=node - -endpoint=unix:///csi/csi.sock - -nodeid=$(KUBE_NODE_NAME) - - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 - - -registryEndpoint=tcp://pmem-csi-controller:10000 - - -caFile=/certs/ca.crt - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 @@ -355,17 +430,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver @@ -383,8 +451,6 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-dir - - mountPath: /certs - name: node-cert - mountPath: /dev name: dev-dir - mountPath: /sys @@ -411,8 +477,36 @@ spec: name: socket-dir - mountPath: /registration name: registration-dir + - args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 + - --metrics-address=:10011 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + name: external-provisioner + ports: + - containerPort: 10011 + name: metrics + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir nodeSelector: storage: pmem + serviceAccountName: pmem-csi-intel-com-controller volumes: - hostPath: path: /var/lib/kubelet/plugins/pmem-csi.intel.com @@ -430,9 +524,6 @@ spec: path: /var/lib/kubelet/pods type: DirectoryOrCreate name: pods-dir - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - hostPath: path: /var/lib/pmem-csi.intel.com type: DirectoryOrCreate diff --git a/deploy/kustomize/driver/pmem-csi.yaml b/deploy/kustomize/driver/pmem-csi.yaml index 821b77fef3..4ca5afb184 100644 --- a/deploy/kustomize/driver/pmem-csi.yaml +++ b/deploy/kustomize/driver/pmem-csi.yaml @@ -1,17 +1,122 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: default --- apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-cfg + namespace: default +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - patch + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - storageclasses # for scheduler extension + - csinodes # for rescheduler + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role-cfg + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-intel-com-webhooks-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-testing + name: pmem-csi-intel-com-webhooks-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-intel-com-webhooks-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-intel-com-webhooks + namespace: default +--- +# TODO: is this service still needed? +apiVersion: v1 kind: Service metadata: - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: default spec: selector: - app: pmem-csi-controller + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com ports: - port: 10000 targetPort: 10000 @@ -19,12 +124,13 @@ spec: apiVersion: v1 kind: Service metadata: - name: pmem-csi-metrics - namespace: default + name: pmem-csi-intel-com-metrics + namespace: pmem-csi spec: type: NodePort selector: - app: pmem-csi-controller + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com ports: - port: 10010 targetPort: 10010 @@ -32,48 +138,34 @@ spec: apiVersion: apps/v1 kind: StatefulSet metadata: - name: pmem-csi-controller + name: pmem-csi-intel-com-controller namespace: default + labels: + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com spec: replicas: 1 selector: matchLabels: - app: pmem-csi-controller - serviceName: pmem-csi-controller + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + serviceName: pmem-csi-intel-com-controller template: metadata: labels: - app: pmem-csi-controller + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: controller + app.kubernetes.io/instance: pmem-csi.intel.com pmem-csi.intel.com/webhook: ignore spec: securityContext: runAsNonRoot: true # UID 1000 is defined in Dockerfile runAsUser: 1000 - serviceAccountName: pmem-csi-controller - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - # By default, the controller will run anywhere in the cluster. - # If that isn't desired, the "pmem-csi.intel.com/controller" label - # can be set to "no" or "false" for a node to prevent the controller - # from running there. - # - # This is used during testing as a workaround for a particular issue - # on Clear Linux where network configuration randomly fails such that - # the driver which runs on the same node as the controller cannot - # connect to the controller (https://github.com/intel/pmem-csi/issues/555). - # - # It may also be useful for other purposes, in particular for deployment - # through the operator: it has the same rule and currently no other API for - # setting affinity. - - key: pmem-csi.intel.com/controller - operator: NotIn - values: - - "no" - - "false" + serviceAccountName: pmem-csi-intel-com-webhooks # Allow this pod to run on a master node. tolerations: - key: "node-role.kubernetes.io/master" @@ -82,104 +174,78 @@ spec: - name: pmem-driver image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - command: [ - "/usr/local/bin/pmem-csi-driver", - "-v=3", - "-logging-format=text", - "-mode=controller", - "-endpoint=unix:///csi/csi-controller.sock", - "-registryEndpoint=tcp://0.0.0.0:10000", - "-nodeid=$(KUBE_NODE_NAME)", - "-caFile=/certs/ca.crt", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", - "-drivername=$(PMEM_CSI_DRIVER_NAME)", - ] + command: + - /usr/local/bin/pmem-csi-driver + - -v=3 + - -logging-format=text + - -mode=webhooks + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -nodeSelector={"storage":"pmem"} + - -caFile=/certs/ca.crt + - -certFile=/certs/tls.crt + - -keyFile=/certs/tls.key + - -schedulerListen=:8000 securityContext: readOnlyRootFilesystem: true - # Passing /dev to container may cause container creation error because - # termination-log is located on /dev/ by default, re-locate to /tmp - terminationMessagePath: /tmp/termination-log + terminationMessagePath: /dev/termination-log volumeMounts: - - name: registry-cert + - name: webhook-cert mountPath: /certs - - name: plugin-socket-dir - mountPath: /csi - - name: tmp-dir - mountPath: /tmp env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - name: TERMINATION_LOG_PATH - value: /tmp/termination-log + value: /dev/termination-log - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - - name: GODEBUG - value: x509ignoreCN=0 - - name: external-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v1.X.Y - imagePullPolicy: IfNotPresent - args: - - -v=3 - - --csi-address=/csi/csi-controller.sock - - --feature-gates=Topology=true - - --strict-topology=true - - --timeout=5m - - --default-fstype=ext4 # see https://github.com/kubernetes-csi/external-provisioner/issues/328#issuecomment-714801581 - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - name: plugin-socket-dir - mountPath: /csi + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumes: - - name: plugin-socket-dir - emptyDir: - - name: registry-cert + - name: webhook-cert secret: - secretName: pmem-csi-registry-secrets - - name: tmp-dir - emptyDir: {} + secretName: pmem-csi-intel-com-controller-secret --- kind: DaemonSet apiVersion: apps/v1 metadata: - name: pmem-csi-node + name: pmem-csi-intel-com-node namespace: default + labels: + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com spec: selector: matchLabels: - app: pmem-csi-node + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/instance: pmem-csi.intel.com template: metadata: labels: - app: pmem-csi-node + app.kubernetes.io/name: pmem-csi-node + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: node + app.kubernetes.io/instance: pmem-csi.intel.com pmem-csi.intel.com/webhook: ignore spec: + serviceAccountName: pmem-csi-intel-com-controller nodeSelector: storage: pmem containers: - name: pmem-driver image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent - command: [ - "/usr/local/bin/pmem-csi-driver", - "-v=3", - "-logging-format=text", - "-mode=node", - "-endpoint=unix:///csi/csi.sock", - "-nodeid=$(KUBE_NODE_NAME)", - "-controllerEndpoint=tcp://$(KUBE_POD_IP):10001", - "-registryEndpoint=tcp://pmem-csi-controller:10000", - "-caFile=/certs/ca.crt", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", - "-statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME)", - "-drivername=$(PMEM_CSI_DRIVER_NAME)", - "-pmemPercentage=100", - ] + command: + - /usr/local/bin/pmem-csi-driver + - -v=3 + - -logging-format=text + - -mode=node + - -endpoint=unix:///csi/csi.sock + - -nodeid=$(KUBE_NODE_NAME) + - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) + - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -pmemPercentage=100 # Passing /dev to container may cause container creation error because # termination-log is located on /dev/ by default, re-locate to /tmp terminationMessagePath: /tmp/termination-log @@ -192,17 +258,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: KUBE_POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com - name: TERMINATION_LOG_PATH value: /tmp/termination-log - - name: GODEBUG - value: x509ignoreCN=0 volumeMounts: - name: mountpoint-dir mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi @@ -210,8 +269,6 @@ spec: - name: pods-dir mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional - - name: node-cert - mountPath: /certs - name : dev-dir mountPath: /dev - name: sys-dir @@ -232,10 +289,10 @@ spec: - name: driver-registrar image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.X.Y imagePullPolicy: IfNotPresent - args: [ - "-v=3", - "--kubelet-registration-path=/var/lib/kubelet/plugins/$(PMEM_CSI_DRIVER_NAME)/csi.sock", - "--csi-address=/csi/csi.sock" ] + args: + - -v=3 + - --kubelet-registration-path=/var/lib/kubelet/plugins/$(PMEM_CSI_DRIVER_NAME)/csi.sock + - --csi-address=/csi/csi.sock securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -249,6 +306,30 @@ spec: env: - name: PMEM_CSI_DRIVER_NAME value: pmem-csi.intel.com + - name: external-provisioner + image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary + imagePullPolicy: IfNotPresent + args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --timeout=5m + - --default-fstype=ext4 # see https://github.com/kubernetes-csi/external-provisioner/issues/328#issuecomment-714801581 + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - name: socket-dir + mountPath: /csi + env: + # Needed by external-provisioner when using --node-deployment. + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName volumes: - name: socket-dir hostPath: @@ -276,9 +357,6 @@ spec: hostPath: path: /var/lib/kubelet/pods type: DirectoryOrCreate - - name: node-cert - secret: - secretName: pmem-csi-node-secrets - name: pmem-state-dir hostPath: # This state directory must always be the same, regardless of the CSI driver name, diff --git a/deploy/kustomize/driver/webhooks-rbac.yaml b/deploy/kustomize/driver/webhooks-rbac.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/kustomize/kubernetes-1.19-alpha-direct-coverage/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-direct-coverage/kustomization.yaml index 13098c0f0c..c036fe5e12 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-direct-coverage/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-direct-coverage/kustomization.yaml @@ -8,6 +8,6 @@ patchesJson6902: - target: group: apps version: v1 - kind: StatefulSet - name: pmem-csi-controller + kind: DaemonSet + name: pmem-csi-intel-com-node path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-1.19-alpha-direct-testing/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-direct-testing/kustomization.yaml index bff3aad50a..968e1e4875 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-direct-testing/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-direct-testing/kustomization.yaml @@ -9,5 +9,5 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-1.19-alpha-direct/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-direct/kustomization.yaml index 2019e38cc2..249eebb9d5 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-direct/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-direct/kustomization.yaml @@ -8,6 +8,6 @@ patchesJson6902: - target: group: apps version: v1 - kind: StatefulSet - name: pmem-csi-controller + kind: DaemonSet + name: pmem-csi-intel-com-node path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-1.19-alpha-lvm-coverage/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-lvm-coverage/kustomization.yaml index 8ce9bf381c..ad27b11679 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-lvm-coverage/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-lvm-coverage/kustomization.yaml @@ -8,6 +8,6 @@ patchesJson6902: - target: group: apps version: v1 - kind: StatefulSet - name: pmem-csi-controller + kind: DaemonSet + name: pmem-csi-intel-com-node path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-1.19-alpha-lvm-testing/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-lvm-testing/kustomization.yaml index 026a4690ba..593c9c4fc2 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-lvm-testing/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-lvm-testing/kustomization.yaml @@ -9,5 +9,5 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-1.19-alpha-lvm/kustomization.yaml b/deploy/kustomize/kubernetes-1.19-alpha-lvm/kustomization.yaml index 95b1150f77..f51a927117 100644 --- a/deploy/kustomize/kubernetes-1.19-alpha-lvm/kustomization.yaml +++ b/deploy/kustomize/kubernetes-1.19-alpha-lvm/kustomization.yaml @@ -8,6 +8,6 @@ patchesJson6902: - target: group: apps version: v1 - kind: StatefulSet - name: pmem-csi-controller + kind: DaemonSet + name: pmem-csi-intel-com-node path: ../patches/external-provisioner-storage-capacity-patch.yaml diff --git a/deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml b/deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml index 9038bd5687..7c33526a3f 100644 --- a/deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml @@ -6,17 +6,16 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller path: ../testing/controller-coverage-patch.yaml - target: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: ../testing/node-coverage-patch.yaml images: - name: intel/pmem-csi-driver newName: intel/pmem-csi-driver-test - diff --git a/deploy/kustomize/kubernetes-base-direct-testing/kustomization.yaml b/deploy/kustomize/kubernetes-base-direct-testing/kustomization.yaml index 3f59c54dab..371863c678 100644 --- a/deploy/kustomize/kubernetes-base-direct-testing/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-direct-testing/kustomization.yaml @@ -10,19 +10,12 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller - path: ../testing/controller-socat-patch.yaml - -- target: - group: apps - version: v1 - kind: StatefulSet - name: pmem-csi-controller - path: ../testing/args-two-containers-patch.yaml + name: pmem-csi-intel-com-controller + path: ../testing/controller-verbosity-patch.yaml - target: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node - path: ../testing/args-two-containers-patch.yaml + name: pmem-csi-intel-com-node + path: ../testing/node-verbosity-patch.yaml diff --git a/deploy/kustomize/kubernetes-base-direct/kustomization.yaml b/deploy/kustomize/kubernetes-base-direct/kustomization.yaml index 7e45e8976c..c703d84add 100644 --- a/deploy/kustomize/kubernetes-base-direct/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-direct/kustomization.yaml @@ -11,5 +11,5 @@ patchesJson6902: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: ../patches/direct-patch.yaml diff --git a/deploy/kustomize/kubernetes-base-fake/kustomization.yaml b/deploy/kustomize/kubernetes-base-fake/kustomization.yaml index 8d0b1248bc..be5c16f152 100644 --- a/deploy/kustomize/kubernetes-base-fake/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-fake/kustomization.yaml @@ -9,5 +9,5 @@ patchesJson6902: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: fake-device-mode-patch.yaml diff --git a/deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml b/deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml index e7873a98bb..e4c6e8aaf9 100644 --- a/deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml @@ -6,14 +6,14 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller path: ../testing/controller-coverage-patch.yaml - target: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: ../testing/node-coverage-patch.yaml images: diff --git a/deploy/kustomize/kubernetes-base-lvm-testing/kustomization.yaml b/deploy/kustomize/kubernetes-base-lvm-testing/kustomization.yaml index 587c247cb1..f8d3978329 100644 --- a/deploy/kustomize/kubernetes-base-lvm-testing/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-lvm-testing/kustomization.yaml @@ -10,19 +10,12 @@ patchesJson6902: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller - path: ../testing/controller-socat-patch.yaml - -- target: - group: apps - version: v1 - kind: StatefulSet - name: pmem-csi-controller - path: ../testing/args-two-containers-patch.yaml + name: pmem-csi-intel-com-controller + path: ../testing/controller-verbosity-patch.yaml - target: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node - path: ../testing/args-two-containers-patch.yaml + name: pmem-csi-intel-com-node + path: ../testing/node-verbosity-patch.yaml diff --git a/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml b/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml index 955e914373..bd7d9293ab 100644 --- a/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml @@ -11,5 +11,5 @@ patchesJson6902: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: ../patches/lvm-patch.yaml diff --git a/deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml b/deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml index 4f8296cd0d..b58ee72291 100644 --- a/deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml +++ b/deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml @@ -1,4 +1,4 @@ -namePrefix: pmem-csi- +namePrefix: pmem-csi-intel-com- resources: - https://github.com/kubernetes-csi/external-provisioner/raw/v2.0.0/deploy/kubernetes/rbac.yaml diff --git a/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml b/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml index 48f98053d8..7e01834813 100644 --- a/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml +++ b/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml @@ -8,12 +8,12 @@ patchesJson6902: group: apps version: v1 kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node path: ../patches/metrics-node.yaml - target: group: apps version: v1 kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller path: ../patches/metrics-controller.yaml diff --git a/deploy/kustomize/operator/operator.yaml b/deploy/kustomize/operator/operator.yaml index bfdd5c9586..2a4f7b3199 100644 --- a/deploy/kustomize/operator/operator.yaml +++ b/deploy/kustomize/operator/operator.yaml @@ -48,6 +48,7 @@ rules: - "" resources: - pods + - secrets verbs: - get --- @@ -77,6 +78,12 @@ rules: - pmemcsideployments/finalizers verbs: - '*' +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - '*' --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/kustomize/patches/controller-role-patch.yaml b/deploy/kustomize/patches/controller-role-patch.yaml index d68542f582..f5217cd5cf 100644 --- a/deploy/kustomize/patches/controller-role-patch.yaml +++ b/deploy/kustomize/patches/controller-role-patch.yaml @@ -1,4 +1,4 @@ # Applies to a [Cluster]RoleBinding and sets the name of the first target to "controller". - op: replace path: /subjects/0/name - value: pmem-csi-controller + value: pmem-csi-intel-com-controller diff --git a/deploy/kustomize/patches/external-provisioner-storage-capacity-patch.yaml b/deploy/kustomize/patches/external-provisioner-storage-capacity-patch.yaml index d62d2cb34e..57f8977437 100644 --- a/deploy/kustomize/patches/external-provisioner-storage-capacity-patch.yaml +++ b/deploy/kustomize/patches/external-provisioner-storage-capacity-patch.yaml @@ -1,16 +1,20 @@ -# Add -connection-timeout to external-controller in second container. - op: add - path: /spec/template/spec/containers/1/args/0 - value: "--capacity-controller-deployment-mode=central" -# Add the necessary environment variables. + path: /spec/template/spec/containers/2/args/0 + value: "--enable-capacity" - op: add - path: /spec/template/spec/containers/1/env + path: /spec/template/spec/containers/2/args/1 + value: "--capacity-ownerref-level=0" +- op: add + path: /spec/template/spec/containers/2/env/- value: - - name: POD_NAMESPACE + name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME +- op: add + path: /spec/template/spec/containers/2/env/- + value: + name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name diff --git a/deploy/kustomize/patches/metrics-controller.yaml b/deploy/kustomize/patches/metrics-controller.yaml index f2c680ad52..3450ce7a3f 100644 --- a/deploy/kustomize/patches/metrics-controller.yaml +++ b/deploy/kustomize/patches/metrics-controller.yaml @@ -14,17 +14,3 @@ - op: add path: /spec/template/spec/containers/0/command/- value: -metricsListen=:10010 - -# external-provisioner: -- op: add - path: /spec/template/metadata/annotations - value: - pmem-csi.intel.com/scrape: containers -- op: add - path: /spec/template/spec/containers/1/ports - value: - - name: metrics - containerPort: 10011 -- op: add - path: /spec/template/spec/containers/1/args/- - value: --metrics-address=:10011 diff --git a/deploy/kustomize/patches/metrics-node.yaml b/deploy/kustomize/patches/metrics-node.yaml index 4ed47c8b7c..254c886698 100644 --- a/deploy/kustomize/patches/metrics-node.yaml +++ b/deploy/kustomize/patches/metrics-node.yaml @@ -16,3 +16,17 @@ value: -metricsListen=:10010 # TODO: node-driver-registrar once it has metrics support. + +# external-provisioner: +- op: add + path: /spec/template/metadata/annotations + value: + pmem-csi.intel.com/scrape: containers +- op: add + path: /spec/template/spec/containers/2/ports + value: + - name: metrics + containerPort: 10011 +- op: add + path: /spec/template/spec/containers/2/args/- + value: --metrics-address=:10011 diff --git a/deploy/kustomize/scheduler/scheduler-service.yaml b/deploy/kustomize/scheduler/scheduler-service.yaml index f32b40a220..7d44c76f63 100644 --- a/deploy/kustomize/scheduler/scheduler-service.yaml +++ b/deploy/kustomize/scheduler/scheduler-service.yaml @@ -1,11 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: pmem-csi-scheduler + name: pmem-csi-intel-com-scheduler + namespace: pmem-csi spec: selector: - app: pmem-csi-controller - type: NodePort + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + type: ClusterIP ports: - targetPort: 8000 port: 443 diff --git a/deploy/kustomize/testing/README.md b/deploy/kustomize/testing/README.md index bf6bf81464..dbcd19899a 100644 --- a/deploy/kustomize/testing/README.md +++ b/deploy/kustomize/testing/README.md @@ -1,14 +1,9 @@ # Testing This mixin for a regular production deployment of PMEM-CSI adds port -forwarding to the outside world: +forwarding to the outside world. -The pmem-csi-controller-testing Service exposes the PMEM-CSI controller's -csi.sock as a TCP service with a dynamically allocated port, on any -node of the cluster. For this to work, the pmem-csi-controller has -to be patched with the controller-socat-patch.yaml. - -The pmem-csi-node-testing DaemonSet forwards +The pmem-csi-intel-com-node-testing DaemonSet forwards /var/lib/kubelet/plugins/pmem-csi.intel.com/csi.sock on all nodes, using the fixed port 9735 (arbitrarily chosen). The advantage of this approach is that: diff --git a/deploy/kustomize/testing/args-two-containers-patch.yaml b/deploy/kustomize/testing/args-two-containers-patch.yaml deleted file mode 100644 index 377b1cda4a..0000000000 --- a/deploy/kustomize/testing/args-two-containers-patch.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Raise log verbosity level to 5 in containers -# and set -testEndpoint to pmem-csi-driver container (expected to -# come first). -- op: add - path: /spec/template/spec/containers/0/command/- - value: "-v=5" - -- op: add - path: /spec/template/spec/containers/0/command/- - value: "-testEndpoint" - -- op: add - path: /spec/template/spec/containers/1/args/- - value: "-v=5" diff --git a/deploy/kustomize/testing/controller-socat-patch.yaml b/deploy/kustomize/testing/controller-socat-patch.yaml deleted file mode 100644 index fc4bb4497d..0000000000 --- a/deploy/kustomize/testing/controller-socat-patch.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- op: add - path: /spec/template/spec/containers/- - value: - name: socat - image: alpine/socat:1.0.3 - args: - - -s - - tcp-listen:10002,fork,reuseaddr - - unix-connect:/csi/csi-controller.sock - volumeMounts: - - mountPath: /csi - name: plugin-socket-dir diff --git a/deploy/kustomize/testing/controller-verbosity-patch.yaml b/deploy/kustomize/testing/controller-verbosity-patch.yaml new file mode 100644 index 0000000000..5deb14e1ec --- /dev/null +++ b/deploy/kustomize/testing/controller-verbosity-patch.yaml @@ -0,0 +1,4 @@ +# Raise log verbosity level to 5 in first container. +- op: add + path: /spec/template/spec/containers/0/command/- + value: "-v=5" diff --git a/deploy/kustomize/testing/node-verbosity-patch.yaml b/deploy/kustomize/testing/node-verbosity-patch.yaml new file mode 100644 index 0000000000..0773b11490 --- /dev/null +++ b/deploy/kustomize/testing/node-verbosity-patch.yaml @@ -0,0 +1,12 @@ +# Raise log verbosity level to 5 in the DaemonSet. +- op: add + path: /spec/template/spec/containers/0/command/- + value: "-v=5" + +- op: add + path: /spec/template/spec/containers/1/args/- + value: "-v=5" + +- op: add + path: /spec/template/spec/containers/2/args/- + value: "-v=5" diff --git a/deploy/kustomize/testing/socat.yaml b/deploy/kustomize/testing/socat.yaml index 3cd3edcc8e..3bd1cb77ca 100644 --- a/deploy/kustomize/testing/socat.yaml +++ b/deploy/kustomize/testing/socat.yaml @@ -1,28 +1,38 @@ apiVersion: v1 kind: Service metadata: - name: pmem-csi-controller-testing + name: pmem-csi-intel-com-controller-testing namespace: default spec: type: NodePort selector: - app: pmem-csi-controller + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com ports: - port: 10002 # port inside the pod where controller-socat-path.yaml forwards the csi.sock --- apiVersion: apps/v1 kind: DaemonSet metadata: - name: pmem-csi-node-testing + name: pmem-csi-intel-com-node-testing namespace: default + labels: + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com spec: selector: matchLabels: - app: pmem-csi-node-testing + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/instance: pmem-csi.intel.com template: metadata: labels: - app: pmem-csi-node-testing + app.kubernetes.io/name: pmem-csi-node-testing + app.kubernetes.io/part-of: pmem-csi + app.kubernetes.io/component: node-testing + app.kubernetes.io/instance: pmem-csi.intel.com pmem-csi.intel.com/webhook: ignore spec: hostNetwork: true diff --git a/deploy/kustomize/vpa-for-pmem-csi/vpa-controller.yaml b/deploy/kustomize/vpa-for-pmem-csi/vpa-controller.yaml index 0106e54324..93e8a43dec 100644 --- a/deploy/kustomize/vpa-for-pmem-csi/vpa-controller.yaml +++ b/deploy/kustomize/vpa-for-pmem-csi/vpa-controller.yaml @@ -1,11 +1,11 @@ apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - name: pmem-csi-controller + name: pmem-csi-intel-com-controller spec: targetRef: apiVersion: "apps/v1" kind: StatefulSet - name: pmem-csi-controller + name: pmem-csi-intel-com-controller updatePolicy: updateMode: "Off" diff --git a/deploy/kustomize/vpa-for-pmem-csi/vpa-node.yaml b/deploy/kustomize/vpa-for-pmem-csi/vpa-node.yaml index 36616b5384..17b3a26d47 100644 --- a/deploy/kustomize/vpa-for-pmem-csi/vpa-node.yaml +++ b/deploy/kustomize/vpa-for-pmem-csi/vpa-node.yaml @@ -1,11 +1,11 @@ apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - name: pmem-csi-node + name: pmem-csi-intel-com-node spec: targetRef: apiVersion: "apps/v1" kind: DaemonSet - name: pmem-csi-node + name: pmem-csi-intel-com-node updatePolicy: updateMode: "Off" diff --git a/deploy/kustomize/webhook/webhook.yaml b/deploy/kustomize/webhook/webhook.yaml index bfd6827439..491be408ec 100644 --- a/deploy/kustomize/webhook/webhook.yaml +++ b/deploy/kustomize/webhook/webhook.yaml @@ -1,7 +1,7 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: - name: pmem-csi-hook + name: pmem-csi-intel-com-hook webhooks: - name: pod-hook.pmem-csi.intel.com namespaceSelector: @@ -14,11 +14,16 @@ webhooks: - key: pmem-csi.intel.com/webhook operator: NotIn values: ["ignore"] - failurePolicy: Fail + # For pods with PMEM volumes using late binding, not activating the + # scheduler extension is okayish (but see https://github.com/kubernetes-csi/external-provisioner/issues/544). + # It's worse for CSI ephemeral volumes because there is no recovery. + # Nonetheless, failing pod scheduling entirely when PMEM-CSI is down + # seems worse, so we let the scheduler continue despite failures. + failurePolicy: Ignore clientConfig: service: - name: pmem-csi-scheduler - namespace: default + name: pmem-csi-intel-com-scheduler + namespace: pmem-csi path: /pod/mutate caBundle: rules: diff --git a/deploy/operator/pmem-csi-operator.yaml b/deploy/operator/pmem-csi-operator.yaml index 830d21eb42..90fbc216c1 100644 --- a/deploy/operator/pmem-csi-operator.yaml +++ b/deploy/operator/pmem-csi-operator.yaml @@ -62,6 +62,7 @@ rules: - "" resources: - pods + - secrets verbs: - get --- @@ -91,6 +92,12 @@ rules: - pmemcsideployments/finalizers verbs: - '*' +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/deploy/yamls.go b/deploy/yamls.go index c98156660f..4c86327884 100644 --- a/deploy/yamls.go +++ b/deploy/yamls.go @@ -40,7 +40,7 @@ func init() { for _, file := range AssetNames() { parts := re.FindStringSubmatch(file) if parts == nil { - panic(fmt.Sprintf("unexpected deployment asset: %s", file)) + continue } kubernetes, err := version.Parse(parts[1]) if err != nil { diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 32f1a9c197..9b71df8821 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -195,18 +195,9 @@ GetCapacity, GetCapabilities, GetPluginInfo, GetPluginCapabilities. Network ports are opened as configured in manifest files: -- registry endpoint: typical port value 10000, used for PMEM-CSI internal communication -- controller endpoint: typical port value 10001, used by the nodes for - providing the serving CSI API to the PMEM-CSI controller - metrics endpoint: typical port values 10010 (PMEM-CSI) and 10011 (external-provisioner) - webhook endpoint: disabled by default, port chosen when [enabling the scheduler extensions](../README.md#enable-scheduler-extensions) -Except for the metrics and webhook endpoint, all ports are protected -via mutual TLS. The metrics endpoint and webhook are supposed to be -easily usable and expose no confidential data, therefore TLS is not -used. - - ### Local sockets Kubernetes CSI API used over local socket inside same host. @@ -218,37 +209,16 @@ Kubernetes CSI API used over local socket inside same host. ### Command line arguments -argument name | meaning | type | range --------------------------|---------------------------------------------------|--------|--- --alsologtostderr | log to standard error as well as files | | --log_backtrace_at value | when logging hits line file:N, emit a stack trace | | --log_dir string | If non-empty, write log files in this directory | string | --log_file string | If non-empty, use this log file | string | --logtostderr | log to standard error instead of files | | --skip_headers | avoid header prefixes in the log messages | | --stderrthreshold value | logs at or above this threshold go to stderr (default 2) | | --v value | log level for V logs | int | --vmodule value | comma-separated list of pattern=N settings for file-filtered logging | string | --caFile string | Root CA certificate file to use for verifying connections | string | | --certFile string | SSL certificate file to use for authenticating client connections(RegistryServer/NodeControllerServer) | string | | --clientCertFile string | Client SSL certificate file to use for authenticating peer connections | string | | certFile --clientKeyFile string | Client private key associated to client certificate | string | | keyFile --controllerEndpoint string | internal node controller endpoint | string | | --deviceManager string | device mode to use. ndctl selects mode which is described as direct mode in documentation. | string | lvm or ndctl | lvm --drivername string | name of the driver | string | | pmem-csi --endpoint string | PMEM CSI endpoint | string | | unix:///tmp/pmem-csi.sock --keyFile string | Private key file associated to certificate | string | | --mode string | driver run mode | string | controller, node | --nodeid string | node id | string | | nodeid --registryEndpoint string | endpoint to connect/listen registry server | string | | --statePath | Directory path where to persist the state of the driver running on a node | string | absolute directory path on node | /var/lib/ --schedulerListen | listen address for scheduler extender and mutating webhook | [address string](https://golang.org/pkg/net/#Listen) | controller | empty (= disabled) --pmemPercentage value | represents the percentage of space to be used by the driver in each PMEM region
(currently only supported by the driver in LVM mode) | int | 0-100 +See the `main.go` files of the [pmem-csi-driver](./pkg/pmem-csi-driver/main.go) and +the [pmem-csi-operator](./pkg/pmem-csi-operator/main.go) commands. ### Environment variables -TEST_WORK is used by registry server unit-test code to specify path to certificates in test system. -Note, THIS IS NOT USED IN PRODUCTION +TEST_WORK is used by registry server unit-test code to specify path to certificates in test system. +Note, THIS IS NOT USED IN PRODUCTION. + +NODE_NAME is a copy of the node name set for the pod which runs the +`external-provisioner` on each node. ### Logging @@ -307,35 +277,6 @@ The default resource requirements used for the driver deployments by the operato are chosen from the VPA recommendations described in this section when using the `stress-driver.sh` script. -## Switching device mode - -If device mode is switched between LVM and direct(aka ndctl), please keep -in mind that PMEM-CSI driver does not clean up or reclaim namespaces, -therefore namespaces plus other related context (LVM state) -created in previous mode will remain stored on device and most likely -will create trouble in another device mode. - -### Going from LVM device mode to direct device mode - -- examine LV groups state on a node: `vgs` -- examine LV physical volumes state on a node: `pvs` -- delete LV groups before deleting namespaces to avoid orphaned volume groups: `vgremove VGNAME` - -NOTE: The following **WILL DELETE ALL NAMESPACES** so be careful! - -- Delete namespaces on a node using CLI: `ndctl destroy-namespace all --force` - -### Going from direct device mode to LVM device mode - -No special steps are needed to clean up namespaces state. - -If PMEM-CSI driver has been operating correctly, there should not be -existing namespaces as CSI volume lifecycle should have been deleted -those after end of life of volume. If there are, you can either keep -those (LVM device mode does honor "foreign" namespaces and leaves those -alone) if you have enough space, or you can choose to delete those -using `ndctl` on node. - ## Accessing system directories in a container The PMEM-CSI driver will run as container, but it needs access to @@ -391,23 +332,6 @@ Source files: The PNG files are committed as repository elements in docs/images/sequence/. -### RegistryServer spec - -pkg/pmem-registry/pmem-registry.pb.go is generated from pkg/pmem-registry/pmem-registry.proto - -protoc comes from package _protobuf-compiler_ on Ubuntu 18.04 -- get protobuf for Go: -``` console -$ git clone https://github.com/golang/protobuf.git && cd protobuf -$ make # installs needed binary in $GOPATH/bin/protoc-gen-go -``` - -- generate by running in \~/go/src/github.com/intel/pmem-csi/pkg/pmem-registry: - -``` console -$ protoc --plugin=protoc-gen-go=$GOPATH/bin/protoc-gen-go --go_out=plugins=grpc:./ pmem-registry.proto -``` - ### Table of Contents in README and DEVELOPMENT Table of Contents can be generated using multiple methods. diff --git a/docs/design.md b/docs/design.md index 487846ed59..4ce00430ab 100644 --- a/docs/design.md +++ b/docs/design.md @@ -5,8 +5,7 @@ - [LVM device mode](#lvm-device-mode) - [Direct device mode](#direct-device-mode) - [Kata Containers support](#kata-containers-support) - - [Driver modes](#driver-modes) - - [Driver Components](#driver-components) + - [Dynamic provisioning of local volumes](#dynamic-provisioning-of-local-volumes) - [Communication between components](#communication-between-components) - [Security](#security) - [Volume Persistency](#volume-persistency) @@ -155,88 +154,35 @@ Kata Containers support has to be enabled explicitly via a [storage class parameter and Kata Containers must be set up appropriately](install.md#kata-containers-support). -## Driver modes - -The PMEM-CSI driver supports running in different modes, which can be -controlled by passing one of the below options to the driver's -'_-mode_' command line option. In each mode, it starts a different set -of open source Remote Procedure Call (gRPC) -[servers](#driver-components) on given driver endpoint(s). - -* **_Controller_** should run as a single instance in cluster level. When the - driver is running in _Controller_ mode, it forwards the pmem volume - create/delete requests to the registered node controller servers - running on the worker node. In this mode, the driver starts the - following gRPC servers: - - * [IdentityServer](#identity-server) - * [NodeRegistryServer](#node-registry-server) - * [MasterControllerServer](#master-controller-server) - -* One **_Node_** instance should run on each - worker node that has persistent memory devices installed. When the - driver starts in such mode, it registers with the _Controller_ - driver running on a given _-registryEndpoint_. In this mode, the - driver starts the following servers: - - * [IdentityServer](#identity-server) - * [NodeControllerServer](#node-controller-server) - * [NodeServer](#node-server) - -## Driver Components - -### Identity Server - -This gRPC server operates on a given endpoint in all driver modes and -implements the CSI [Identity -interface](https://github.com/container-storage-interface/spec/blob/master/spec.md#identity-service-rpc). - -### Node Registry Server - -When the PMEM-CSI driver runs in _Controller_ mode, it starts a gRPC -server on a given endpoint(_-registryEndpoint_) and serves the -[RegistryServer](/pkg/pmem-registry/pmem-registry.proto) interface. The -driver(s) running in _Node_ mode can register themselves with node -specific information such as node id, -[NodeControllerServer](#node-controller-server) endpoint, and their -available persistent memory capacity. - -### Master Controller Server - -This gRPC server is started by the PMEM-CSI driver running in -_Controller_ mode and serves the -[Controller](https://github.com/container-storage-interface/spec/blob/master/spec.md#controller-service-rpc) -interface defined by the CSI specification. The server responds to -CreateVolume(), DeleteVolume(), ControllerPublishVolume(), -ControllerUnpublishVolume(), and ListVolumes() calls coming from -external-provisioner() and external-attacher() sidecars. It -forwards the publish and unpublish volume requests to the appropriate -[Node controller server](#node-controller-server) running on a worker -node that was registered with the driver. - -### Node Controller Server - -This gRPC server is started by the PMEM-CSI driver running in _Node_ -mode and implements the -[ControllerPublishVolume](https://github.com/container-storage-interface/spec/blob/master/spec.md#controllerpublishvolume) -and -[ControllerUnpublishVolume](https://github.com/container-storage-interface/spec/blob/master/spec.md#controllerunpublishvolume) -methods of the [Controller -service](https://github.com/container-storage-interface/spec/blob/master/spec.md#controller-service-rpc) -interface defined by the CSI specification. It serves the -ControllerPublishVolume() and ControllerUnpublish() requests coming -from the [Master controller server](#master-controller-server) and -creates/deletes persistent memory devices. - -### Node Server - -This gRPC server is started by the driver running in _Node_ mode and -implements the [Node -service](https://github.com/container-storage-interface/spec/blob/master/spec.md#node-service-rpc) -interface defined in the CSI specification. It serves the -NodeStageVolume(), NodeUnstageVolume(), NodePublishVolume(), and -NodeUnpublishVolume() requests coming from the Container Orchestrator -(CO). +## Dynamic provisioning of local volumes + +Traditionally, Kubernetes expects that a driver deployment has a +central component, usually implemented with the `external-provisioner` +and a custom CSI driver component which implements volume creation. +That central component is hard to implement for a CSI driver that +creates volumes locally on a node. + +PMEM-CSI solves this problem by deploying `external-provisioner` +alongside each node driver and enabling ["distributed +provisioning"](https://github.com/kubernetes-csi/external-provisioner/tree/v2.1.0#deployment-on-each-node): +- For volumes with storage classes that use late binding (aka "wait + for first consumer"), a volume is tentatively assigned to a node + before creating it, in which case the `external-provisioner` running + on that node can tell that it is responsible for provisioning. +- For volumes with storage classes that use immediate binding, the + different `external-provisioner` instances compete with each for + ownership of the volume by setting the "selected node" + annotation. Delays are used to avoid the thundering herd problem. + Once a node has been selected, provisioning continues as with late + binding. This is less efficient and therefore "late binding" is the + recommended binding mode. + +PMEM-CSI also has a central component which implements the [scheduler +extender](#scheduler-extender) webhook. That component needs to know +on which nodes the PMEM-CSI driver is running and how much capacity is +available there. This information is retrieved by dynamically +discovering PMEM-CSI pods and connecting to their [metrics +endpoint](/docs/install.md#metrics-support). ## Communication between components @@ -245,18 +191,17 @@ The following diagram illustrates the communication channels between driver comp ## Security -All PMEM-CSI specific communication [shown in above -section](#communication-between-components) between Master -Controller([RegistryServer](#node-registry-server), -[MasterControllerServer](#master-controller-server)) and -NodeControllers([NodeControllerServer](#node-controller-server)) is -protected by mutual TLS. Both client and server must identify -themselves and the certificate they present must be trusted. The -host name in each certificate is used to identify the different -components. The following host names have a special meaning: +The data exposed via the [metrics +endpoint](/docs/install.md#metrics-support) is not considered +confidential and therefore offered without access control via +HTTP. This also simplifies scraping that data with tools like +Prometheus. -- `pmem-registry` is used by the [RegistryServer](#node-registry-server). -- `pmem-node-controller` is used by [NodeControllerServers](#node-controller-server) +The communication between Kubernetes and the scheduler extender +webhook is protected by TLS because this is encouraged and supported +by Kubernetes. But as the webhook only exposes information that is +already available, it accepts all incoming connection without +checking the client certificate. The [`test/setup-ca.sh`](/test/setup-ca.sh) script shows how to generate self-signed certificates. The test cluster is set @@ -297,19 +242,10 @@ created for it on that node. When the application stops, the volume is deleted. The volume cannot be shared with other applications. Data on this volume is retained only while the application runs. -* **Cache volumes** -Volumes are pre-created on a certain set of nodes, each with its own -local data. Applications are started on those nodes and then get to -use the volume on their node. Data persists across application -restarts. This is useful when the data is only cached information that -can be discarded and reconstructed at any time *and* the application -can reuse existing local data when restarting. - Volume | Kubernetes | PMEM-CSI | Limitations --- | --- | --- | --- Persistent | supported | supported | topology aware scheduling1 Ephemeral | supported2 | supported | resource constraints3 -Cache | supported | supported | topology aware scheduling1 1 [Topology aware scheduling](https://github.com/kubernetes/enhancements/issues/490) @@ -321,12 +257,16 @@ onto the right node(s). 2 [CSI ephemeral volumes](https://kubernetes.io/docs/concepts/storage/volumes/#csi-ephemeral-volumes) feature support is alpha in Kubernetes v1.15, and beta in v1.16. -3 The upstream design for ephemeral volumes currently does -not take [resource +3 The upstream design for CSI ephemeral volumes does not +take [resource constraints](https://github.com/kubernetes/enhancements/pull/716#discussion_r250536632) into account. If an application gets scheduled onto a node and then creating the ephemeral volume on that node fails, the application on -the node cannot start until resources become available. +the node cannot start until resources become available. This will be +solved with [generic ephemeral +volumes](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) +which are an alpha feature in Kubernetes 1.19 and supported by +PMEM-CSI because they use the normal volume provisioning process. See [exposing persistent and cache volumes](install.md#expose-persistent-and-cache-volumes-to-applications) for configuration information. @@ -336,30 +276,37 @@ PMEM-CSI implements the CSI `GetCapacity` call, but Kubernetes currently doesn't call that and schedules pods onto nodes without being aware of available storage capacity on the nodes. The effect is that pods using volumes with late binding may get tentatively assigned -to a node and then get stuck because that decision is not reconsidered -when the volume cannot be created there ([a -bug](https://github.com/kubernetes/kubernetes/issues/72031)). Even if -that decision is reconsidered, the same node may get selected again -because Kubernetes does not get informed about the insufficient -storage. Pods with ephemeral inline volumes always get stuck because -the decision to use the node [is final](https://github.com/kubernetes-sigs/descheduler/issues/62). - -Work is [under -way](https://github.com/kubernetes/enhancements/pull/1353) to enhance -scheduling in Kubernetes. In the meantime, PMEM-CSI provides two components -that help with pod scheduling: +to a node and then may have to be rescheduled repeatedly until by +chance they land on a node with enough capacity. Pods using multiple +volumes with immediate binding may be unable to run permanently if +those volumes were created on different nodes. + +[Storage capacity +tracking](https://kubernetes.io/docs/concepts/storage/storage-capacity/) +was added as alpha feature in Kubernetes 1.19 to enhance support for +pod scheduling with late binding of volumes. + +Until that feature becomes generally available, PMEM-CSI provides two +components that help with pod scheduling: ### Scheduler extender -When a pod requests the special [extended +When a pod requests a special [extended resource](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#extended-resources) -called `pmem-csi.intel.com/scheduler`, the Kubernetes scheduler calls +, the Kubernetes scheduler calls a [scheduler extender](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/scheduler_extender.md) provided by PMEM-CSI with a list of nodes that a pod might run -on. This extender is implemented in the master controller and thus can -connect to the controller on each of these nodes to check for -capacity. PMEM-CSI then filters out all nodes which currently do not +on. + +The name of that special resource is `/scheduler`, +i.e. `pmem-csi.intel.com/scheduler` when the default PMEM-CSI driver +name is used. It is possible to configure one extender per PMEM-CSI +deployment because each deployment has its own unique driver name. + +This extender is implemented in the PMEM-CSI controller and retrieves +metrics data from each PMEM-CSI node driver instance to filter out all +nodes which currently do not have enough storage left for the volumes that still need to be created. This considers inline ephemeral volumes and all unbound volumes, regardless whether they use late binding or immediate @@ -387,7 +334,7 @@ See our [implementation](http://github.com/intel/pmem-csi/tree/devel/pkg/schedul ### Pod admission webhook -Having to add `pmem-csi.intel.com/scheduler` manually is not +Having to add the `/scheduler` extended resource manually is not user-friendly. To simplify this, PMEM-CSI provides a [mutating admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) diff --git a/docs/diagrams/pmem-csi-communication-diagram.dia b/docs/diagrams/pmem-csi-communication-diagram.dia index 9c7d293a70..3a9b6b5836 100644 Binary files a/docs/diagrams/pmem-csi-communication-diagram.dia and b/docs/diagrams/pmem-csi-communication-diagram.dia differ diff --git a/docs/images/communication/pmem-csi-communication-diagram.png b/docs/images/communication/pmem-csi-communication-diagram.png index 7eea79c8ed..a9f71217bc 100644 Binary files a/docs/images/communication/pmem-csi-communication-diagram.png and b/docs/images/communication/pmem-csi-communication-diagram.png differ diff --git a/docs/install.md b/docs/install.md index d0e9c0ae6e..7364e85755 100644 --- a/docs/install.md +++ b/docs/install.md @@ -8,7 +8,7 @@ - [Install PMEM-CSI driver](#install-pmem-csi-driver) - [Install using the operator](#install-using-the-operator) - [Install from source](#install-from-source) - - [Expose persistent and cache volumes to applications](#expose-persistent-and-cache-volumes-to-applications) + - [Volume parameters](#volume-parameters) - [Kata Containers support](#kata-containers-support) - [Ephemeral inline volumes](#ephemeral-inline-volumes) - [Raw block volumes](#raw-block-volumes) @@ -58,6 +58,11 @@ installation by name, which indirectly determines the device mode. A storage class also chooses which filesystem is used (xfs or ext4) and enables [Kata Containers support](#kata-containers-support). +It is recommended that storage classes use `allowedTopologies` as in +the [`pmem-storageclass.yaml`](/deploy/kustomize/storageclass/pmem-storageclass.yaml) +to ensure that pods with volumes that use late binding land on a node +where the driver is available. + Optionally, the administrator can enable [the scheduler extensions](#enable-scheduler-extensions) (recommended) and monitoring of resource usage via the [metrics support](#metrics-support). @@ -76,8 +81,8 @@ release notes. When using YAML files, the only reliable way of up- or downgrading is to remove the installation and install anew. -Users can then create PMEM volumes via [volume -claims](#expose-persistent-and-cache-volumes-to-applications) that +Users can then create PMEM volumes via [persistent volume +claims](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) that reference the storage classes or via [ephemeral inline volumes](#ephemeral-inline-volumes). @@ -426,11 +431,15 @@ This can be changed by: - **Set up certificates** -Certificates are required as explained in [Security](design.md#security). -If you are not using the test cluster described in -[Starting and stopping a test cluster](autotest.md#starting-and-stopping-a-test-cluster) -where certificates are created automatically, you must set up certificates manually. -This can be done by running the `./test/setup-ca-kubernetes.sh` script for your cluster. +Certificates are required as explained in [Security](design.md#security) for +running the PMEM-CSI [scheduler extender](design.md#scheduler-extender) and +[webhook](design.md#pod-admission-webhook). If those are not used, then certificate +creation can be skipped. However, the YAML deployment files always create the PMEM-CSI +controller StatefulSet which needs the certificates. Without them, the +`pmem-csi-intel-com-controller-0` pod cannot start, so it is recommended to create +certificates or customize the deployment so that this StatefulSet is not created. + +Certificates can be created by running the `./test/setup-ca-kubernetes.sh` script for your cluster. This script requires "cfssl" tools which can be downloaded. These are the steps for manual set-up of certificates: @@ -521,14 +530,14 @@ for `kubectl kustomize`. For example: ``` console $ kubectl get pods -n pmem-csi NAME READY STATUS RESTARTS AGE -pmem-csi-node-8kmxf 2/2 Running 0 3m15s -pmem-csi-node-bvx7m 2/2 Running 0 3m15s -pmem-csi-controller-0 2/2 Running 0 3m15s -pmem-csi-node-fbmpg 2/2 Running 0 3m15s +pmem-csi-intel-com-node-8kmxf 2/2 Running 0 3m15s +pmem-csi-intel-com-node-bvx7m 2/2 Running 0 3m15s +pmem-csi-intel-com-controller-0 2/2 Running 0 3m15s +pmem-csi-intel-com-node-fbmpg 2/2 Running 0 3m15s ``` -Once after the driver deployed using one of the methods mentioned above -verify that the node labels have been configured correctly +After the driver is deployed using one of the methods mentioned above, +verify that the node labels have been updated correctly: ``` console $ kubectl get nodes --show-labels @@ -608,74 +617,38 @@ $ kubectl exec my-csi-app-2 -- mount |grep /data /dev/ndbus0region0fsdax/5cc9b19e-551d-11e9-a584-928299ac4b17 on /data type xfs (rw,relatime,attr2,dax,inode64,noquota) ``` -#### Expose persistent and cache volumes to applications - -Kubernetes cluster administrators can expose persistent and cache volumes -to applications using -[`StorageClass -Parameters`](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters). An -optional `persistencyModel` parameter differentiates how the -provisioned volume can be used: - -* no `persistencyModel` parameter or `persistencyModel: normal` in `StorageClass` - - A normal Kubernetes persistent volume. In this case - PMEM-CSI creates PMEM volume on a node and the application that - claims to use this volume is supposed to be scheduled onto this node - by Kubernetes. Choosing of node is depend on StorageClass - `volumeBindingMode`. In case of `volumeBindingMode: Immediate` - PMEM-CSI chooses a node randomly, and in case of `volumeBindingMode: - WaitForFirstConsumer` (also known as late binding) Kubernetes first chooses a node for scheduling - the application, and PMEM-CSI creates the volume on that - node. Applications which claim a normal persistent volume has to use - `ReadOnlyOnce` access mode in its `accessModes` list. This - [diagram](/docs/images/sequence/pmem-csi-persistent-sequence-diagram.png) - illustrates how a normal persistent volume gets provisioned in - Kubernetes using PMEM-CSI driver. - -* `persistencyModel: cache` - - Volumes of this type shall be used in combination with - `volumeBindingMode: Immediate`. In this case, PMEM-CSI creates a set - of PMEM volumes each volume on different node. The number of PMEM - volumes to create can be specified by `cacheSize` StorageClass - parameter. Applications which claim a `cache` volume can use - `ReadWriteMany` in its `accessModes` list. Try it out with the provided - [cache storage class](/deploy/common/pmem-storageclass-cache.yaml) - example. This - [diagram](/docs/images/sequence/pmem-csi-cache-sequence-diagram.png) - illustrates how a cache volume gets provisioned in Kubernetes using - PMEM-CSI driver. - -**NOTE**: Cache volumes are associated with a node, not a pod. Multiple -pods using the same cache volume on the same node will not get their -own instance but will end up sharing the same PMEM volume instead. -Application deployment has to consider this and use available Kubernetes -mechanisms like [node -anti-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). -Try it out with provided -[cache application](/deploy/common/pmem-app-cache.yaml) example. - -**WARNING**: late binding (`volumeBindingMode:WaitForFirstConsume`) has some caveats: -* Pod creation may get stuck when there isn't enough capacity left for - the volumes; see the next section for details. -* A node is only chosen the first time a pod starts. After that it will always restart - on that node, because that is where the persistent volume was created. +#### Volume parameters + +Kubernetes cluster administrators can make persistent volumes available +to applications using storage classes, with the behavior of the volumes +determined by [`StorageClass +Parameters`](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters). + +In addition to the normal parameters defined by Kubernetes, PMEM-CSI supports +the following custom parameters in a storage class: + +|key|meaning|optional|values| +|---|-------|--------|-------------| +|`eraseAfter`|Clear all data after use and before
deleting the volume|Yes|`true` (default),
`false`| +|`kataContainers`|Prepare volume for use with DAX in Kata Containers.|Yes|`false/0/f/FALSE` (default),
`true/1/t/TRUE`| + ### Kata Containers support [Kata Containers support](design.md#kata-containers-support) gets enabled via -the `kataContainers` storage class parameter. It accepts the following -values: -* `true/1/t/TRUE` - Create the filesystem inside a partition inside a file, try to mount - on the host through a loop device with `-o dax` but proceed without - `-o dax` when the kernel does not support that. Currently Linux up - to and including 5.4 do not support it. In other words, on the host - such volumes are usable, but only without DAX. Inside Kata - Containers, DAX works. -* `false/0/f/FALSE` (default) - Create the filesystem directly on the volume. +the `kataContainers` storage class parameter. PMEM-CSI then +creates a filesystem inside a partition inside a file. When such a volume +is used inside Kata Containers, the Kata Containers runtime makes sure that +the filesystem is mounted on an emulated NVDIMM device with full DAX support. + +On the host, PMEM-CSI will try to mount through a loop device with `-o +dax` but proceed without `-o dax` when the kernel does not support +that. Currently Linux up to and including 5.4 do not support it and it +is unclear when that support will be added In other words, on the host +such volumes are usable, but only without DAX. + +When disabled, volumes support DAX on the host and are usable without +DAX inside Kata Containers. [Raw block volumes](#raw-block-volumes) are only supported with `kataContainers: false`. Attempts to create them with `kataContainers: @@ -784,41 +757,27 @@ The PMEM-CSI scheduler extender and admission webhook are provided by the PMEM-CSI controller. They need to be enabled during deployment via the `--schedulerListen=[]:` parameter. The listen address is optional and can be left out. The port is where a -HTTPS server will run. It uses the same certificates as the internal -gRPC service. When using the CA creation script described above, they -will contain alternative names for the URLs described in this section -(service names, `127.0.0.1` IP address). - -This parameter can be added to one of the existing deployment files -with `kustomize`. All of the following examples assume that the -current directory contains the `deploy` directory from the PMEM-CSI -repository. It is also possible to reference the base via a -[URL](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md). - -``` ShellSession -$ mkdir my-pmem-csi-deployment - -$ cat >my-pmem-csi-deployment/kustomization.yaml <my-pmem-csi-deployment/scheduler-patch.yaml <my-scheduler/node-port-patch.yaml < 10010 Forwarding from [::1]:10010 -> 10010 ``` @@ -1105,12 +1075,6 @@ $ curl --silent http://localhost:10010/metrics | grep '# ' # HELP build_info A metric with a constant '1' value labeled by version. # TYPE build_info gauge ... -# HELP csi_plugin_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total -# TYPE csi_plugin_operations_seconds histogram -... -# HELP pmem_csi_controller_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total -# TYPE pmem_csi_controller_operations_seconds histogram -... ``` @@ -1209,6 +1173,8 @@ pmem_csi_node_operations_seconds_count{method_name="/csi.v1.Controller/CreateVol ## PMEM-CSI Deployment CRD +TODO update operator + `PmemCSIDeployment` is a cluster-scoped Kubernetes resource in the `pmem-csi.intel.com` API group. It describes how a PMEM-CSI driver instance is to be created. @@ -1263,6 +1229,9 @@ of the API specification. | logLevel | integer | PMEM-CSI driver logging level | 3 | | logFormat | text | log output format | "text" or "json" 3 | | deviceMode | string | Device management mode to use. Supports one of `lvm` or `direct` | `lvm` +| controllerTLSSecret | string | Name of an existing secret in the driver's namespace which contains ca.crt, tls.crt and tls.key data for the scheduler extender and pod mutation webhook. A controller is started if (and only if) this secret is specified. | empty +| mutatePods | Always/Try/Never | Defines how a mutating pod webhook is configured if a controller is started. The field is ignored if the controller is not enabled. "Never" disables pod mutation. "Try" configured it so that pod creation is allowed to proceed even when the webhook fails. "Always" requires that the webhook gets invoked successfully before creating a pod. | Try +| schedulerNodePort | If non-zero, the scheduler service is created as a NodeService with that fixed port number. Otherwise that service is created as a cluster service. The number must be from the range reserved by Kubernetes for node ports. This is useful if the kube-scheduler cannot reach the scheduler extender via a cluster service. | 0 | controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod.
4_Deprecated and only available in `v1alpha1`._ | | nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s).
_4Deprecated and only available in `v1alpha1`._ | | controllerDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller driver container running on master node. Available since `v1beta1`. | diff --git a/go.mod b/go.mod index c8f80c75ec..07a12a49b3 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,10 @@ require ( github.com/emicklei/go-restful v2.9.6+incompatible // indirect github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-logr/logr v0.3.0 - github.com/go-logr/zapr v0.2.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.4.2 - github.com/google/go-cmp v0.5.2 // indirect + github.com/google/go-cmp v0.5.2 github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.1.2 - github.com/googleapis/gnostic v0.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.12.1 // indirect github.com/imdario/mergo v0.3.11 // indirect @@ -29,6 +26,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/operator-framework/operator-lib v0.2.0 github.com/prometheus/client_golang v1.7.1 + github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.14.0 github.com/prometheus/procfs v0.2.0 // indirect github.com/stretchr/testify v1.6.1 @@ -37,35 +35,33 @@ require ( golang.org/x/crypto v0.0.0-20200930160638-afb6bcd081ae // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200930145003-4acb6c075d10 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f - golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect - google.golang.org/appengine v1.6.6 // indirect google.golang.org/genproto v0.0.0-20200930140634-01fc692af84b // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.25.0 // indirect gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124 gopkg.in/yaml.v2 v2.3.0 honnef.co/go/tools v0.0.1-2020.1.4 // indirect - k8s.io/api v0.19.2 - k8s.io/apiextensions-apiserver v0.19.2 - k8s.io/apimachinery v0.19.2 + k8s.io/api v0.20.1 + k8s.io/apiextensions-apiserver v0.20.1 + k8s.io/apimachinery v0.20.1 k8s.io/apiserver v0.19.2 // indirect k8s.io/client-go v12.0.0+incompatible k8s.io/cloud-provider v0.19.2 // indirect - k8s.io/component-base v0.19.2 + k8s.io/component-base v0.20.1 k8s.io/csi-translation-lib v0.19.2 // indirect k8s.io/klog/v2 v2.3.0 k8s.io/kube-openapi v0.0.0-20200923155610-8b5066479488 // indirect k8s.io/kube-scheduler v0.19.2 k8s.io/kubectl v0.19.2 k8s.io/kubernetes v1.19.2 - k8s.io/utils v0.0.0-20200912215256-4140de9c8800 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.12 // indirect - sigs.k8s.io/controller-runtime v0.6.3 + sigs.k8s.io/controller-runtime v0.8.0 + sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.2.0 + sigs.k8s.io/yaml v1.2.0 ) replace ( diff --git a/go.sum b/go.sum index 5904db265a..a3dfda68af 100644 --- a/go.sum +++ b/go.sum @@ -401,6 +401,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -420,6 +421,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -484,6 +486,8 @@ github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPx github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= +github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -538,6 +542,7 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= @@ -547,6 +552,7 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= @@ -600,6 +606,7 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -614,6 +621,7 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4= @@ -737,6 +745,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -746,6 +756,7 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -819,6 +830,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -847,6 +859,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -874,6 +887,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -885,6 +899,7 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -939,9 +954,11 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1124,8 +1141,8 @@ k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -1138,9 +1155,11 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQb sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.12 h1:2XkvsmLI1ZEaTcRt0rwWChxsqgXkZahuf2EV9WUFejc= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.12/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= -sigs.k8s.io/controller-runtime v0.6.3 h1:SBbr+inLPEKhvlJtrvDcwIpm+uhDvp63Bl72xYJtoOE= -sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= +sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc= +sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.2.0 h1:W9pg6FBDxI8A/G0FbDjwKXvIG7ZDfyQODtoGzHFxa60= +sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.2.0/go.mod h1:DhZ52sQMJHW21+JXyA2LRUPRIxKnrNrwh+QFV+2tVA4= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/setup-va.sh b/hack/setup-va.sh index 51bc649928..bd931e4f38 100755 --- a/hack/setup-va.sh +++ b/hack/setup-va.sh @@ -5,7 +5,7 @@ set -ex if ! [ -d _work/autoscaler ]; then - git clone git@github.com:kubernetes/autoscaler.git _work/autoscaler + git clone https://github.com/kubernetes/autoscaler _work/autoscaler fi cd _work/autoscaler git fetch origin diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types.go b/pkg/apis/pmemcsi/v1beta1/deployment_types.go index 9550ee862f..53b5ebde2b 100644 --- a/pkg/apis/pmemcsi/v1beta1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types.go @@ -59,6 +59,20 @@ const ( LogFormatJSON LogFormat = "json" ) +type MutatePods string + +const ( + // MutatePodsAlways enables the mutating pod webhook so that a failure is considered fatal. + MutatePodsAlways MutatePods = "Always" + + // MutatePodsTry enables the mutating pod webhook so that it a pod can be created even + // when the webhook fails. + MutatePodsTry MutatePods = "Try" + + // MutatePodsNever disables the mutating pod webhook. + MutatePodsNever MutatePods = "Never" +) + // +k8s:deepcopy-gen=true // DeploymentSpec defines the desired state of Deployment type DeploymentSpec struct { @@ -78,8 +92,24 @@ type DeploymentSpec struct { NodeRegistrarResources *corev1.ResourceRequirements `json:"nodeRegistrarResources,omitempty"` // NodeDriverResources Compute resources required by driver container running on worker nodes NodeDriverResources *corev1.ResourceRequirements `json:"nodeDriverResources,omitempty"` - // ControllerDriverResources Compute resources required by driver container running on master node + // ControllerDriverResources Compute resources required by central driver container ControllerDriverResources *corev1.ResourceRequirements `json:"controllerDriverResources,omitempty"` + // ControllerTLSSecret is the name of a secret which contains ca.crt, tls.crt and tls.key data + // for the scheduler extender and pod mutation webhook. A controller is started if (and only if) + // this secret is specified. + ControllerTLSSecret string `json:"controllerTLSSecret,omitempty"` + // MutatePod defines how a mutating pod webhook is configured if a controller + // is started. The field is ignored if the controller is not enabled. + // The default is "Try". + // +kubebuilder:validation:Enum=Always;Try;Never + MutatePods MutatePods `json:"mutatePods,omitempty"` + // SchedulerNodePort, if non-zero, ensures that the "scheduler" service + // is created as a NodeService with that fixed port number. Otherwise + // that service is created as a cluster service. The number must be + // from the range reserved by Kubernetes for + // node ports. This is useful if the kube-scheduler cannot reach the scheduler + // extender via a cluster service. + SchedulerNodePort int32 `json:"schedulerNodePort,omitempty"` // DeviceMode to use to manage PMEM devices. // +kubebuilder:validation:Enum=lvm;direct DeviceMode DeviceMode `json:"deviceMode,omitempty"` @@ -89,21 +119,6 @@ type DeploymentSpec struct { // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=text;json LogFormat LogFormat `json:"logFormat,omitempty"` - // RegistryCert encoded certificate signed by a CA for registry server authentication - // If not provided, provisioned one by the operator using self-signed CA - RegistryCert []byte `json:"registryCert,omitempty"` - // RegistryPrivateKey encoded private key used for registry server certificate - // If not provided, provisioned one by the operator - RegistryPrivateKey []byte `json:"registryKey,omitempty"` - // NodeControllerCert encoded certificate signed by a CA for node controller server authentication - // If not provided, provisioned one by the operator using self-signed CA - NodeControllerCert []byte `json:"nodeControllerCert,omitempty"` - // NodeControllerPrivateKey encoded private key used for node controller server certificate - // If not provided, provisioned one by the operator - NodeControllerPrivateKey []byte `json:"nodeControllerKey,omitempty"` - // CACert encoded root certificate of the CA by which the registry and node controller certificates are signed - // If not provided operator uses a self-signed CA certificate - CACert []byte `json:"caCert,omitempty"` // NodeSelector node labels to use for selection of driver node NodeSelector map[string]string `json:"nodeSelector,omitempty"` // PMEMPercentage represents the percentage of space to be used by the driver in each PMEM region @@ -122,11 +137,6 @@ type DeploymentSpec struct { type DeploymentConditionType string const ( - // CertsVerified means the provided deployment secrets are verified and valid for usage - CertsVerified DeploymentConditionType = "CertsVerified" - // CertsReady means secrests/certificates required for running the PMEM-CSI driver - // are ready and the deployment could progress further - CertsReady DeploymentConditionType = "CertsReady" // DriverDeployed means that the all the sub-resources required for the deployment CR // got created DriverDeployed DeploymentConditionType = "DriverDeployed" @@ -248,11 +258,14 @@ const ( // DefaultDriverImage default PMEM-CSI driver docker image DefaultDriverImage = defaultDriverImageName + ":" + defaultDriverImageTag + DefaultMutatePods = MutatePodsTry + // The sidecar versions must be kept in sync with the // deploy/kustomize YAML files! - defaultProvisionerImageName = "k8s.gcr.io/sig-storage/csi-provisioner" - defaultProvisionerImageTag = "v2.0.2" + // TODO: use released image + defaultProvisionerImageName = "gcr.io/k8s-staging-sig-storage/csi-provisioner" + defaultProvisionerImageTag = "canary" // DefaultProvisionerImage default external provisioner image to use DefaultProvisionerImage = defaultProvisionerImageName + ":" + defaultProvisionerImageTag @@ -324,6 +337,16 @@ const ( DeploymentPhaseFailed DeploymentPhase = "Failed" ) +// A TLS secret must contain three data items. +const ( + // TLSSecretCA is the CA bundle. + TLSSecretCA = "ca.crt" + // TLSSecretKey is the secret key to be used by the server. + TLSSecretKey = "tls.key" + // TLSSecretCert is the public key to used by the server. + TLSSecretCert = "tls.crt" +) + func (d *PmemCSIDeployment) SetCondition(t DeploymentConditionType, state corev1.ConditionStatus, reason string) { for _, c := range d.Status.Conditions { if c.Type == t { @@ -366,6 +389,14 @@ func (d *PmemCSIDeployment) EnsureDefaults(operatorImage string) error { return fmt.Errorf("invalid device mode %q", d.Spec.DeviceMode) } + switch d.Spec.MutatePods { + case "": + d.Spec.MutatePods = DefaultMutatePods + case MutatePodsAlways, MutatePodsTry, MutatePodsNever: + default: + return fmt.Errorf("invalid MutatePods value: %s", d.Spec.MutatePods) + } + if d.Spec.Image == "" { // If provided use operatorImage if operatorImage != "" { @@ -497,9 +528,51 @@ func (d *PmemCSIDeployment) MetricsServiceName() string { return d.GetHyphenedName() + "-metrics" } -// ServiceAccountName returns the name of the ServiceAccount -// object used by the deployment -func (d *PmemCSIDeployment) ServiceAccountName() string { +// SchedulerServiceName returns the name of the controller's scheduler +// Service object +func (d *PmemCSIDeployment) SchedulerServiceName() string { + return d.GetHyphenedName() + "-scheduler" +} + +// WebhooksServiceAccountName returns the name of the service account +// used by the StatefulSet with the webhooks. +func (d *PmemCSIDeployment) WebhooksServiceAccountName() string { + return d.GetHyphenedName() + "-webhooks" +} + +// WebhooksRoleName returns the name of the webhooks' +// RBAC Role object name used by the deployment +func (d *PmemCSIDeployment) WebhooksRoleName() string { + return d.GetHyphenedName() + "-webhooks-cfg" +} + +// WebhooksRoleBindingName returns the name of the webhooks' +// RoleBinding object name used by the deployment +func (d *PmemCSIDeployment) WebhooksRoleBindingName() string { + return d.GetHyphenedName() + "-webhooks-role-cfg" +} + +// WebhooksClusterRoleName returns the name of the +// webhooks' ClusterRole object name used by the deployment +func (d *PmemCSIDeployment) WebhooksClusterRoleName() string { + return d.GetHyphenedName() + "-webhooks-runner" +} + +// WebhooksClusterRoleBindingName returns the name of the +// webhooks' ClusterRoleBinding object name used by the deployment +func (d *PmemCSIDeployment) WebhooksClusterRoleBindingName() string { + return d.GetHyphenedName() + "-webhooks-role" +} + +// MutatingWebhookName returns the name of the +// MutatingWebhookConfiguration +func (d *PmemCSIDeployment) MutatingWebhookName() string { + return d.GetHyphenedName() + "-hook" +} + +// NodeServiceAccountName returns the name of the service account +// used by the DaemonSet with the external-provisioner +func (d *PmemCSIDeployment) ProvisionerServiceAccountName() string { return d.GetHyphenedName() + "-controller" } @@ -553,29 +626,3 @@ func (d *PmemCSIDeployment) GetOwnerReference() metav1.OwnerReference { Controller: &isController, } } - -// HaveCertificatesConfigured checks if the configured deployment -// certificate fields are valid. Returns -// - true with nil error if provided certificates are valid. -// - false with nil error if no certificates are provided. -// - false with appropriate error if invalid/incomplete certificates provided. -func (d *PmemCSIDeployment) HaveCertificatesConfigured() (bool, error) { - // Encoded private keys and certificates - caCert := d.Spec.CACert - registryPrKey := d.Spec.RegistryPrivateKey - ncPrKey := d.Spec.NodeControllerPrivateKey - registryCert := d.Spec.RegistryCert - ncCert := d.Spec.NodeControllerCert - - // sanity check - if caCert == nil { - if registryCert != nil || ncCert != nil { - return false, fmt.Errorf("incomplete deployment configuration: missing root CA certificate by which the provided certificates are signed") - } - return false, nil - } else if registryCert == nil || registryPrKey == nil || ncCert == nil || ncPrKey == nil { - return false, fmt.Errorf("incomplete deployment configuration: certificates and corresponding private keys must be provided") - } - - return true, nil -} diff --git a/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go index 7d40cf8e3a..03ca1aeacb 100644 --- a/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go @@ -48,31 +48,6 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } - if in.RegistryCert != nil { - in, out := &in.RegistryCert, &out.RegistryCert - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.RegistryPrivateKey != nil { - in, out := &in.RegistryPrivateKey, &out.RegistryPrivateKey - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.NodeControllerCert != nil { - in, out := &in.NodeControllerCert, &out.NodeControllerCert - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.NodeControllerPrivateKey != nil { - in, out := &in.NodeControllerPrivateKey, &out.NodeControllerPrivateKey - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.CACert != nil { - in, out := &in.CACert, &out.CACert - *out = make([]byte, len(*in)) - copy(*out, *in) - } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index 81a61222e7..ebcfe3fe85 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -15,6 +15,7 @@ import ( "github.com/intel/pmem-csi/deploy" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" + "github.com/intel/pmem-csi/pkg/types" "github.com/intel/pmem-csi/pkg/version" corev1 "k8s.io/api/core/v1" @@ -24,11 +25,12 @@ import ( // LoadObjects reads all objects stored in a pmem-csi.yaml reference file. func LoadObjects(kubernetes version.Version, deviceMode api.DeviceMode) ([]unstructured.Unstructured, error) { - return loadObjects(kubernetes, deviceMode, nil, nil) + return loadYAML(yamlPath(kubernetes, deviceMode), nil, nil, nil) } var pmemImage = regexp.MustCompile(`image: intel/pmem-csi-driver(-test)?:\S+`) -var nameRegex = regexp.MustCompile(`(name|app|secretName|serviceName|serviceAccountName): pmem-csi`) +var nameRegex = regexp.MustCompile(`(name|secretName|serviceName|serviceAccountName): pmem-csi-intel-com`) +var driverNameRegex = regexp.MustCompile(`(?m)(name|app\.kubernetes.io/instance): pmem-csi.intel.com$`) // LoadAndCustomizeObjects reads all objects stored in a pmem-csi.yaml reference file // and updates them on-the-fly according to the deployment spec, namespace and name. @@ -39,9 +41,10 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo // our deployments. But because we controll the input, we can do some // things like renaming with a simple text search/replace. patchYAML := func(yaml *[]byte) { - // This renames the objects. A hyphen is used instead of a dot, - // except for CSIDriver which needs the exact name. + // This renames the objects and labels. A hyphen is used instead of a dot, + // except for CSIDriver and instance label which need the exact name. *yaml = nameRegex.ReplaceAll(*yaml, []byte("$1: "+deployment.GetHyphenedName())) + *yaml = driverNameRegex.ReplaceAll(*yaml, []byte("$1: "+deployment.Name)) // Update the driver name inside the state and socket dir. *yaml = bytes.ReplaceAll(*yaml, []byte("path: /var/lib/pmem-csi.intel.com"), []byte("path: /var/lib/"+deployment.Name)) @@ -75,12 +78,30 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo []byte(fmt.Sprintf("-logging-format=%s", deployment.Spec.LogFormat))) } + nodeSelector := types.NodeSelector(deployment.Spec.NodeSelector) + *yaml = bytes.ReplaceAll(*yaml, + []byte(`-nodeSelector={"storage":"pmem"}`), + []byte("-nodeSelector="+nodeSelector.String())) + *yaml = pmemImage.ReplaceAll(*yaml, []byte("image: "+deployment.Spec.Image)) } + enabled := func(obj *unstructured.Unstructured) bool { + // The controller is always enabled, but the mutating webhook depends on the spec. + switch obj.GetKind() + "/" + obj.GetName() { + case "MutatingWebhookConfiguration/" + deployment.MutatingWebhookName(): + return deployment.Spec.ControllerTLSSecret != "" && deployment.Spec.MutatePods != api.MutatePodsNever + default: + return true + } + } + patchUnstructured := func(obj *unstructured.Unstructured) { if deployment.Spec.Labels != nil { labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } for key, value := range deployment.Spec.Labels { labels[key] = value } @@ -88,12 +109,9 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo } switch obj.GetKind() { - case "CSIDriver": - obj.SetName(deployment.GetName()) case "StatefulSet": resources := map[string]*corev1.ResourceRequirements{ - "pmem-driver": deployment.Spec.ControllerDriverResources, - "external-provisioner": deployment.Spec.ProvisionerResources, + "pmem-driver": deployment.Spec.ControllerDriverResources, } if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic @@ -101,8 +119,9 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo } case "DaemonSet": resources := map[string]*corev1.ResourceRequirements{ - "pmem-driver": deployment.Spec.NodeDriverResources, - "driver-registrar": deployment.Spec.NodeRegistrarResources, + "pmem-driver": deployment.Spec.NodeDriverResources, + "external-provisioner": deployment.Spec.ProvisionerResources, + "driver-registrar": deployment.Spec.NodeRegistrarResources, } if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic @@ -118,10 +137,46 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo } spec["nodeSelector"] = selector } + case "MutatingWebhookConfiguration": + webhooks := obj.Object["webhooks"].([]interface{}) + failurePolicy := "Ignore" + if deployment.Spec.MutatePods == api.MutatePodsAlways { + failurePolicy = "Fail" + } + webhooks[0].(map[string]interface{})["failurePolicy"] = failurePolicy + case "Service": + switch obj.GetName() { + case deployment.SchedulerServiceName(): + if deployment.Spec.SchedulerNodePort != 0 { + spec := obj.Object["spec"].(map[string]interface{}) + spec["type"] = "NodePort" + ports := spec["ports"].([]interface{}) + ports[0].(map[string]interface{})["nodePort"] = deployment.Spec.SchedulerNodePort + } + } } } - return loadObjects(kubernetes, deviceMode, patchYAML, patchUnstructured) + objects, err := loadYAML(yamlPath(kubernetes, deviceMode), patchYAML, enabled, patchUnstructured) + if err != nil { + return nil, err + } + + scheduler, err := loadYAML("deploy/kustomize/scheduler/scheduler-service.yaml", patchYAML, enabled, patchUnstructured) + if err != nil { + return nil, err + } + objects = append(objects, scheduler...) + + if deployment.Spec.MutatePods != api.MutatePodsNever { + webhook, err := loadYAML("deploy/kustomize/webhook/webhook.yaml", patchYAML, enabled, patchUnstructured) + if err != nil { + return nil, err + } + objects = append(objects, webhook...) + } + + return objects, nil } func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDeployment, resources map[string]*corev1.ResourceRequirements) error { @@ -134,6 +189,10 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl spec := template["spec"].(map[string]interface{}) metadata := template["metadata"].(map[string]interface{}) + // isController := strings.Contains(obj.Object["metadata"].(map[string]interface{})["name"].(string), "controller") + isController := strings.Contains(obj.GetName(), "controller") + stripTLS := isController && deployment.Spec.ControllerTLSSecret == "" + if deployment.Spec.Labels != nil { labels := metadata["labels"] var labelsMap map[string]interface{} @@ -161,6 +220,10 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl return obj, nil } + if stripTLS { + spec["volumes"] = nil + } + containers := spec["containers"].([]interface{}) for _, container := range containers { container := container.(map[string]interface{}) @@ -171,6 +234,23 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl } container["resources"] = obj + if stripTLS && container["name"].(string) == "pmem-driver" { + container["volumeMounts"] = nil + var command []interface{} + for _, arg := range container["command"].([]interface{}) { + switch arg.(string) { + case "-caFile=/certs/ca.crt", + "-certFile=/certs/tls.crt", + "-keyFile=/certs/tls.key", + "-schedulerListen=:8000": + // remove these parameters + default: + command = append(command, arg) + } + } + container["command"] = command + } + // Override driver name in env var. env := container["env"] if env != nil { @@ -205,14 +285,27 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl } } + if deployment.Spec.ControllerTLSSecret != "" { + volumes := spec["volumes"].([]interface{}) + for _, volume := range volumes { + volume := volume.(map[string]interface{}) + volumeName := volume["name"].(string) + if volumeName == "webhook-cert" { + volume["secret"].(map[string]interface{})["secretName"] = deployment.Spec.ControllerTLSSecret + } + } + } return nil } -func loadObjects(kubernetes version.Version, deviceMode api.DeviceMode, +func yamlPath(kubernetes version.Version, deviceMode api.DeviceMode) string { + return fmt.Sprintf("deploy/kubernetes-%s/%s/pmem-csi.yaml", kubernetes, deviceMode) +} + +func loadYAML(path string, patchYAML func(yaml *[]byte), + enabled func(obj *unstructured.Unstructured) bool, patchUnstructured func(obj *unstructured.Unstructured)) ([]unstructured.Unstructured, error) { - path := fmt.Sprintf("deploy/kubernetes-%s/%s/pmem-csi.yaml", kubernetes, deviceMode) - // We load the builtin yaml files. yaml, err := deploy.Asset(path) if err != nil { @@ -238,6 +331,9 @@ func loadObjects(kubernetes version.Version, deviceMode api.DeviceMode, if err != nil { return nil, fmt.Errorf("decode item %q from file %q: %v", item, path, err) } + if enabled != nil && !enabled(&obj) { + continue + } if patchUnstructured != nil { patchUnstructured(&obj) } diff --git a/pkg/k8sutil/client.go b/pkg/k8sutil/client.go index 1550248962..4a70709f9d 100644 --- a/pkg/k8sutil/client.go +++ b/pkg/k8sutil/client.go @@ -8,6 +8,7 @@ package k8sutil import ( "fmt" + "os" "regexp" "strconv" @@ -15,16 +16,26 @@ import ( "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" ) -// NewInClusterClient connects code that runs inside a Kubernetes pod to the -// API server. -func NewInClusterClient() (kubernetes.Interface, error) { - config, err := rest.InClusterConfig() +// NewClient connects to an API server either through KUBECONFIG (if set) or +// through the in-cluster env variables. +func NewClient(qps float64, burst int) (kubernetes.Interface, error) { + var config *rest.Config + var err error + + if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + config, err = clientcmd.BuildConfigFromFlags("" /* master */, kubeconfig) + } else { + config, err = rest.InClusterConfig() + } if err != nil { - return nil, fmt.Errorf("build in-cluster Kubernetes client configuration: %v", err) + return nil, fmt.Errorf("create Kubernetes REST config: %v", err) } + config.QPS = float32(qps) + config.Burst = burst client, err := kubernetes.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create Kubernetes client: %v", err) diff --git a/pkg/pmem-csi-driver/controllerserver-master.go b/pkg/pmem-csi-driver/controllerserver-master.go deleted file mode 100644 index a388cf072b..0000000000 --- a/pkg/pmem-csi-driver/controllerserver-master.go +++ /dev/null @@ -1,526 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package pmemcsidriver - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "math" - "strconv" - "sync" - - "github.com/container-storage-interface/spec/lib/go/csi" - "golang.org/x/net/context" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/klog/v2" - "k8s.io/utils/keymutex" - - grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" - "github.com/intel/pmem-csi/pkg/pmem-csi-driver/parameters" - "github.com/intel/pmem-csi/pkg/registryserver" -) - -//VolumeStatus type representation for volume status -type VolumeStatus int - -const ( - //Created volume created - Created VolumeStatus = iota + 1 - //Deleted volume deleted - Deleted -) - -type volume struct { - // VolumeID published to outside world - id string - // Name of volume - name string - // Size of the volume - size int64 - // ID of nodes where the volume provisioned/attached - // It would be one if simple volume, else would be more than one for "cached" volume - nodeIDs map[string]VolumeStatus -} - -type masterController struct { - *DefaultControllerServer - rs *registryserver.RegistryServer - volumes map[string]*volume //map of reqID:Volume - mutex sync.Mutex // mutex for Volumes -} - -var _ csi.ControllerServer = &masterController{} -var _ grpcserver.Service = &masterController{} -var _ registryserver.RegistryListener = &masterController{} -var volumeMutex = keymutex.NewHashed(-1) - -func GenerateVolumeID(caller string, name string) string { - // VolumeID is hashed from Volume Name. - // Hashing guarantees same ID for repeated requests. - // Why do we generate new VolumeID via hashing? - // We can not use Name directly as VolumeID because of at least 2 reasons: - // 1. allowed max. Name length by CSI spec is 128 chars, which does not fit - // into LVM volume name (for that we use VolumeID), where groupname+volumename - // must fit into 126 chars. - // Ndctl namespace name is even shorter, it can be 63 chars long. - // 2. CSI spec. allows characters in Name that are not allowed in LVM names. - hasher := sha256.New224() - hasher.Write([]byte(name)) - hash := hex.EncodeToString(hasher.Sum(nil)) - // Use first characters of Name in VolumeID to help humans. - // This also lowers collision probability even more, as an attacker - // attempting to cause VolumeID collision, has to find another Name - // producing same sha-224 hash, while also having common first N chars. - use := 6 - if len(name) < 6 { - use = len(name) - } - id := name[0:use] + "-" + hash - klog.V(4).Infof("%s: Create VolumeID:%s based on name:%s", caller, id, name) - return id -} - -func NewMasterControllerServer(rs *registryserver.RegistryServer) *masterController { - serverCaps := []csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_LIST_VOLUMES, - csi.ControllerServiceCapability_RPC_GET_CAPACITY, - } - cs := &masterController{ - DefaultControllerServer: NewDefaultControllerServer(serverCaps), - rs: rs, - volumes: map[string]*volume{}, - } - - rs.AddListener(cs) - - return cs -} - -func (cs *masterController) RegisterService(rpcServer *grpc.Server) { - csi.RegisterControllerServer(rpcServer, cs) -} - -// OnNodeAdded retrieves the existing volumes at recently added Node. -// It uses ControllerServer.ListVolume() CSI call to retrieve volumes. -func (cs *masterController) OnNodeAdded(ctx context.Context, node *registryserver.NodeInfo) error { - conn, err := cs.rs.ConnectToNodeController(node.NodeID) - if err != nil { - return fmt.Errorf("Connection failure on given endpoint %s : %s", node.Endpoint, err.Error()) - } - defer conn.Close() - - csiClient := csi.NewControllerClient(conn) - resp, err := csiClient.ListVolumes(ctx, &csi.ListVolumesRequest{}) - if err != nil { - return fmt.Errorf("Node failed to report volumes: %s", err.Error()) - } - - klog.V(5).Infof("Found Volumes at %s: %v", node.NodeID, resp.Entries) - - cs.mutex.Lock() - defer cs.mutex.Unlock() - - for _, entry := range resp.Entries { - v := entry.GetVolume() - if v == nil { /* this shouldn't happen */ - continue - } - if vol, ok := cs.volumes[v.VolumeId]; ok && vol != nil { - // This is possibly Cache volume, so just add this node id. - vol.nodeIDs[node.NodeID] = Created - } else { - cs.volumes[v.VolumeId] = &volume{ - id: v.VolumeId, - size: v.CapacityBytes, - name: v.VolumeContext["Name"], - nodeIDs: map[string]VolumeStatus{ - node.NodeID: Created, - }, - } - } - } - - return nil -} - -func (cs *masterController) OnNodeDeleted(ctx context.Context, node *registryserver.NodeInfo) { -} - -func (cs *masterController) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - var vol *volume - chosenNodes := map[string]VolumeStatus{} - - if err := cs.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { - klog.Errorf("invalid create volume req: %v", req) - return nil, err - } - - if req.GetVolumeCapabilities() == nil { - return nil, status.Error(codes.InvalidArgument, "Volume Capabilities missing in request") - } - - if len(req.GetName()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Name missing in request") - } - - asked := req.GetCapacityRange().GetRequiredBytes() - p, err := parameters.Parse(parameters.CreateVolumeOrigin, req.Parameters) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - outTopology := []*csi.Topology{} - klog.V(3).Infof("Controller CreateVolume: Name:%v required_bytes:%v limit_bytes:%v", req.Name, asked, req.GetCapacityRange().GetLimitBytes()) - if vol = cs.getVolumeByName(req.Name); vol != nil { - // Check if the size of existing volume can cover the new request - klog.V(4).Infof("CreateVolume: Vol %s exists, Size: %v", req.Name, vol.size) - if vol.size < asked { - return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Smaller volume with the same name:%s already exists", req.Name)) - } - - chosenNodes = vol.nodeIDs - } else { - volumeID := GenerateVolumeID("Controller CreateVolume", req.Name) - // Check do we have entry with newly generated VolumeID already - if vol := cs.getVolumeByID(volumeID); vol != nil { - // if we have, that has to be VolumeID collision, because above we checked - // that we don't have entry with such Name. VolumeID collision is very-very - // unlikely so we should not get here in any near future, if otherwise state is good. - klog.V(3).Infof("Controller CreateVolume: VolumeID:%s collision: existing name:%s new name:%s", - volumeID, vol.name, req.Name) - return nil, status.Error(codes.Internal, "VolumeID/hash collision, can not create unique Volume ID") - } - inTopology := []*csi.Topology{} - - if reqTop := req.GetAccessibilityRequirements(); reqTop != nil { - inTopology = reqTop.Preferred - if inTopology == nil { - inTopology = reqTop.Requisite - } - } - - if len(inTopology) == 0 { - // No topology provided, so we are free to choose from all available - // nodes - for node := range cs.rs.NodeClients() { - inTopology = append(inTopology, &csi.Topology{ - Segments: map[string]string{ - DriverTopologyKey: node, - }, - }) - } - } - - // Sent required parameters (and only those) plus the volume ID chosen by us. - p.VolumeID = &volumeID - req.Parameters = p.ToContext() - numVolumes := uint(1) - if p.GetPersistency() == parameters.PersistencyCache { - numVolumes = p.GetCacheSize() - } - for _, top := range inTopology { - if numVolumes == 0 { - break - } - node := top.Segments[DriverTopologyKey] - conn, err := cs.rs.ConnectToNodeController(node) - if err != nil { - klog.Warningf("failed to connect to %s: %s", node, err.Error()) - continue - } - - defer conn.Close() - - csiClient := csi.NewControllerClient(conn) - - if _, err := csiClient.CreateVolume(ctx, req); err != nil { - klog.Warningf("failed to create volume name:%s id:%s on %s: %s", node, req.Name, volumeID, err.Error()) - continue - } - numVolumes = numVolumes - 1 - chosenNodes[node] = Created - } - - if len(chosenNodes) == 0 { - return nil, status.Error(codes.ResourceExhausted, fmt.Sprintf("No node found with %v capacity", asked)) - } - - klog.V(3).Infof("Chosen nodes: %v", chosenNodes) - - vol = &volume{ - id: volumeID, - name: req.Name, - size: asked, - nodeIDs: chosenNodes, - } - cs.mutex.Lock() - defer cs.mutex.Unlock() - cs.volumes[volumeID] = vol - klog.V(3).Infof("Controller CreateVolume: Record new volume as %v", *vol) - } - - for node := range chosenNodes { - outTopology = append(outTopology, &csi.Topology{ - Segments: map[string]string{ - DriverTopologyKey: node, - }, - }) - } - - // Volume ID and name are not the same. Store the original - // name in the volume context for logging purposes. - name := req.GetName() - p.Name = &name - - return &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: vol.id, - CapacityBytes: asked, - AccessibleTopology: outTopology, - VolumeContext: p.ToContext(), - }, - }, nil -} - -func (cs *masterController) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - if err := cs.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { - klog.Errorf("invalid delete volume req: %v", req) - return nil, err - } - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - - // Serialize by VolumeId - volumeMutex.LockKey(req.VolumeId) - defer volumeMutex.UnlockKey(req.VolumeId) //nolint: errcheck - - klog.V(4).Infof("DeleteVolume: requested volumeID: %v", req.GetVolumeId()) - if vol := cs.getVolumeByID(req.GetVolumeId()); vol != nil { - for node := range vol.nodeIDs { - conn, err := cs.rs.ConnectToNodeController(node) - if err != nil { - return nil, status.Error(codes.Internal, "Failed to connect to node "+node+": "+err.Error()) - } - defer conn.Close() // nolint:errcheck - klog.V(4).Infof("Asking node %s to delete volume name:%s id:%s", node, vol.name, vol.id) - if _, err := csi.NewControllerClient(conn).DeleteVolume(ctx, req); err != nil { - return nil, err - } - } - cs.mutex.Lock() - defer cs.mutex.Unlock() - delete(cs.volumes, vol.id) - klog.V(4).Infof("Controller DeleteVolume: volume name:%s id:%s deleted", vol.name, vol.id) - } else { - klog.Warningf("Volume %s not created by this controller", req.GetVolumeId()) - } - - return &csi.DeleteVolumeResponse{}, nil -} - -func (cs *masterController) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - cs.mutex.Lock() - defer cs.mutex.Unlock() - - _, found := cs.volumes[req.VolumeId] - if !found { - return nil, status.Error(codes.NotFound, "No volume found with id "+req.VolumeId) - } - - if req.GetVolumeCapabilities() == nil { - return nil, status.Error(codes.InvalidArgument, "Volume capabilities missing in request") - } - - for _, cap := range req.VolumeCapabilities { - if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - return &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: nil, - Message: "Driver does not support '" + cap.AccessMode.Mode.String() + "' mode", - }, nil - } - } - - /* - * FIXME(avalluri): Need to validate other capabilities against the existing volume - */ - return &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeCapabilities: req.VolumeCapabilities, - VolumeContext: req.GetVolumeContext(), - }, - }, nil -} - -func (cs *masterController) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { - klog.V(5).Info("ListVolumes") - if err := cs.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_VOLUMES); err != nil { - klog.Errorf("invalid list volumes req: %v", req) - return nil, err - } - - cs.mutex.Lock() - defer cs.mutex.Unlock() - - // Copy from map into array for pagination. - vols := make([]*volume, 0, len(cs.volumes)) - for _, vol := range cs.volumes { - vols = append(vols, vol) - } - - // Code originally copied from https://github.com/kubernetes-csi/csi-test/blob/f14e3d32125274e0c3a3a5df380e1f89ff7c132b/mock/service/controller.go#L309-L365 - - var ( - ulenVols = int32(len(vols)) - maxEntries = req.MaxEntries - startingToken int32 - ) - - if v := req.StartingToken; v != "" { - i, err := strconv.ParseUint(v, 10, 32) - if err != nil { - return nil, status.Errorf( - codes.Aborted, - "startingToken=%d !< int32=%d", - startingToken, math.MaxUint32) - } - startingToken = int32(i) - } - - if startingToken > ulenVols { - return nil, status.Errorf( - codes.Aborted, - "startingToken=%d > len(vols)=%d", - startingToken, ulenVols) - } - - // Discern the number of remaining entries. - rem := ulenVols - startingToken - - // If maxEntries is 0 or greater than the number of remaining entries then - // set maxEntries to the number of remaining entries. - if maxEntries == 0 || maxEntries > rem { - maxEntries = rem - } - - var ( - i int - j = startingToken - entries = make( - []*csi.ListVolumesResponse_Entry, - maxEntries) - ) - - for i = 0; i < len(entries); i++ { - vol := vols[j] - entries[i] = &csi.ListVolumesResponse_Entry{ - Volume: &csi.Volume{ - VolumeId: vol.id, - CapacityBytes: vol.size, - }, - } - j++ - } - - var nextToken string - if n := startingToken + int32(i); n < ulenVols { - nextToken = fmt.Sprintf("%d", n) - } - - return &csi.ListVolumesResponse{ - Entries: entries, - NextToken: nextToken, - }, nil -} - -func (cs *masterController) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - var capacity int64 - if err := cs.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_GET_CAPACITY); err != nil { - return nil, err - } - - if top := req.GetAccessibleTopology(); top != nil { - node, err := cs.rs.GetNodeController(top.Segments[DriverTopologyKey]) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) - } - cap, err := cs.getNodeCapacity(ctx, node, req) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get node %s capacity: %s", node.NodeID, err.Error()) - } - capacity = cap - } else { - for _, node := range cs.rs.NodeClients() { - cap, err := cs.getNodeCapacity(ctx, *node, req) - if err != nil { - klog.Warningf("Error while fetching '%s' node capacity: %s", node.NodeID, err.Error()) - continue - } - capacity += cap - } - } - - return &csi.GetCapacityResponse{ - AvailableCapacity: capacity, - }, nil -} - -func (cs *masterController) getNodeCapacity(ctx context.Context, node registryserver.NodeInfo, req *csi.GetCapacityRequest) (int64, error) { - conn, err := cs.rs.ConnectToNodeController(node.NodeID) - if err != nil { - return 0, fmt.Errorf("failed to connect to node %s: %s", node.NodeID, err.Error()) - } - - defer conn.Close() - - csiClient := csi.NewControllerClient(conn) - resp, err := csiClient.GetCapacity(ctx, req) - if err != nil { - return 0, fmt.Errorf("Error while fetching '%s' node capacity: %s", node.NodeID, err.Error()) - } - - return resp.AvailableCapacity, nil -} - -func (cs *masterController) getVolumeByID(volumeID string) *volume { - cs.mutex.Lock() - defer cs.mutex.Unlock() - if vol, ok := cs.volumes[volumeID]; ok { - return vol - } - return nil -} - -func (cs *masterController) getVolumeByName(Name string) *volume { - cs.mutex.Lock() - defer cs.mutex.Unlock() - for _, vol := range cs.volumes { - if vol.name == Name { - return vol - } - } - return nil -} - -func (cs *masterController) ControllerExpandVolume(context.Context, *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *masterController) ControllerGetVolume(context.Context, *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} diff --git a/pkg/pmem-csi-driver/controllerserver-node.go b/pkg/pmem-csi-driver/controllerserver-node.go index bcd3071e99..1114bcec6a 100644 --- a/pkg/pmem-csi-driver/controllerserver-node.go +++ b/pkg/pmem-csi-driver/controllerserver-node.go @@ -7,8 +7,12 @@ SPDX-License-Identifier: Apache-2.0 package pmemcsidriver import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "math" + "strconv" "sync" "golang.org/x/net/context" @@ -215,19 +219,16 @@ func (cs *nodeControllerServer) createVolumeInternal(ctx context.Context, } klog.V(4).Infof("Node CreateVolume: Name:%q req.Required:%v req.Limit:%v", volumeName, asked, capacity.GetLimitBytes()) - volumeID = p.GetVolumeID() - if volumeID == "" { - volumeID = GenerateVolumeID("Node CreateVolume", volumeName) - // Check do we have entry with newly generated VolumeID already - if vol := cs.getVolumeByID(volumeID); vol != nil { - // if we have, that has to be VolumeID collision, because above we checked - // that we don't have entry with such Name. VolumeID collision is very-very - // unlikely so we should not get here in any near future, if otherwise state is good. - klog.V(3).Infof("Controller CreateVolume: VolumeID:%s collision: existing name:%s new name:%s", - volumeID, vol.Params[parameters.Name], volumeName) - statusErr = status.Error(codes.Internal, "VolumeID/hash collision, can not create unique Volume") - return - } + volumeID = generateVolumeID("Node CreateVolume", volumeName) + // Check do we have entry with newly generated VolumeID already + if vol := cs.getVolumeByID(volumeID); vol != nil { + // if we have, that has to be VolumeID collision, because above we checked + // that we don't have entry with such Name. VolumeID collision is very-very + // unlikely so we should not get here in any near future, if otherwise state is good. + klog.V(3).Infof("Controller CreateVolume: VolumeID:%s collision: existing name:%s new name:%s", + volumeID, vol.Params[parameters.Name], volumeName) + statusErr = status.Error(codes.Internal, "VolumeID/hash collision, cannot create unique Volume") + return } // Set which device manager was used to create the volume @@ -376,22 +377,78 @@ func (cs *nodeControllerServer) ListVolumes(ctx context.Context, req *csi.ListVo klog.Errorf("invalid list volumes req: %v", req) return nil, err } + cs.mutex.Lock() defer cs.mutex.Unlock() - // List namespaces - var entries []*csi.ListVolumesResponse_Entry + + // Copy from map into array for pagination. + vols := make([]*nodeVolume, 0, len(cs.pmemVolumes)) for _, vol := range cs.pmemVolumes { - entries = append(entries, &csi.ListVolumesResponse_Entry{ + vols = append(vols, vol) + } + + // Code originally copied from https://github.com/kubernetes-csi/csi-test/blob/f14e3d32125274e0c3a3a5df380e1f89ff7c132b/mock/service/controller.go#L309-L365 + + var ( + ulenVols = int32(len(vols)) + maxEntries = req.MaxEntries + startingToken int32 + ) + + if v := req.StartingToken; v != "" { + i, err := strconv.ParseUint(v, 10, 32) + if err != nil { + return nil, status.Errorf( + codes.Aborted, + "startingToken=%d !< int32=%d", + startingToken, math.MaxUint32) + } + startingToken = int32(i) + } + + if startingToken > ulenVols { + return nil, status.Errorf( + codes.Aborted, + "startingToken=%d > len(vols)=%d", + startingToken, ulenVols) + } + + // Discern the number of remaining entries. + rem := ulenVols - startingToken + + // If maxEntries is 0 or greater than the number of remaining entries then + // set maxEntries to the number of remaining entries. + if maxEntries == 0 || maxEntries > rem { + maxEntries = rem + } + + var ( + i int + j = startingToken + entries = make( + []*csi.ListVolumesResponse_Entry, + maxEntries) + ) + + for i = 0; i < len(entries); i++ { + vol := vols[j] + entries[i] = &csi.ListVolumesResponse_Entry{ Volume: &csi.Volume{ VolumeId: vol.ID, CapacityBytes: vol.Size, - VolumeContext: vol.Params, }, - }) + } + j++ + } + + var nextToken string + if n := startingToken + int32(i); n < ulenVols { + nextToken = fmt.Sprintf("%d", n) } return &csi.ListVolumesResponse{ - Entries: entries, + Entries: entries, + NextToken: nextToken, }, nil } @@ -436,3 +493,29 @@ func (cs *nodeControllerServer) ControllerExpandVolume(context.Context, *csi.Con func (cs *nodeControllerServer) ControllerGetVolume(context.Context, *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } + +func generateVolumeID(caller string, name string) string { + // VolumeID is hashed from Volume Name. + // Hashing guarantees same ID for repeated requests. + // Why do we generate new VolumeID via hashing? + // We can not use Name directly as VolumeID because of at least 2 reasons: + // 1. allowed max. Name length by CSI spec is 128 chars, which does not fit + // into LVM volume name (for that we use VolumeID), where groupname+volumename + // must fit into 126 chars. + // Ndctl namespace name is even shorter, it can be 63 chars long. + // 2. CSI spec. allows characters in Name that are not allowed in LVM names. + hasher := sha256.New224() + hasher.Write([]byte(name)) + hash := hex.EncodeToString(hasher.Sum(nil)) + // Use first characters of Name in VolumeID to help humans. + // This also lowers collision probability even more, as an attacker + // attempting to cause VolumeID collision, has to find another Name + // producing same sha-224 hash, while also having common first N chars. + use := 6 + if len(name) < 6 { + use = len(name) + } + id := name[0:use] + "-" + hash + klog.V(4).Infof("%s: Create VolumeID:%s based on name:%s", caller, id, name) + return id +} diff --git a/pkg/pmem-csi-driver/identityserver.go b/pkg/pmem-csi-driver/identityserver.go index 1fc86b366d..33aebbbb66 100644 --- a/pkg/pmem-csi-driver/identityserver.go +++ b/pkg/pmem-csi-driver/identityserver.go @@ -21,7 +21,7 @@ type identityServer struct { var _ grpcserver.Service = &identityServer{} -func NewIdentityServer(name, version string) (*identityServer, error) { +func NewIdentityServer(name, version string) *identityServer { return &identityServer{ name: name, version: version, @@ -41,7 +41,7 @@ func NewIdentityServer(name, version string) (*identityServer, error) { }, }, }, - }, nil + } } func (ids *identityServer) RegisterService(rpcServer *grpc.Server) { diff --git a/pkg/pmem-csi-driver/main.go b/pkg/pmem-csi-driver/main.go index 25ad9c5cdd..a006c77154 100644 --- a/pkg/pmem-csi-driver/main.go +++ b/pkg/pmem-csi-driver/main.go @@ -15,14 +15,13 @@ import ( "k8s.io/klog/v2" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" - "github.com/intel/pmem-csi/pkg/k8sutil" "github.com/intel/pmem-csi/pkg/logger" pmemcommon "github.com/intel/pmem-csi/pkg/pmem-common" ) var ( config = Config{ - Mode: Controller, + Mode: Node, DeviceManager: api.DeviceModeLVM, } showVersion = flag.Bool("version", false, "Show release version and exit") @@ -35,13 +34,13 @@ func init() { flag.StringVar(&config.DriverName, "drivername", "pmem-csi.intel.com", "name of the driver") flag.StringVar(&config.NodeID, "nodeid", "nodeid", "node id") flag.StringVar(&config.Endpoint, "endpoint", "unix:///tmp/pmem-csi.sock", "PMEM CSI endpoint") - flag.StringVar(&config.RegistryEndpoint, "registryEndpoint", "tcp://pmem-csi-controller:10000", "endpoint for internal registry server (controller listens, node connects)") - flag.Var(&config.Mode, "mode", "driver run mode: controller or node") - flag.StringVar(&config.CAFile, "caFile", "", "Root CA certificate file to use for verifying connections") - flag.StringVar(&config.CertFile, "certFile", "", "SSL certificate file to use for authenticating client connections(RegistryServer/NodeControllerServer)") - flag.StringVar(&config.KeyFile, "keyFile", "", "Private key file associated to certificate") - flag.StringVar(&config.ClientCertFile, "clientCertFile", "", "Client SSL certificate file to use for authenticating peer connections, defaults to 'certFile'") - flag.StringVar(&config.ClientKeyFile, "clientKeyFile", "", "Client private key associated to client certificate, defaults to 'keyFile'") + flag.Var(&config.Mode, "mode", "driver run mode") + flag.StringVar(&config.CAFile, "caFile", "ca.pem", "Root CA certificate file to use for verifying connections") + flag.StringVar(&config.CertFile, "certFile", "pmem-registry.pem", "SSL certificate file to use for authenticating client connections") + flag.StringVar(&config.KeyFile, "keyFile", "pmem-registry-key.pem", "Private key file associated to certificate") + + flag.Float64Var(&config.KubeAPIQPS, "kube-api-qps", 5, "QPS to use while communicating with the Kubernetes apiserver. Defaults to 5.0.") + flag.IntVar(&config.KubeAPIBurst, "kube-api-burst", 10, "Burst to use while communicating with the Kubernetes apiserver. Defaults to 10.") /* metrics options */ flag.StringVar(&config.metricsListen, "metricsListen", "", "listen address (like :8001) for prometheus metrics endpoint, disabled by default") @@ -49,10 +48,9 @@ func init() { /* Controller mode options */ flag.StringVar(&config.schedulerListen, "schedulerListen", "", "controller: listen address (like :8000) for scheduler extender and mutating webhook, disabled by default") + flag.Var(&config.nodeSelector, "nodeSelector", "controller: reschedule PVCs with a selected node where PMEM-CSI is not meant to run because the node does not have these labels (represented as JSON map)") /* Node mode options */ - flag.StringVar(&config.ControllerEndpoint, "controllerEndpoint", "tcp://:10001", "node: internal node controller endpoint") - flag.BoolVar(&config.TestEndpoint, "testEndpoint", false, "node: also expose controller interface via CSI endpoint (for testing only)") flag.Var(&config.DeviceManager, "deviceManager", "node: device manager to use to manage pmem devices, supported types: 'lvm' or 'direct' (= 'ndctl')") flag.StringVar(&config.StateBasePath, "statePath", "", "node: directory path where to persist the state of the driver, defaults to /var/lib/") flag.UintVar(&config.PmemPercentage, "pmemPercentage", 100, "node: percentage of space to be used by the driver in each PMEM region") @@ -70,17 +68,9 @@ func Main() int { klog.V(3).Info("Version: ", version) - if config.schedulerListen != "" { - if config.Mode != Controller { - pmemcommon.ExitError("scheduler listening", errors.New("only supported in the controller")) - return 1 - } - c, err := k8sutil.NewInClusterClient() - if err != nil { - pmemcommon.ExitError("scheduler setup", err) - return 1 - } - config.client = c + if config.schedulerListen != "" && config.Mode != Webhooks { + pmemcommon.ExitError("scheduler listening", errors.New("only supported in the controller")) + return 1 } config.Version = version diff --git a/pkg/pmem-csi-driver/nodeserver.go b/pkg/pmem-csi-driver/nodeserver.go index 34dc2ac429..0f6d445b54 100644 --- a/pkg/pmem-csi-driver/nodeserver.go +++ b/pkg/pmem-csi-driver/nodeserver.go @@ -20,6 +20,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" + "k8s.io/utils/keymutex" "k8s.io/utils/mount" pmemerr "github.com/intel/pmem-csi/pkg/errors" @@ -64,6 +65,7 @@ type nodeServer struct { var _ csi.NodeServer = &nodeServer{} var _ grpcserver.Service = &nodeServer{} +var volumeMutex = keymutex.NewHashed(-1) func NewNodeServer(cs *nodeControllerServer, mountDirectory string) *nodeServer { return &nodeServer{ diff --git a/pkg/pmem-csi-driver/parameters/parameters.go b/pkg/pmem-csi-driver/parameters/parameters.go index e025fb88f4..256f80ff84 100644 --- a/pkg/pmem-csi-driver/parameters/parameters.go +++ b/pkg/pmem-csi-driver/parameters/parameters.go @@ -20,12 +20,10 @@ type Origin int // Beware of API and backwards-compatibility breaking when changing these string constants! const ( - CacheSize = "cacheSize" EraseAfter = "eraseafter" KataContainers = "kataContainers" Name = "name" PersistencyModel = "persistencyModel" - VolumeID = "_id" Size = "size" DeviceMode = "deviceMode" @@ -39,8 +37,7 @@ const ( // Added by https://github.com/kubernetes-csi/external-provisioner/blob/feb67766f5e6af7db5c03ac0f0b16255f696c350/pkg/controller/controller.go#L584 ProvisionerID = "storage.kubernetes.io/csiProvisionerIdentity" - PersistencyNormal Persistency = "normal" // In releases <= 0.6.x this was called "none", but not documented. - PersistencyCache Persistency = "cache" + PersistencyNormal Persistency = "normal" // In releases <= 0.6.x this was called "none", but not documented. PersistencyEphemeral Persistency = "ephemeral" // only used internally //CreateVolumeOrigin is for parameters from the storage class in controller CreateVolume. @@ -59,22 +56,11 @@ const ( var valid = map[Origin][]string{ // Parameters from Kubernetes and users for a persistent volume. CreateVolumeOrigin: []string{ - CacheSize, EraseAfter, KataContainers, PersistencyModel, }, - // These parameters are prepared by the master controller. - CreateVolumeInternalOrigin: []string{ - CacheSize, - EraseAfter, - KataContainers, - PersistencyModel, - - VolumeID, - }, - // Parameters from Kubernetes and users. EphemeralVolumeOrigin: []string{ EraseAfter, @@ -89,7 +75,6 @@ var valid = map[Origin][]string{ // doesn't) and add the volume name for logging purposes. // Kubernetes adds pod info and provisioner ID. PersistentVolumeOrigin: []string{ - CacheSize, EraseAfter, KataContainers, PersistencyModel, @@ -102,7 +87,6 @@ var valid = map[Origin][]string{ // Internally we store everything except the volume ID, // which is handled separately. NodeVolumeOrigin: []string{ - CacheSize, EraseAfter, KataContainers, Name, @@ -117,13 +101,11 @@ var valid = map[Origin][]string{ // The accessor functions always return a value, if unset // the default. type Volume struct { - CacheSize *uint EraseAfter *bool KataContainers *bool Name *string Persistency *Persistency Size *int64 - VolumeID *string DeviceMode *api.DeviceMode } @@ -155,13 +137,10 @@ func Parse(origin Origin, stringmap map[string]string) (Volume, error) { switch key { case Name: result.Name = &value - case VolumeID: - /* volume id provided by master controller (needed for cache volumes) */ - result.VolumeID = &value case PersistencyModel: p := Persistency(value) switch p { - case PersistencyNormal, PersistencyCache: + case PersistencyNormal: result.Persistency = &p case PersistencyEphemeral: if origin != NodeVolumeOrigin { @@ -175,13 +154,6 @@ func Parse(origin Origin, stringmap map[string]string) (Volume, error) { default: return result, fmt.Errorf("parameter %q: unknown value: %q", key, value) } - case CacheSize: - c, err := strconv.ParseUint(value, 10, 32) - if err != nil { - return result, fmt.Errorf("parameter %q: failed to parse %q as uint: %v", key, value, err) - } - u := uint(c) - result.CacheSize = &u case KataContainers: b, err := strconv.ParseBool(value) if err != nil { @@ -225,9 +197,6 @@ func Parse(origin Origin, stringmap map[string]string) (Volume, error) { } // Some sanity checks. - if result.CacheSize != nil && result.GetPersistency() != PersistencyCache { - return result, fmt.Errorf("parameter %q: invalid for %q = %q", CacheSize, PersistencyModel, result.GetPersistency()) - } if origin == EphemeralVolumeOrigin && result.Size == nil { return result, fmt.Errorf("required parameter %q not specified", Size) } @@ -248,9 +217,6 @@ func (v Volume) ToContext() VolumeContext { // Intentionally not stored: // - volumeID - if v.CacheSize != nil { - result[CacheSize] = fmt.Sprintf("%d", *v.CacheSize) - } if v.EraseAfter != nil { result[EraseAfter] = fmt.Sprintf("%v", *v.EraseAfter) } @@ -273,13 +239,6 @@ func (v Volume) ToContext() VolumeContext { return result } -func (v Volume) GetCacheSize() uint { - if v.CacheSize != nil { - return *v.CacheSize - } - return 1 -} - func (v Volume) GetEraseAfter() bool { if v.EraseAfter != nil { return *v.EraseAfter @@ -315,13 +274,6 @@ func (v Volume) GetKataContainers() bool { return false } -func (v Volume) GetVolumeID() string { - if v.VolumeID != nil { - return *v.VolumeID - } - return "" -} - func (v Volume) GetDeviceMode() api.DeviceMode { if v.DeviceMode != nil { return *v.DeviceMode diff --git a/pkg/pmem-csi-driver/parameters/parameters_test.go b/pkg/pmem-csi-driver/parameters/parameters_test.go index bf97e74756..e01a5f660f 100644 --- a/pkg/pmem-csi-driver/parameters/parameters_test.go +++ b/pkg/pmem-csi-driver/parameters/parameters_test.go @@ -17,15 +17,10 @@ import ( ) func TestParameters(t *testing.T) { - five := uint(5) yes := true - no := false - cache := PersistencyCache normal := PersistencyNormal - foo := "foo" gig := "1Gi" gigNum := int64(1 * 1024 * 1024 * 1024) - name := "joe" tests := []struct { name string @@ -34,54 +29,6 @@ func TestParameters(t *testing.T) { parameters Volume err string }{ - { - name: "createvolume", - origin: CreateVolumeOrigin, - stringmap: VolumeContext{ - CacheSize: "5", - EraseAfter: "false", - PersistencyModel: "cache", - }, - parameters: Volume{ - CacheSize: &five, - EraseAfter: &no, - Persistency: &cache, - }, - }, - { - name: "bad-volumeid", - origin: CreateVolumeOrigin, - stringmap: VolumeContext{ - VolumeID: foo, - }, - err: `parameter "_id" invalid in this context`, - }, - { - name: "good-volumeid", - origin: CreateVolumeInternalOrigin, - stringmap: VolumeContext{ - VolumeID: "foo", - }, - parameters: Volume{ - VolumeID: &foo, - }, - }, - { - name: "createvolumeinternal", - origin: CreateVolumeInternalOrigin, - stringmap: VolumeContext{ - CacheSize: "5", - EraseAfter: "false", - PersistencyModel: "cache", - VolumeID: "foo", - }, - parameters: Volume{ - CacheSize: &five, - EraseAfter: &no, - Persistency: &cache, - VolumeID: &foo, - }, - }, { name: "ephemeral", origin: EphemeralVolumeOrigin, @@ -95,68 +42,15 @@ func TestParameters(t *testing.T) { Size: &gigNum, }, }, - { - name: "publishpersistent", - origin: PersistentVolumeOrigin, - stringmap: VolumeContext{ - CacheSize: "5", - EraseAfter: "false", - PersistencyModel: "cache", - - Name: name, - "csi.storage.k8s.io/foo": "bar", - ProvisionerID: "provisioner XYZ", - }, - parameters: Volume{ - CacheSize: &five, - EraseAfter: &no, - Persistency: &cache, - Name: &name, - }, - }, - { - name: "node", - origin: NodeVolumeOrigin, - stringmap: VolumeContext{ - CacheSize: "5", - EraseAfter: "false", - PersistencyModel: "cache", - Size: gig, - Name: name, - }, - parameters: Volume{ - CacheSize: &five, - EraseAfter: &no, - Persistency: &cache, - Size: &gigNum, - Name: &name, - }, - }, // Various parameters which are not allowed in this context. { name: "invalid-parameter-create", origin: CreateVolumeOrigin, stringmap: VolumeContext{ - VolumeID: "volume-id-chosen-by-attacker", - }, - err: "parameter \"_id\" invalid in this context", - }, - { - name: "invalid-parameter-create-internal", - origin: CreateVolumeInternalOrigin, - stringmap: VolumeContext{ - Ephemeral: "false", - }, - err: "parameter \"csi.storage.k8s.io/ephemeral\" invalid in this context", - }, - { - name: "invalid-ephemeral-context", - origin: EphemeralVolumeOrigin, - stringmap: VolumeContext{ - CacheSize: gig, + Size: "100", }, - err: "parameter \"cacheSize\" invalid in this context", + err: "parameter \"size\" invalid in this context", }, { name: "invalid-persistent-context", @@ -170,9 +64,9 @@ func TestParameters(t *testing.T) { name: "invalid-node-context", origin: NodeVolumeOrigin, stringmap: VolumeContext{ - VolumeID: "volume-id", + "foo": "bar", }, - err: "parameter \"_id\" invalid in this context", + err: "parameter \"foo\" invalid in this context", }, // Parse errors for size. @@ -219,8 +113,7 @@ func TestParameters(t *testing.T) { value = "normal" } } - if key != VolumeID && - key != ProvisionerID && + if key != ProvisionerID && !strings.HasPrefix(key, PodInfoPrefix) { result[key] = value } diff --git a/pkg/pmem-csi-driver/pmem-csi-driver.go b/pkg/pmem-csi-driver/pmem-csi-driver.go index a8f90ca188..fd442bc119 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver.go @@ -22,22 +22,17 @@ import ( api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" + "github.com/intel/pmem-csi/pkg/k8sutil" pmdmanager "github.com/intel/pmem-csi/pkg/pmem-device-manager" pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" - registry "github.com/intel/pmem-csi/pkg/pmem-registry" pmemstate "github.com/intel/pmem-csi/pkg/pmem-state" - "github.com/intel/pmem-csi/pkg/registryserver" "github.com/intel/pmem-csi/pkg/scheduler" + "github.com/intel/pmem-csi/pkg/types" "github.com/kubernetes-csi/csi-lib-utils/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/status" "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) @@ -45,13 +40,17 @@ const ( connectionTimeout time.Duration = 10 * time.Second retryTimeout time.Duration = 10 * time.Second requestTimeout time.Duration = 10 * time.Second + + // Resyncing should never be needed for correct operation, + // so this is so high that it shouldn't matter in practice. + resyncPeriod = 10000 * time.Hour ) type DriverMode string func (mode *DriverMode) Set(value string) error { switch value { - case string(Controller), string(Node): + case string(Node), string(Webhooks): *mode = DriverMode(value) default: // The flag package will add the value to the final output, no need to do it here. @@ -67,10 +66,10 @@ func (mode *DriverMode) String() string { // The mode strings are part of the metrics API (-> csi_controller, // csi_node as subsystem), do not change them! const ( - //Controller definition for controller driver mode - Controller DriverMode = "controller" - //Node definition for noder driver mode + // Node driver with support for provisioning. Node DriverMode = "node" + // Just the webhooks, using metrics instead of gRPC over TCP. + Webhooks DriverMode = "webhooks" ) var ( @@ -86,27 +85,6 @@ var ( }, []string{"version"}, ) - - pmemMaxDesc = prometheus.NewDesc( - "pmem_amount_max_volume_size", - "The size of the largest PMEM volume that can be created.", - nil, nil, - ) - pmemAvailableDesc = prometheus.NewDesc( - "pmem_amount_available", - "Remaining amount of PMEM on the host that can be used for new volumes.", - nil, nil, - ) - pmemManagedDesc = prometheus.NewDesc( - "pmem_amount_managed", - "Amount of PMEM on the host that is managed by PMEM-CSI.", - nil, nil, - ) - pmemTotalDesc = prometheus.NewDesc( - "pmem_amount_total", - "Total amount of PMEM on the host.", - nil, nil, - ) ) func init() { @@ -121,25 +99,14 @@ type Config struct { NodeID string //Endpoint exported csi driver endpoint Endpoint string - //TestEndpoint adds the controller service to the server listening on Endpoint. - //Only needed for testing. - TestEndpoint bool //Mode mode fo the driver Mode DriverMode - //RegistryEndpoint exported registry server endpoint - RegistryEndpoint string //CAFile Root certificate authority certificate file CAFile string //CertFile certificate for server authentication CertFile string //KeyFile server private key file KeyFile string - //ClientCertFile certificate for client side authentication - ClientCertFile string - //ClientKeyFile client private key - ClientKeyFile string - //ControllerEndpoint exported node controller endpoint - ControllerEndpoint string //DeviceManager device manager to use DeviceManager api.DeviceMode //Directory where to persist the node driver state @@ -149,9 +116,19 @@ type Config struct { // PmemPercentage percentage of space to be used by the driver in each PMEM region PmemPercentage uint + // KubeAPIQPS is the average rate of requests to the Kubernetes API server, + // enforced locally in client-go. + KubeAPIQPS float64 + + // KubeAPIQPS is the number of requests that a client is + // allowed to send above the average rate of request. + KubeAPIBurst int + // parameters for Kubernetes scheduler extender schedulerListen string - client kubernetes.Interface + + // parameters for rescheduler + nodeSelector types.NodeSelector // parameters for Prometheus metrics metricsListen string @@ -159,65 +136,11 @@ type Config struct { } type csiDriver struct { - cfg Config - serverTLSConfig *tls.Config - clientTLSConfig *tls.Config - gatherers prometheus.Gatherers -} - -// deviceManagerCollector is a wrapper around a PMEM device manager which -// takes GetCapacity values and turns them into metrics data. -type deviceManagerCollector struct { - pmdmanager.PmemDeviceManager -} - -// Describe implements prometheus.Collector.Describe. -func (dm deviceManagerCollector) Describe(ch chan<- *prometheus.Desc) { - prometheus.DescribeByCollect(dm, ch) -} - -// Collect implements prometheus.Collector.Collect. -func (dm deviceManagerCollector) Collect(ch chan<- prometheus.Metric) { - capacity, err := dm.GetCapacity() - if err != nil { - return - } - ch <- prometheus.MustNewConstMetric( - pmemMaxDesc, - prometheus.GaugeValue, - float64(capacity.MaxVolumeSize), - ) - ch <- prometheus.MustNewConstMetric( - pmemAvailableDesc, - prometheus.GaugeValue, - float64(capacity.Available), - ) - ch <- prometheus.MustNewConstMetric( - pmemManagedDesc, - prometheus.GaugeValue, - float64(capacity.Managed), - ) - ch <- prometheus.MustNewConstMetric( - pmemTotalDesc, - prometheus.GaugeValue, - float64(capacity.Total), - ) + cfg Config + gatherers prometheus.Gatherers } -var _ prometheus.Collector = deviceManagerCollector{} - func GetCSIDriver(cfg Config) (*csiDriver, error) { - validModes := map[DriverMode]struct{}{ - Controller: struct{}{}, - Node: struct{}{}, - } - var serverConfig *tls.Config - var clientConfig *tls.Config - var err error - - if _, ok := validModes[cfg.Mode]; !ok { - return nil, fmt.Errorf("Invalid driver mode: %s", string(cfg.Mode)) - } if cfg.DriverName == "" { return nil, errors.New("driver name configuration option missing") } @@ -227,48 +150,9 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { if cfg.Mode == Node && cfg.NodeID == "" { return nil, errors.New("node ID configuration option missing") } - if cfg.Mode == Controller && cfg.RegistryEndpoint == "" { - return nil, errors.New("registry endpoint configuration option missing") - } - if cfg.Mode == Node && cfg.ControllerEndpoint == "" { - return nil, errors.New("internal controller endpoint configuration option missing") - } if cfg.Mode == Node && cfg.StateBasePath == "" { cfg.StateBasePath = "/var/lib/" + cfg.DriverName } - if cfg.Endpoint == cfg.RegistryEndpoint { - return nil, fmt.Errorf("CSI and registry endpoints must be different, both are: %q", cfg.Endpoint) - } - if cfg.Endpoint == cfg.ControllerEndpoint { - return nil, fmt.Errorf("CSI and internal control endpoints must be different, both are: %q", cfg.Endpoint) - } - - peerName := "pmem-registry" - if cfg.Mode == Controller { - //When driver running in Controller mode, we connect to node controllers - //so use appropriate peer name - peerName = "pmem-node-controller" - } - - if cfg.CertFile != "" && cfg.KeyFile != "" { - serverConfig, err = pmemgrpc.LoadServerTLS(cfg.CAFile, cfg.CertFile, cfg.KeyFile, peerName) - if err != nil { - return nil, err - } - } - - /* if no client certificate details provided use same server certificate to connect to peer server */ - if cfg.ClientCertFile == "" { - cfg.ClientCertFile = cfg.CertFile - cfg.ClientKeyFile = cfg.KeyFile - } - - if cfg.ClientCertFile != "" && cfg.ClientKeyFile != "" { - clientConfig, err = pmemgrpc.LoadClientTLS(cfg.CAFile, cfg.ClientCertFile, cfg.ClientKeyFile, peerName) - if err != nil { - return nil, err - } - } DriverTopologyKey = cfg.DriverName + "/node" @@ -277,9 +161,7 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { buildInfo.With(prometheus.Labels{"version": cfg.Version}).Set(1) return &csiDriver{ - cfg: cfg, - serverTLSConfig: serverConfig, - clientTLSConfig: clientConfig, + cfg: cfg, // We use the default Prometheus registry here in addition to // any custom CSIMetricsManager. Therefore we also return all // data that is registered globally, including (but not @@ -292,12 +174,6 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { } func (csid *csiDriver) Run() error { - // Create GRPC servers - ids, err := NewIdentityServer(csid.cfg.DriverName, csid.cfg.Version) - if err != nil { - return err - } - s := grpcserver.NewNonBlockingGRPCServer() // Ensure that the server is stopped before we return. defer func() { @@ -307,30 +183,86 @@ func (csid *csiDriver) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // On the csi.sock endpoint we gather statistics for incoming - // CSI method calls like any other CSI driver. - cmm := metrics.NewCSIMetricsManagerWithOptions(csid.cfg.DriverName, - metrics.WithProcessStartTime(false), - metrics.WithSubsystem(metrics.SubsystemPlugin), - ) - csid.gatherers = append(csid.gatherers, cmm.GetRegistry()) - switch csid.cfg.Mode { - case Controller: - rs := registryserver.New(csid.clientTLSConfig, csid.cfg.DriverName) - csid.gatherers = append(csid.gatherers, rs.GetMetricsGatherer()) - cs := NewMasterControllerServer(rs) + case Webhooks: + client, err := k8sutil.NewClient(config.KubeAPIQPS, config.KubeAPIBurst) + if err != nil { + return fmt.Errorf("connect to apiserver: %v", err) + } - if err := s.Start(csid.cfg.Endpoint, nil, cmm, ids, cs); err != nil { - return err + // A factory for all namespaces. Some of these are only needed by + // scheduler webhooks or deprovisioner, but because the normal + // setup is to have both enabled, the logic here is simplified so that + // everything gets initialized. + // + // The PV informer is not really needed, but there is no good way to + // tell the lib that it should watch PVs. An informer for a fake client + // did not work: + // Failed to watch *v1.PersistentVolume: unhandled watch: testing.WatchActionImpl + globalFactory := informers.NewSharedInformerFactory(client, resyncPeriod) + pvcInformer := globalFactory.Core().V1().PersistentVolumeClaims().Informer() + pvcLister := globalFactory.Core().V1().PersistentVolumeClaims().Lister() + scLister := globalFactory.Storage().V1().StorageClasses().Lister() + scInformer := globalFactory.Storage().V1().StorageClasses().Informer() + pvInformer := globalFactory.Core().V1().PersistentVolumes().Informer() + csiNodeLister := globalFactory.Storage().V1().CSINodes().Lister() + + var pcp *pmemCSIProvisioner + if csid.cfg.nodeSelector != nil { + serverVersion, err := client.Discovery().ServerVersion() + if err != nil { + return fmt.Errorf("discover server version: %v", err) + } + + // Create rescheduler. This has to be done before starting the factory + // because it will indirectly add a new index. + pcp = newRescheduler(ctx, + csid.cfg.DriverName, + client, pvcInformer, scInformer, pvInformer, csiNodeLister, + csid.cfg.nodeSelector, + serverVersion.GitVersion) } - if err := s.Start(csid.cfg.RegistryEndpoint, csid.serverTLSConfig, nil /* no metrics gathering for registry at the moment */, rs); err != nil { - return err + + // Now that all informers and indices are created we can run the factory. + globalFactory.Start(ctx.Done()) + cacheSyncResult := globalFactory.WaitForCacheSync(ctx.Done()) + klog.V(5).Infof("synchronized caches: %+v", cacheSyncResult) + for t, v := range cacheSyncResult { + if !v { + return fmt.Errorf("failed to sync informer for type %v", t) + } } - // Also run scheduler extender? - if _, err := csid.startScheduler(ctx, cancel, rs); err != nil { - return err + if csid.cfg.schedulerListen != "" { + // Factory for the driver's namespace. + namespace := os.Getenv("POD_NAMESPACE") + if namespace == "" { + return errors.New("POD_NAMESPACE env variable is not set") + } + localFactory := informers.NewSharedInformerFactoryWithOptions(client, resyncPeriod, + informers.WithNamespace(namespace), + ) + podLister := localFactory.Core().V1().Pods().Lister() + c := scheduler.CapacityViaMetrics(namespace, csid.cfg.DriverName, podLister) + localFactory.Start(ctx.Done()) + + sched, err := scheduler.NewScheduler( + csid.cfg.DriverName, + c, + client, + pvcLister, + scLister, + ) + if err != nil { + return fmt.Errorf("create scheduler: %v", err) + } + if _, err := csid.startHTTPSServer(ctx, cancel, csid.cfg.schedulerListen, sched, true /* TLS */); err != nil { + return err + } + } + + if pcp != nil { + pcp.startRescheduler(ctx, cancel) } case Node: dm, err := pmdmanager.New(csid.cfg.DeviceManager, csid.cfg.PmemPercentage) @@ -341,39 +273,27 @@ func (csid *csiDriver) Run() error { if err != nil { return err } - cs := NewNodeControllerServer(csid.cfg.NodeID, dm, sm) - ns := NewNodeServer(cs, filepath.Clean(csid.cfg.StateBasePath)+"/mount") - // Internal CSI calls are tracked on the server side - // with a custom "pmem_csi_node" subsystem. The - // corresponding client calls use "pmem_csi_controller" with - // a tag that identifies the node that is being called. - cmmInternal := metrics.NewCSIMetricsManagerWithOptions(csid.cfg.DriverName, + // On the csi.sock endpoint we gather statistics for incoming + // CSI method calls like any other CSI driver. + cmm := metrics.NewCSIMetricsManagerWithOptions(csid.cfg.DriverName, metrics.WithProcessStartTime(false), - metrics.WithSubsystem("pmem_csi_node"), - // Always add the instance label to allow correlating with - // the controller calls. - metrics.WithLabels(map[string]string{registryserver.NodeLabel: csid.cfg.NodeID}), + metrics.WithSubsystem(metrics.SubsystemPlugin), ) - csid.gatherers = append(csid.gatherers, cmmInternal.GetRegistry()) - if err := s.Start(csid.cfg.ControllerEndpoint, csid.serverTLSConfig, cmmInternal, cs); err != nil { - return err - } - if err := csid.registerNodeController(); err != nil { - return err - } - services := []grpcserver.Service{ids, ns} - if csid.cfg.TestEndpoint { - services = append(services, cs) - } + csid.gatherers = append(csid.gatherers, cmm.GetRegistry()) + + // Create GRPC servers + ids := NewIdentityServer(csid.cfg.DriverName, csid.cfg.Version) + cs := NewNodeControllerServer(csid.cfg.NodeID, dm, sm) + ns := NewNodeServer(cs, filepath.Clean(csid.cfg.StateBasePath)+"/mount") + + services := []grpcserver.Service{ids, ns, cs} if err := s.Start(csid.cfg.Endpoint, nil, cmm, services...); err != nil { return err } // Also collect metrics data via the device manager. - prometheus.WrapRegistererWith(prometheus.Labels{registryserver.NodeLabel: csid.cfg.NodeID}, prometheus.DefaultRegisterer).MustRegister( - deviceManagerCollector{dm}, - ) + pmdmanager.CapacityCollector{PmemDeviceCapacity: dm}.MustRegister(prometheus.DefaultRegisterer, csid.cfg.NodeID, csid.cfg.DriverName) default: return fmt.Errorf("Unsupported device mode '%v", csid.cfg.Mode) } @@ -391,80 +311,25 @@ func (csid *csiDriver) Run() error { signal.Notify(c, os.Interrupt, syscall.SIGTERM) select { case sig := <-c: - // Here we want to shut down cleanly, i.e. let running - // gRPC calls complete. klog.V(3).Infof("Caught signal %s, terminating.", sig) + // We sleep briefly to give sidecars a chance to shut down cleanly + // before we close the CSI socket and force them to shut down + // abnormally, because the latter causes lots of debug output + // due to usage of klog.Fatal (https://github.com/intel/pmem-csi/issues/856). + time.Sleep(time.Second) case <-ctx.Done(): // The scheduler HTTP server must have failed (to start). - // We quit in that case. + // We quit directly in that case. } + + // Here (in contrast to the s.ForceStop() above) we let the gRPC server finish + // its work on any pending call. s.Stop() s.Wait() return nil } -func (csid *csiDriver) registerNodeController() error { - var err error - var conn *grpc.ClientConn - - for { - klog.V(3).Infof("Connecting to registry server at: %s\n", csid.cfg.RegistryEndpoint) - conn, err = pmemgrpc.Connect(csid.cfg.RegistryEndpoint, csid.clientTLSConfig) - if err == nil { - break - } - klog.Warningf("Failed to connect registry server: %s, retrying after %v seconds...", err.Error(), retryTimeout.Seconds()) - time.Sleep(retryTimeout) - } - - req := ®istry.RegisterControllerRequest{ - NodeId: csid.cfg.NodeID, - Endpoint: csid.cfg.ControllerEndpoint, - } - - if err := register(context.Background(), conn, req); err != nil { - return err - } - go waitAndWatchConnection(conn, req) - - return nil -} - -// startScheduler starts the scheduler extender if it is enabled. It -// logs errors and cancels the context when it runs into a problem, -// either during the startup phase (blocking) or later at runtime (in -// a go routine). -func (csid *csiDriver) startScheduler(ctx context.Context, cancel func(), rs *registryserver.RegistryServer) (string, error) { - if csid.cfg.schedulerListen == "" { - return "", nil - } - - resyncPeriod := 1 * time.Hour - factory := informers.NewSharedInformerFactory(csid.cfg.client, resyncPeriod) - pvcLister := factory.Core().V1().PersistentVolumeClaims().Lister() - scLister := factory.Storage().V1().StorageClasses().Lister() - sched, err := scheduler.NewScheduler( - csid.cfg.DriverName, - scheduler.CapacityViaRegistry(rs), - csid.cfg.client, - pvcLister, - scLister, - ) - if err != nil { - return "", fmt.Errorf("create scheduler: %v", err) - } - factory.Start(ctx.Done()) - cacheSyncResult := factory.WaitForCacheSync(ctx.Done()) - klog.V(5).Infof("synchronized caches: %+v", cacheSyncResult) - for t, v := range cacheSyncResult { - if !v { - return "", fmt.Errorf("failed to sync informer for type %v", t) - } - } - return csid.startHTTPSServer(ctx, cancel, csid.cfg.schedulerListen, sched, true /* TLS */) -} - // startMetrics starts the HTTPS server for the Prometheus endpoint, if one is configured. // Error handling is the same as for startScheduler. func (csid *csiDriver) startMetrics(ctx context.Context, cancel func()) (string, error) { @@ -528,49 +393,3 @@ func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), list return tcpListener.Addr().String(), nil } - -// waitAndWatchConnection Keeps watching for connection changes, and whenever the -// connection state changed from lost to ready, it re-register the node controller with registry server. -func waitAndWatchConnection(conn *grpc.ClientConn, req *registry.RegisterControllerRequest) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - connectionLost := false - - for { - s := conn.GetState() - if s == connectivity.Ready { - if connectionLost { - klog.V(4).Info("ReConnected.") - if err := register(ctx, conn, req); err != nil { - klog.Warning(err) - } - } - } else { - connectionLost = true - klog.V(4).Info("Connection state: ", s) - } - conn.WaitForStateChange(ctx, s) - } -} - -// register Tries to register with RegistryServer in endless loop till, -// either the registration succeeds or RegisterController() returns only possible InvalidArgument error. -func register(ctx context.Context, conn *grpc.ClientConn, req *registry.RegisterControllerRequest) error { - client := registry.NewRegistryClient(conn) - for { - klog.V(3).Info("Registering controller...") - if _, err := client.RegisterController(ctx, req); err != nil { - if s, ok := status.FromError(err); ok && s.Code() == codes.InvalidArgument { - return fmt.Errorf("Registration failed: %s", s.Message()) - } - klog.Warningf("Failed to register: %s, retrying after %v seconds...", err.Error(), retryTimeout.Seconds()) - time.Sleep(retryTimeout) - } else { - break - } - } - klog.V(4).Info("Registration success") - - return nil -} diff --git a/pkg/pmem-csi-driver/pmem-csi-driver_test.go b/pkg/pmem-csi-driver/pmem-csi-driver_test.go index f49b603ea7..407285a4a8 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver_test.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver_test.go @@ -50,17 +50,16 @@ build_info{version="foo-bar-test"} 1 t.Run(n, func(t *testing.T) { path := "/metrics2" pmemd, err := GetCSIDriver(Config{ - Mode: Controller, - DriverName: "pmem-csi", - NodeID: "testnode", - Endpoint: "unused", - RegistryEndpoint: "unused2", - Version: "foo-bar-test", - CAFile: caFile, - CertFile: certFile, - KeyFile: keyFile, - metricsPath: path, - metricsListen: "127.0.0.1:", // port allocated dynamically + Mode: Webhooks, + DriverName: "pmem-csi", + NodeID: "testnode", + Endpoint: "unused", + Version: "foo-bar-test", + CAFile: caFile, + CertFile: certFile, + KeyFile: keyFile, + metricsPath: path, + metricsListen: "127.0.0.1:", // port allocated dynamically }) require.NoError(t, err, "get PMEM-CSI driver") diff --git a/pkg/pmem-csi-driver/rescheduler.go b/pkg/pmem-csi-driver/rescheduler.go new file mode 100644 index 0000000000..aacff70d0e --- /dev/null +++ b/pkg/pmem-csi-driver/rescheduler.go @@ -0,0 +1,210 @@ +/* +Copyright 2021 Intel Corporation. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package pmemcsidriver + +import ( + "context" + "errors" + "fmt" + + "github.com/intel/pmem-csi/pkg/logger" + "github.com/intel/pmem-csi/pkg/types" + + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + storagelistersv1 "k8s.io/client-go/listers/storage/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller" +) + +const ( + annSelectedNode = "volume.kubernetes.io/selected-node" +) + +// newRescheduler creates an instance of +// sig-storage-lib-external-provisioner which has only one purpose: it +// detects PVCs that were assigned to a node which doesn't have a +// PMEM-CSI node driver running and triggers re-scheduling of those +// PVCs by removing the "selected node" annotation. It never +// provisions volumes. That is handled by the node instances. +func newRescheduler(ctx context.Context, + driverName string, + client kubernetes.Interface, + pvcInformer cache.SharedIndexInformer, + scInformer cache.SharedIndexInformer, + pvInformer cache.SharedIndexInformer, + csiNodeLister storagelistersv1.CSINodeLister, + nodeSelector types.NodeSelector, + serverGitVersion string) *pmemCSIProvisioner { + provisionerOptions := []func(*controller.ProvisionController) error{ + controller.LeaderElection(false), + controller.ClaimsInformer(pvcInformer), + controller.ClassesInformer(scInformer), + controller.VolumesInformer(pvInformer), + } + + pcp := &pmemCSIProvisioner{ + driverName: driverName, + nodeSelector: nodeSelector, + csiNodeLister: csiNodeLister, + } + + provisionController := controller.NewProvisionController( + client, + driverName, + pcp, + serverGitVersion, + provisionerOptions..., + ) + + pcp.provisionController = provisionController + return pcp +} + +type pmemCSIProvisioner struct { + driverName string + nodeSelector types.NodeSelector + csiNodeLister storagelistersv1.CSINodeLister + provisionController *controller.ProvisionController +} + +// startRescheduler logs errors and cancels the context when it runs +// into a problem, either during the startup phase (blocking) or later +// at runtime (in a go routine). +func (pcp *pmemCSIProvisioner) startRescheduler(ctx context.Context, cancel func()) { + l := logger.Get(ctx).WithName("rescheduler") + + l.Info("starting") + go func() { + defer cancel() + defer l.Info("stopped") + pcp.provisionController.Run(ctx) + }() +} + +// ShouldProvision is called for each pending PVC before the lib +// starts working on the PVC. We only deal with those which need to be +// rescheduled. +func (pcp *pmemCSIProvisioner) ShouldProvision(ctx context.Context, pvc *v1.PersistentVolumeClaim) bool { + l := logger.Get(ctx) + + reschedule, err := pcp.shouldReschedule(ctx, pvc, nil) + if err != nil { + // Something went wrong. We have to allow the lib to + // start working on this PVC, otherwise users will + // never see error events. + l.Error(err, "deprovision check failed") + reschedule = true + } + return reschedule +} + +// Provision is called after the lib has emitted an event about "starting to provision". +// Despite the name, the only outcome is "no change" (= leave PVC unmodified) +// or "reschedule" (= remove selected node annotation). +func (pcp *pmemCSIProvisioner) Provision(ctx context.Context, opts controller.ProvisionOptions) (*v1.PersistentVolume, controller.ProvisioningState, error) { + reschedule, err := pcp.shouldReschedule(ctx, opts.PVC, opts.SelectedNode) + if err != nil { + return nil, controller.ProvisioningNoChange, fmt.Errorf("deprovision check failed: %v", err) + } + if reschedule { + return nil, controller.ProvisioningReschedule, fmt.Errorf("reschedule PVC %s/%s because it is assigned to node %s which has no PMEM-CSI driver", + opts.PVC.Namespace, opts.PVC.Name, opts.SelectedNode.Name) + } + if opts.SelectedNode != nil { + err = &controller.IgnoredError{ + Reason: fmt.Sprintf("not responsible for provisioning of PVC %s/%s because it will be handled by the PMEM-CSI driver on node %q", + opts.PVC.Namespace, opts.PVC.Name, opts.SelectedNode.Name), + } + } else { + err = &controller.IgnoredError{ + Reason: fmt.Sprintf("not responsible for provisioning of PVC %s/%s because it is not assigned to a node", + opts.PVC.Namespace, opts.PVC.Name), + } + } + return nil, controller.ProvisioningNoChange, err +} + +func (pcp *pmemCSIProvisioner) Delete(context.Context, *v1.PersistentVolume) error { + return errors.New("not implemented") +} + +func (pcp *pmemCSIProvisioner) shouldReschedule(ctx context.Context, pvc *v1.PersistentVolumeClaim, node *v1.Node) (bool, error) { + l := logger.Get(ctx).WithName("ShouldReschedulePVC").WithValues("pvc", klog.KObj(pvc)) + if node != nil { + l = l.WithValues("node", klog.KObj(node)) + } + + // "node" might be nil. Check the label directly. + var selectedNode string + if pvc.Annotations != nil { + selectedNode = pvc.Annotations[annSelectedNode] + } + if selectedNode == "" { + // No need to reschedule. + l.V(5).Info("no need to reschedule, no selected node") + return false, nil + } + + // We have to be absolutely certain that the PVC is not going + // to be handled on the node. If we remove the annotation + // while a driver node instance starts to provisision it, + // volumes may leak. + // + // Therefore we check both the labels on the node ("Should a + // PMEM-CSI driver run here?") and the CSINode object ("Does a + // PMEM-CSI driver (still) run here?"). + // + // The node check is expensive. We either would have to watch + // all nodes (which is expensive) or do a GET per check (also + // expensive). To mitigate this, the node is only checked when + // called through Provision() with a Node object already + // retrieved by the lib. + // + // When the scheduler extensions work, we should rarely get to + // Provision() because typically PVCs get assigned to nodes + // with PMEM-CSI and thus the CSINode check already bails out + // of ShouldProvision. + // + // Only when the extensions are off, then Provision() may get + // called more often. Such a cluster setup should better be + // avoided. + driverIsRunning := true + csiNode, err := pcp.csiNodeLister.Get(selectedNode) + switch { + case err == nil: + driverIsRunning = hasDriver(csiNode, pcp.driverName) + case apierrs.IsNotFound(err): + driverIsRunning = false + default: + return false, fmt.Errorf("retrieve CSINode %s: %v", selectedNode, err) + } + if node == nil { + // Decide only based on CSINode. + reschedule := !driverIsRunning + l.V(3).Info("result", "reschedule", reschedule, "driverIsRunning", driverIsRunning) + return reschedule, nil + } + + driverMightRun := pcp.nodeSelector.MatchesLabels(node.Labels) + + reschedule := !driverMightRun && !driverIsRunning + l.V(3).Info("result", "reschedule", reschedule, "driverMightRun", driverMightRun, "driverIsRunning", driverIsRunning) + return reschedule, nil +} + +func hasDriver(csiNode *storagev1.CSINode, driverName string) bool { + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == driverName { + return true + } + } + return false +} diff --git a/pkg/pmem-csi-driver/rescheduler_test.go b/pkg/pmem-csi-driver/rescheduler_test.go new file mode 100644 index 0000000000..37fa8406a6 --- /dev/null +++ b/pkg/pmem-csi-driver/rescheduler_test.go @@ -0,0 +1,231 @@ +/* +Copyright 2021 Intel Corporation. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package pmemcsidriver + +import ( + "context" + "errors" + "testing" + + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller" + + "github.com/intel/pmem-csi/pkg/logger" + "github.com/intel/pmem-csi/pkg/logger/testinglogger" + "github.com/intel/pmem-csi/pkg/types" +) + +type testcase struct { + driverName string + haveCSIDriver bool + haveCSINode bool + selectedNode string + nodeLabels map[string]string + nodeSelector types.NodeSelector + + expectError bool + expectReschedulePreCheck bool + expectRescheduleFinalCheck bool +} + +const ( + driverName = "pmem-csi.intel.com" + nodeLabelName = "storage" + nodeLabelValue = "pmem" + nodeName = "pmem-worker" +) + +func TestRescheduler(t *testing.T) { + testcases := map[string]testcase{ + "node-okay": { + driverName: driverName, + haveCSIDriver: true, + haveCSINode: true, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{ + nodeLabelName: nodeLabelValue, + }, + }, + "missing-csi-node": { + driverName: driverName, + haveCSIDriver: true, + haveCSINode: false, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{ + nodeLabelName: nodeLabelValue, + }, + + expectReschedulePreCheck: true, + expectRescheduleFinalCheck: false, + }, + "missing-csi-driver": { + driverName: driverName, + haveCSIDriver: false, + haveCSINode: true, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{ + nodeLabelName: nodeLabelValue, + }, + + expectReschedulePreCheck: true, + expectRescheduleFinalCheck: false, + }, + "wrong-csi-driver": { + driverName: "other." + driverName, + haveCSIDriver: false, + haveCSINode: true, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{ + nodeLabelName: nodeLabelValue, + }, + + expectReschedulePreCheck: true, + expectRescheduleFinalCheck: false, + }, + "missing-node-labels": { + driverName: driverName, + haveCSIDriver: true, + haveCSINode: true, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{}, + + expectReschedulePreCheck: false, + expectRescheduleFinalCheck: false, + }, + "reschedule": { + driverName: driverName, + haveCSIDriver: false, + haveCSINode: true, + selectedNode: nodeName, + nodeSelector: types.NodeSelector{ + nodeLabelName: nodeLabelValue, + }, + nodeLabels: map[string]string{}, + + expectReschedulePreCheck: true, + expectRescheduleFinalCheck: true, + }, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + ctx := logger.Set(context.Background(), testinglogger.New(t)) + pcp := pmemCSIProvisioner{ + driverName: driverName, + nodeSelector: tc.nodeSelector, + csiNodeLister: fakeCSINodeLister{ + driverName: tc.driverName, + haveCSIDriver: tc.haveCSIDriver, + haveCSINode: tc.haveCSINode, + }, + } + + pvc := &v1.PersistentVolumeClaim{} + if tc.selectedNode != "" { + pvc.Annotations = map[string]string{ + annSelectedNode: tc.selectedNode, + } + } + + if pcp.ShouldProvision(ctx, pvc) != tc.expectReschedulePreCheck { + t.Errorf("ShouldProvision unexpectedly returned %v", !tc.expectReschedulePreCheck) + } + + node := &v1.Node{} + node.Name = tc.selectedNode + node.Labels = tc.nodeLabels + pv, state, err := pcp.Provision(ctx, controller.ProvisionOptions{ + PVC: pvc, + SelectedNode: node, + }) + if pv != nil { + t.Error("Provision returned non-nil PV") + } + switch { + case tc.expectError: + if state != controller.ProvisioningNoChange { + t.Errorf("expected state %s, got %s", controller.ProvisioningNoChange, state) + } + if err == nil { + t.Error("expected error, got nil") + } else { + _, ignored := err.(*controller.IgnoredError) + if ignored { + t.Errorf("expected normal error, got IgnoredError: %v", err) + } + } + case tc.expectRescheduleFinalCheck: + if state != controller.ProvisioningReschedule { + t.Errorf("expected state %s, got %s", controller.ProvisioningReschedule, state) + } + if err == nil { + t.Error("expected error, got nil") + } else { + _, ignored := err.(*controller.IgnoredError) + if ignored { + t.Errorf("expected normal error, got IgnoredError: %v", err) + } + } + default: + if state != controller.ProvisioningNoChange { + t.Errorf("expected state %s, got %s", controller.ProvisioningNoChange, state) + } + if err == nil { + t.Error("expected error, got nil") + } else { + _, ignored := err.(*controller.IgnoredError) + if !ignored { + t.Errorf("expected ignored error, got normal error: %v", err) + } + } + } + }) + } +} + +type fakeCSINodeLister struct { + driverName string + haveCSIDriver bool + haveCSINode bool +} + +func (f fakeCSINodeLister) Get(nodeName string) (*storagev1.CSINode, error) { + if !f.haveCSINode { + return nil, apierrs.NewNotFound(schema.GroupResource{}, "nodeName") + } + csiNode := &storagev1.CSINode{} + csiNode.Name = nodeName + if f.haveCSIDriver { + csiNode.Spec.Drivers = []storagev1.CSINodeDriver{ + {Name: f.driverName}, + } + } + return csiNode, nil +} + +func (f fakeCSINodeLister) List(labels.Selector) ([]*storagev1.CSINode, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index 8c99a949f7..f1871bd9c8 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -8,22 +8,15 @@ package deployment import ( "context" - "crypto/rsa" - "crypto/tls" "fmt" - "io/ioutil" - "os" - "path" "reflect" - "runtime" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" - grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" "github.com/intel/pmem-csi/pkg/logger" - pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" - pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" + "github.com/intel/pmem-csi/pkg/types" "github.com/intel/pmem-csi/pkg/version" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -35,7 +28,6 @@ import ( apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubectl/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -45,6 +37,7 @@ const ( nodeControllerPort = 10001 nodeMetricsPort = 10010 provisionerMetricsPort = 10011 + schedulerPort = 8000 ) func typeMeta(gv schema.GroupVersion, kind string) metav1.TypeMeta { @@ -60,7 +53,7 @@ func typeMeta(gv schema.GroupVersion, kind string) metav1.TypeMeta { // // The RBAC rules in deploy/kustomize/operator/operator.yaml must // allow all of the operations (creation, patching, etc.). -var currentObjects = []apiruntime.Object{ +var currentObjects = []client.Object{ &rbacv1.ClusterRole{TypeMeta: typeMeta(rbacv1.SchemeGroupVersion, "ClusterRole")}, &rbacv1.ClusterRoleBinding{TypeMeta: typeMeta(rbacv1.SchemeGroupVersion, "ClusterRoleBinding")}, &storagev1beta1.CSIDriver{TypeMeta: typeMeta(storagev1beta1.SchemeGroupVersion, "CSIDriver")}, @@ -71,11 +64,12 @@ var currentObjects = []apiruntime.Object{ &corev1.Service{TypeMeta: typeMeta(corev1.SchemeGroupVersion, "Service")}, &corev1.ServiceAccount{TypeMeta: typeMeta(corev1.SchemeGroupVersion, "ServiceAccount")}, &appsv1.StatefulSet{TypeMeta: typeMeta(appsv1.SchemeGroupVersion, "StatefulSet")}, + &admissionregistrationv1beta1.MutatingWebhookConfiguration{TypeMeta: typeMeta(admissionregistrationv1beta1.SchemeGroupVersion, "MutatingWebhookConfiguration")}, } // CurrentObjects returns the active sub-object types used by the operator // for a driver deployment. -func CurrentObjects() []apiruntime.Object { +func CurrentObjects() []client.Object { return currentObjects } @@ -86,7 +80,7 @@ func CurrentObjects() []apiruntime.Object { // // The RBAC rules in deploy/kustomize/operator/operator.yaml must // allow listing and removing of these objects. -var obsoleteObjects = []apiruntime.Object{ +var obsoleteObjects = []client.Object{ &corev1.ConfigMap{TypeMeta: typeMeta(corev1.SchemeGroupVersion, "ConfigMap")}, // included only for testing purposes } @@ -116,17 +110,19 @@ type pmemCSIDeployment struct { // operator's namespace used for creating sub-resources namespace string k8sVersion version.Version + + controllerCABundle []byte } // objectPatch combines a modified object and the patch against // the current revision of that object that produces the modified // object. type objectPatch struct { - obj apiruntime.Object + obj client.Object patch client.Patch } -func newObjectPatch(obj, copy apiruntime.Object) *objectPatch { +func newObjectPatch(obj client.Object, copy apiruntime.Object) *objectPatch { return &objectPatch{ obj: obj, patch: client.MergeFrom(copy), @@ -165,18 +161,13 @@ func (op objectPatch) diff() ([]byte, error) { // Apply sends the changes to API Server // Creates new object if not existing, otherwise patches it with changes -func (op *objectPatch) apply(ctx context.Context, c client.Client, labels map[string]string) error { +func (op *objectPatch) apply(ctx context.Context, c client.Client) error { objMeta, err := meta.Accessor(op.obj) if err != nil { return fmt.Errorf("internal error %T: %v", op.obj, err) } l := logger.Get(ctx).WithName("objectPatch/apply") - // NOTE(avalluri): Set labels just before creating/patching. - // Setting them before creating the client.Patch makes - // they get lost from the final diff. - objMeta.SetLabels(labels) - if op.isNew() { // For unknown reason client.Create() clearing off the // GVK on obj, So restore it manually. @@ -211,21 +202,36 @@ func (d *pmemCSIDeployment) reconcile(ctx context.Context, r *ReconcileDeploymen } l := logger.Get(ctx).WithName("reconcile") - l.V(3).Info("start", "deployment", d.Name, "phase", d.Status.Phase) - var allObjects []apiruntime.Object - redeployAll := func() error { - var o apiruntime.Object - var err error - s, err := d.redeploySecrets(ctx, r) - if err != nil { - return err + if d.Spec.ControllerTLSSecret != "" { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, } - for _, o := range s { - allObjects = append(allObjects, o) + if err := r.client.Get(ctx, + client.ObjectKey{ + Namespace: d.namespace, + Name: d.Spec.ControllerTLSSecret}, + secret); err != nil { + return fmt.Errorf("loading ControllerTLSSecret %s from namespace %s: %v", d.Spec.ControllerTLSSecret, d.namespace, err) + } + ca, ok := secret.Data[api.TLSSecretCA] + if !ok { + return fmt.Errorf("ControllerTLSSecret %s in namespace %s contains no %s", d.Spec.ControllerTLSSecret, d.namespace, api.TLSSecretCA) } + d.controllerCABundle = ca + } + l.V(3).Info("start", "deployment", d.Name, "phase", d.Status.Phase) + var allObjects []apiruntime.Object + redeployAll := func() error { for name, handler := range subObjectHandlers { - if o, err = d.redeploy(ctx, r, handler); err != nil { + if handler.enabled != nil && !handler.enabled(d) { + continue + } + o, err := d.redeploy(ctx, r, handler) + if err != nil { return fmt.Errorf("failed to update %s: %v", name, err) } allObjects = append(allObjects, o) @@ -252,7 +258,7 @@ func (d *pmemCSIDeployment) reconcile(ctx context.Context, r *ReconcileDeploymen // getSubObject retrieves the latest revision of given object type from the API server // And checks if that object is owned by the current deployment CR -func (d *pmemCSIDeployment) getSubObject(ctx context.Context, r *ReconcileDeployment, obj apiruntime.Object) error { +func (d *pmemCSIDeployment) getSubObject(ctx context.Context, r *ReconcileDeployment, obj client.Object) error { objMeta, err := meta.Accessor(obj) if err != nil { return fmt.Errorf("internal error %T: %v", obj, err) @@ -275,16 +281,12 @@ func (d *pmemCSIDeployment) getSubObject(ctx context.Context, r *ReconcileDeploy return nil } -// updateSubObject writes the object changes to the API server. -func (d *pmemCSIDeployment) updateSubObject(ctx context.Context, r *ReconcileDeployment, op *objectPatch) error { - return op.apply(ctx, r.client, d.Spec.Labels) -} - type redeployObject struct { objType reflect.Type - object func(*pmemCSIDeployment) apiruntime.Object - modify func(*pmemCSIDeployment, apiruntime.Object) error - postUpdate func(*pmemCSIDeployment, apiruntime.Object) error + enabled func(*pmemCSIDeployment) bool + object func(*pmemCSIDeployment) client.Object + modify func(*pmemCSIDeployment, client.Object) error + postUpdate func(*pmemCSIDeployment, client.Object) error } // redeploy resets/patches the object returned by ro.object() @@ -297,7 +299,7 @@ type redeployObject struct { // 5. Call objectPatch.Apply() to submit the chanages to the APIServer. // 6. If the update in step-5 was success, then call the ro.postUpdate() callback // to run any post update steps. -func (d *pmemCSIDeployment) redeploy(ctx context.Context, r *ReconcileDeployment, ro redeployObject) (apiruntime.Object, error) { +func (d *pmemCSIDeployment) redeploy(ctx context.Context, r *ReconcileDeployment, ro redeployObject) (client.Object, error) { o := ro.object(d) if o == nil { return nil, fmt.Errorf("nil object") @@ -309,105 +311,50 @@ func (d *pmemCSIDeployment) redeploy(ctx context.Context, r *ReconcileDeployment if err := ro.modify(d, o); err != nil { return nil, err } - if err := d.updateSubObject(ctx, r, op); err != nil { - return nil, err - } - if ro.postUpdate != nil { - if err := ro.postUpdate(d, o); err != nil { - return nil, err - } - } - return o, nil -} - -// redeploySecrets ensures that the secrets get (re)deployed that are -// required for running the driver. -// -// First it checks if the deployment is configured with the needed certificates. -// If provided, validate and (re)create secrets using them. -// Else, provision new certificates(only if no existing secrets found) and deploy. -// -// We cannot use d.redeploy() as secrets needs to be provisioned if not preset. -// This special case cannot be fit into generice redeploy logic. -func (d *pmemCSIDeployment) redeploySecrets(ctx context.Context, r *ReconcileDeployment) ([]*corev1.Secret, error) { - rs := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: d.getObjectMeta(d.RegistrySecretName(), false), - } - if err := d.getSubObject(ctx, r, rs); err != nil { - return nil, err - } - rop := newObjectPatch(rs, rs.DeepCopy()) - ns := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: d.getObjectMeta(d.NodeSecretName(), false), + // Add the additional labels before patching. + objMeta, err := meta.Accessor(o) + if err != nil { + return nil, fmt.Errorf("internal error %T: %v", op.obj, err) } - if err := d.getSubObject(ctx, r, ns); err != nil { - return nil, err + labels := objMeta.GetLabels() + if labels == nil { + labels = map[string]string{} } - nop := newObjectPatch(ns, ns.DeepCopy()) - - update := func() error { - d.getRegistrySecrets(rs) - if err := d.updateSubObject(ctx, r, rop); err != nil { - return fmt.Errorf("failed to update registry secrets: %w", err) - } - - d.getNodeSecrets(ns) - if err := d.updateSubObject(ctx, r, nop); err != nil { - return fmt.Errorf("failed to update node secrets: %w", err) - } - return nil + for key, value := range d.Spec.Labels { + labels[key] = value } + objMeta.SetLabels(labels) - certsProvided, err := d.HaveCertificatesConfigured() - if err != nil { + if err := op.apply(ctx, r.client); err != nil { return nil, err } - - updateSecrets := false - if certsProvided { - // Use provided certificates - if err := d.validateCertificates(); err != nil { - d.SetCondition(api.CertsVerified, corev1.ConditionFalse, err.Error()) - return nil, err - } - d.SetCondition(api.CertsVerified, corev1.ConditionTrue, "Driver certificates validated.") - updateSecrets = true - } else if rop.isNew() || nop.isNew() { - // Provision new self-signed certificates if not already present - if err := d.provisionCertificates(ctx); err != nil { - d.SetCondition(api.CertsReady, corev1.ConditionFalse, err.Error()) - return nil, err - } - updateSecrets = true - } - - if updateSecrets { - if err := update(); err != nil { + if ro.postUpdate != nil { + if err := ro.postUpdate(d, o); err != nil { return nil, err } } - d.SetCondition(api.CertsReady, corev1.ConditionTrue, "Driver certificates are available.") + return o, nil +} - return []*corev1.Secret{rs, ns}, nil +func mutatingWebhookEnabled(d *pmemCSIDeployment) bool { + return d.Spec.ControllerTLSSecret != "" && d.Spec.MutatePods != api.MutatePodsNever } var subObjectHandlers = map[string]redeployObject{ "node driver": { objType: reflect.TypeOf(&appsv1.DaemonSet{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: d.getObjectMeta(d.NodeDriverName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getNodeDaemonSet(o.(*appsv1.DaemonSet)) return nil }, - postUpdate: func(d *pmemCSIDeployment, o apiruntime.Object) error { + postUpdate: func(d *pmemCSIDeployment, o client.Object) error { ds := o.(*appsv1.DaemonSet) // Update node driver status is status object status := "NotReady" @@ -426,17 +373,17 @@ var subObjectHandlers = map[string]redeployObject{ }, "controller driver": { objType: reflect.TypeOf(&appsv1.StatefulSet{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{Kind: "StatefulSet", APIVersion: "apps/v1"}, ObjectMeta: d.getObjectMeta(d.ControllerDriverName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerStatefulSet(o.(*appsv1.StatefulSet)) return nil }, - postUpdate: func(d *pmemCSIDeployment, o apiruntime.Object) error { + postUpdate: func(d *pmemCSIDeployment, o client.Object) error { ss := o.(*appsv1.StatefulSet) // Update controller status is status object status := "NotReady" @@ -456,104 +403,196 @@ var subObjectHandlers = map[string]redeployObject{ }, "controller service": { objType: reflect.TypeOf(&corev1.Service{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: d.getObjectMeta(d.ControllerServiceName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerService(o.(*corev1.Service)) return nil }, }, "metrics service": { objType: reflect.TypeOf(&corev1.Service{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: d.getObjectMeta(d.MetricsServiceName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getMetricsService(o.(*corev1.Service)) return nil }, }, "CSIDriver": { objType: reflect.TypeOf(&storagev1beta1.CSIDriver{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &storagev1beta1.CSIDriver{ TypeMeta: metav1.TypeMeta{Kind: "CSIDriver", APIVersion: "storage.k8s.io/v1beta1"}, ObjectMeta: d.getObjectMeta(d.CSIDriverName(), true), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getCSIDriver(o.(*storagev1beta1.CSIDriver)) return nil }, }, + "webhooks role": { + objType: reflect.TypeOf(&rbacv1.Role{}), + object: func(d *pmemCSIDeployment) client.Object { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksRoleName(), false), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getWebhooksRole(o.(*rbacv1.Role)) + return nil + }, + }, + "webhooks role binding": { + objType: reflect.TypeOf(&rbacv1.RoleBinding{}), + object: func(d *pmemCSIDeployment) client.Object { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksRoleBindingName(), false), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getWebhooksRoleBinding(o.(*rbacv1.RoleBinding)) + return nil + }, + }, + "webhooks cluster role": { + objType: reflect.TypeOf(&rbacv1.ClusterRole{}), + object: func(d *pmemCSIDeployment) client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksClusterRoleName(), true), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getWebhooksClusterRole(o.(*rbacv1.ClusterRole)) + return nil + }, + }, + "webhooks cluster role binding": { + objType: reflect.TypeOf(&rbacv1.ClusterRoleBinding{}), + object: func(d *pmemCSIDeployment) client.Object { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksClusterRoleBindingName(), true), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getWebhooksClusterRoleBinding(o.(*rbacv1.ClusterRoleBinding)) + return nil + }, + }, + "webhooks service account": { + objType: reflect.TypeOf(&corev1.ServiceAccount{}), + object: func(d *pmemCSIDeployment) client.Object { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksServiceAccountName(), false), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + // nothing to customize for service account + return nil + }, + }, + "mutating webhook configuration": { + objType: reflect.TypeOf(&admissionregistrationv1beta1.MutatingWebhookConfiguration{}), + enabled: mutatingWebhookEnabled, + object: func(d *pmemCSIDeployment) client.Object { + return &admissionregistrationv1beta1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{Kind: "MutatingWebhookConfiguration", APIVersion: "admissionregistration.k8s.io/v1beta1"}, + ObjectMeta: d.getObjectMeta(d.MutatingWebhookName(), true), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getMutatingWebhookConfig(o.(*admissionregistrationv1beta1.MutatingWebhookConfiguration)) + return nil + }, + }, + "scheduler service": { + objType: reflect.TypeOf(&corev1.Service{}), + object: func(d *pmemCSIDeployment) client.Object { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: d.getObjectMeta(d.SchedulerServiceName(), false), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getSchedulerService(o.(*corev1.Service)) + return nil + }, + }, "provisioner role": { objType: reflect.TypeOf(&rbacv1.Role{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &rbacv1.Role{ TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: d.getObjectMeta(d.ProvisionerRoleName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerProvisionerRole(o.(*rbacv1.Role)) return nil }, }, "provisioner role binding": { objType: reflect.TypeOf(&rbacv1.RoleBinding{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &rbacv1.RoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: d.getObjectMeta(d.ProvisionerRoleBindingName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerProvisionerRoleBinding(o.(*rbacv1.RoleBinding)) return nil }, }, "provisioner cluster role": { objType: reflect.TypeOf(&rbacv1.ClusterRole{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: d.getObjectMeta(d.ProvisionerClusterRoleName(), true), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerProvisionerClusterRole(o.(*rbacv1.ClusterRole)) return nil }, }, "provisioner cluster role binding": { objType: reflect.TypeOf(&rbacv1.ClusterRoleBinding{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: d.getObjectMeta(d.ProvisionerClusterRoleBindingName(), true), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { d.getControllerProvisionerClusterRoleBinding(o.(*rbacv1.ClusterRoleBinding)) return nil }, }, - "service account": { + "provisioner service account": { objType: reflect.TypeOf(&corev1.ServiceAccount{}), - object: func(d *pmemCSIDeployment) apiruntime.Object { + object: func(d *pmemCSIDeployment) client.Object { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, - ObjectMeta: d.getObjectMeta(d.ServiceAccountName(), false), + ObjectMeta: d.getObjectMeta(d.ProvisionerServiceAccountName(), false), } }, - modify: func(d *pmemCSIDeployment, o apiruntime.Object) error { + modify: func(d *pmemCSIDeployment, o client.Object) error { // nothing to customize for service account return nil }, @@ -571,6 +610,9 @@ func (d *pmemCSIDeployment) handleEvent(ctx context.Context, metaData metav1.Obj objName := metaData.GetName() for name, handler := range subObjectHandlers { + if handler.enabled != nil && !handler.enabled(d) { + continue + } if objType != handler.objType { continue } @@ -588,13 +630,6 @@ func (d *pmemCSIDeployment) handleEvent(ctx context.Context, metaData metav1.Obj } } - if objType == v1SecretPtr { - l.V(3).Info("redeploying", "name", "driver secrets", "object", logger.KObjWithType(metaData)) - if _, err := d.redeploySecrets(ctx, r); err != nil { - return fmt.Errorf("failed to redeploy %q secrets: %v", metaData.GetName(), err) - } - } - return nil } @@ -654,19 +689,7 @@ func (d *pmemCSIDeployment) deleteObsoleteObjects(ctx context.Context, r *Reconc continue } l.V(3).Info("deleting obsolete object", "name", obj.GetName(), "gkv", obj.GetObjectKind().GroupVersionKind()) - - o, err := scheme.Scheme.New(obj.GetObjectKind().GroupVersionKind()) - if err != nil { - return err - } - metaObj, err := meta.Accessor(o) - if err != nil { - return err - } - metaObj.SetName(obj.GetName()) - metaObj.SetNamespace(obj.GetNamespace()) - - if err := r.Delete(o); err != nil && !errors.IsNotFound(err) { + if err := r.Delete(&obj); err != nil && !errors.IsNotFound(err) { return err } } @@ -674,105 +697,6 @@ func (d *pmemCSIDeployment) deleteObsoleteObjects(ctx context.Context, r *Reconc return nil } -func (d *pmemCSIDeployment) getRegistrySecrets(secret *corev1.Secret) { - d.getSecret(secret, "registry-secrets", d.Spec.CACert, d.Spec.RegistryPrivateKey, d.Spec.RegistryCert) -} - -func (d *pmemCSIDeployment) getNodeSecrets(secret *corev1.Secret) { - d.getSecret(secret, "node-secrets", d.Spec.CACert, d.Spec.NodeControllerPrivateKey, d.Spec.NodeControllerCert) -} - -func (d *pmemCSIDeployment) provisionCertificates(ctx context.Context) error { - l := logger.Get(ctx).WithName("provisionCertificates") - var prKey *rsa.PrivateKey - - l.V(3).Info("provisioning new certificates") - ca, err := pmemtls.NewCA(nil, nil) - if err != nil { - return fmt.Errorf("failed to initialize CA: %v", err) - } - d.Spec.CACert = ca.EncodedCertificate() - - if d.Spec.RegistryPrivateKey != nil { - prKey, err = pmemtls.DecodeKey(d.Spec.RegistryPrivateKey) - } else { - prKey, err = pmemtls.NewPrivateKey() - d.Spec.RegistryPrivateKey = pmemtls.EncodeKey(prKey) - } - if err != nil { - return err - } - - cert, err := ca.GenerateCertificate("pmem-registry", prKey.Public()) - if err != nil { - return fmt.Errorf("failed to generate registry certificate: %v", err) - } - d.Spec.RegistryCert = pmemtls.EncodeCert(cert) - - if d.Spec.NodeControllerPrivateKey == nil { - prKey, err = pmemtls.NewPrivateKey() - d.Spec.NodeControllerPrivateKey = pmemtls.EncodeKey(prKey) - } else { - prKey, err = pmemtls.DecodeKey(d.Spec.NodeControllerPrivateKey) - } - if err != nil { - return err - } - - cert, err = ca.GenerateCertificate("pmem-node-controller", prKey.Public()) - if err != nil { - return err - } - d.Spec.NodeControllerCert = pmemtls.EncodeCert(cert) - - // Instead of waiting for next GC cycle, initiate garbage collector manually - // so that the unneeded CA key, certificate get removed. - defer runtime.GC() - - return nil -} - -// validateCertificates ensures that the given keys and certificates are valid -// to start PMEM-CSI driver by running a mutual-tls registry server and initiating -// a tls client connection to that sever using the provided keys and certificates. -// As we use mutual-tls, testing one server is enough to make sure that the provided -// certificates works -func (d *pmemCSIDeployment) validateCertificates() error { - tmp, err := ioutil.TempDir("", "pmem-csi-validate-certs-*") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - // Registry server config - regCfg, err := pmemgrpc.ServerTLS(d.Spec.CACert, d.Spec.RegistryCert, d.Spec.RegistryPrivateKey, "pmem-node-controller") - if err != nil { - return err - } - - clientCfg, err := pmemgrpc.ClientTLS(d.Spec.CACert, d.Spec.NodeControllerCert, d.Spec.NodeControllerPrivateKey, "pmem-registry") - if err != nil { - return err - } - - // start a registry server - server := grpcserver.NewNonBlockingGRPCServer() - path := path.Join(tmp, "socket") - if err := server.Start("unix://"+path, regCfg, nil); err != nil { - return fmt.Errorf("registry certificate: %w", err) - } - defer server.ForceStop() - - conn, err := tls.Dial("unix", path, clientCfg) - if err != nil { - return fmt.Errorf("node certificate: %w", err) - } - - conn.Close() - - return nil -} - func (d *pmemCSIDeployment) getCSIDriver(csiDriver *storagev1beta1.CSIDriver) { attachRequired := false podInfoOnMount := true @@ -791,16 +715,6 @@ func (d *pmemCSIDeployment) getCSIDriver(csiDriver *storagev1beta1.CSIDriver) { } } -func (d *pmemCSIDeployment) getSecret(secret *corev1.Secret, cn string, ca, encodedKey, encodedCert []byte) { - secret.Type = corev1.SecretTypeTLS - secret.Data = map[string][]byte{ - // Same names as in the example secrets and in the v1 API. - "ca.crt": ca, // no standard name for this one - "tls.key": encodedKey, // v1.TLSPrivateKeyKey - "tls.crt": encodedCert, // v1.TLSCertKey - } -} - func (d *pmemCSIDeployment) getService(service *corev1.Service, t corev1.ServiceType, port int32) { service.Spec.Type = t if service.Spec.Ports == nil { @@ -811,7 +725,8 @@ func (d *pmemCSIDeployment) getService(service *corev1.Service, t corev1.Service IntVal: port, } service.Spec.Selector = map[string]string{ - "app": d.GetHyphenedName() + "-controller", + "app.kubernetes.io/name": "pmem-csi-controller", + "app.kubernetes.io/instance": d.Name, } } @@ -823,6 +738,136 @@ func (d *pmemCSIDeployment) getMetricsService(service *corev1.Service) { d.getService(service, corev1.ServiceTypeNodePort, controllerMetricsPort) } +func (d *pmemCSIDeployment) getWebhooksRole(role *rbacv1.Role) { + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{ + "get", "watch", "list", + }, + }, + } +} + +func (d *pmemCSIDeployment) getWebhooksRoleBinding(rb *rbacv1.RoleBinding) { + rb.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: d.WebhooksServiceAccountName(), + Namespace: d.namespace, + }, + } + rb.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: d.WebhooksRoleName(), + } +} + +func (d *pmemCSIDeployment) getWebhooksClusterRole(cr *rbacv1.ClusterRole) { + cr.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes", "nodes"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{ + "get", "list", "watch", "patch", "update", + }, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{ + "get", "list", "watch", "patch", "update", "create", + }, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses", "csinodes"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + } +} + +func (d *pmemCSIDeployment) getWebhooksClusterRoleBinding(crb *rbacv1.ClusterRoleBinding) { + crb.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: d.WebhooksServiceAccountName(), + Namespace: d.namespace, + }, + } + crb.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: d.WebhooksClusterRoleName(), + } +} + +func (d *pmemCSIDeployment) getMutatingWebhookConfig(hook *admissionregistrationv1beta1.MutatingWebhookConfiguration) { + selector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "pmem-csi.intel.com/webhook", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"ignore"}, + }, + }, + } + failurePolicy := admissionregistrationv1beta1.Ignore + if d.Spec.MutatePods == api.MutatePodsAlways { + failurePolicy = admissionregistrationv1beta1.Fail + } + path := "/pod/mutate" + hook.Webhooks = []admissionregistrationv1beta1.MutatingWebhook{ + { + // Name must be "fully-qualified" (i.e. with domain) but not unique, so + // here "pmem-csi.intel.com" is not the default driver name. + // https://pkg.go.dev/k8s.io/api/admissionregistration/v1beta1#MutatingWebhook + Name: "pod-hook.pmem-csi.intel.com", + NamespaceSelector: selector, + ObjectSelector: selector, + FailurePolicy: &failurePolicy, + ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{ + Service: &admissionregistrationv1beta1.ServiceReference{ + Name: d.SchedulerServiceName(), + Namespace: d.namespace, + Path: &path, + }, + CABundle: d.controllerCABundle, // loaded earlier in reconcile() + }, + Rules: []admissionregistrationv1beta1.RuleWithOperations{ + { + Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.Create}, + Rule: admissionregistrationv1beta1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + }, + } +} + +func (d *pmemCSIDeployment) getSchedulerService(service *corev1.Service) { + d.getService(service, corev1.ServiceTypeClusterIP, 443) + service.Spec.Ports[0].TargetPort.IntVal = schedulerPort + service.Spec.Ports[0].NodePort = d.Spec.SchedulerNodePort + if d.Spec.SchedulerNodePort != 0 { + service.Spec.Type = corev1.ServiceTypeNodePort + } +} + func (d *pmemCSIDeployment) getControllerProvisionerRole(role *rbacv1.Role) { role.Rules = []rbacv1.PolicyRule{ { @@ -870,14 +915,14 @@ func (d *pmemCSIDeployment) getControllerProvisionerRoleBinding(rb *rbacv1.RoleB rb.Subjects = []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: d.GetHyphenedName() + "-controller", + Name: d.ProvisionerServiceAccountName(), Namespace: d.namespace, }, } rb.RoleRef = rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "Role", - Name: d.GetHyphenedName() + "-external-provisioner-cfg", + Name: d.ProvisionerRoleName(), } } @@ -946,7 +991,7 @@ func (d *pmemCSIDeployment) getControllerProvisionerClusterRoleBinding(crb *rbac crb.Subjects = []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: d.ServiceAccountName(), + Name: d.ProvisionerServiceAccountName(), Namespace: d.namespace, }, } @@ -962,21 +1007,33 @@ func (d *pmemCSIDeployment) getControllerStatefulSet(ss *appsv1.StatefulSet) { true := true pmemcsiUser := int64(1000) + if ss.Labels == nil { + ss.Labels = map[string]string{} + } + ss.Labels["app.kubernetes.io/name"] = "pmem-csi-controller" + ss.Labels["app.kubernetes.io/part-of"] = "pmem-csi" + ss.Labels["app.kubernetes.io/component"] = "controller" + ss.Labels["app.kubernetes.io/instance"] = d.Name + ss.Spec = appsv1.StatefulSetSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": d.GetHyphenedName() + "-controller", + "app.kubernetes.io/name": "pmem-csi-controller", + "app.kubernetes.io/instance": d.Name, }, }, - ServiceName: d.GetHyphenedName() + "-controller", + ServiceName: d.ControllerServiceName(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: joinMaps( d.Spec.Labels, map[string]string{ - "app": d.GetHyphenedName() + "-controller", - "pmem-csi.intel.com/webhook": "ignore", + "app.kubernetes.io/name": "pmem-csi-controller", + "app.kubernetes.io/part-of": "pmem-csi", + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": d.Name, + "pmem-csi.intel.com/webhook": "ignore", }), Annotations: map[string]string{ "pmem-csi.intel.com/scrape": "containers", @@ -988,40 +1045,9 @@ func (d *pmemCSIDeployment) getControllerStatefulSet(ss *appsv1.StatefulSet) { RunAsNonRoot: &true, RunAsUser: &pmemcsiUser, }, - ServiceAccountName: d.GetHyphenedName() + "-controller", + ServiceAccountName: d.GetHyphenedName() + "-webhooks", Containers: []corev1.Container{ d.getControllerContainer(), - d.getProvisionerContainer(), - }, - Affinity: &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - // By default, the controller will run anywhere in the cluster. - // If that isn't desired, the "pmem-csi.intel.com/controller" label - // can be set to "no" or "false" for a node to prevent the controller - // from running there. - // - // This is used during testing as a workaround for a particular issue - // on Clear Linux where network configuration randomly fails such that - // the driver which runs on the same node as the controller cannot - // connect to the controller (https://github.com/intel/pmem-csi/issues/555). - // - // It may also be useful for other purposes, in particular for deployment - // through the operator: it has the same rule and currently no other API for - // setting affinity. - { - Key: "pmem-csi.intel.com/controller", - Operator: corev1.NodeSelectorOpNotIn, - Values: []string{"no", "false"}, - }, - }, - }, - }, - }, - }, }, Tolerations: []corev1.Toleration{ { @@ -1030,39 +1056,38 @@ func (d *pmemCSIDeployment) getControllerStatefulSet(ss *appsv1.StatefulSet) { Effect: "NoSchedule", }, }, - Volumes: []corev1.Volume{ - { - Name: "plugin-socket-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "registry-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: d.GetHyphenedName() + "-registry-secrets", - }, - }, - }, - { - Name: "tmp-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, }, }, } + if d.Spec.ControllerTLSSecret != "" { + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "webhook-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: d.Spec.ControllerTLSSecret, + }, + }, + }) + } } func (d *pmemCSIDeployment) getNodeDaemonSet(ds *appsv1.DaemonSet) { directoryOrCreate := corev1.HostPathDirectoryOrCreate + + if ds.Labels == nil { + ds.Labels = map[string]string{} + } + ds.Labels["app.kubernetes.io/name"] = "pmem-csi-node" + ds.Labels["app.kubernetes.io/part-of"] = "pmem-csi" + ds.Labels["app.kubernetes.io/component"] = "node" + ds.Labels["app.kubernetes.io/instance"] = d.Name + ds.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": d.GetHyphenedName() + "-node", + "app.kubernetes.io/name": "pmem-csi-node", + "app.kubernetes.io/instance": d.Name, }, }, Template: corev1.PodTemplateSpec{ @@ -1070,18 +1095,23 @@ func (d *pmemCSIDeployment) getNodeDaemonSet(ds *appsv1.DaemonSet) { Labels: joinMaps( d.Spec.Labels, map[string]string{ - "app": d.GetHyphenedName() + "-node", - "pmem-csi.intel.com/webhook": "ignore", + "app.kubernetes.io/name": "pmem-csi-node", + "app.kubernetes.io/part-of": "pmem-csi", + "app.kubernetes.io/component": "node", + "app.kubernetes.io/instance": d.Name, + "pmem-csi.intel.com/webhook": "ignore", }), Annotations: map[string]string{ "pmem-csi.intel.com/scrape": "containers", }, }, Spec: corev1.PodSpec{ - NodeSelector: d.Spec.NodeSelector, + ServiceAccountName: d.ProvisionerServiceAccountName(), + NodeSelector: d.Spec.NodeSelector, Containers: []corev1.Container{ d.getNodeDriverContainer(), d.getNodeRegistrarContainer(), + d.getProvisionerContainer(), }, Volumes: []corev1.Volume{ { @@ -1120,14 +1150,6 @@ func (d *pmemCSIDeployment) getNodeDaemonSet(ds *appsv1.DaemonSet) { }, }, }, - { - Name: "node-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: d.GetHyphenedName() + "-node-secrets", - }, - }, - }, { Name: "pmem-state-dir", VolumeSource: corev1.VolumeSource{ @@ -1162,20 +1184,28 @@ func (d *pmemCSIDeployment) getNodeDaemonSet(ds *appsv1.DaemonSet) { } func (d *pmemCSIDeployment) getControllerCommand() []string { - return []string{ + nodeSelector := types.NodeSelector(d.Spec.NodeSelector) + args := []string{ "/usr/local/bin/pmem-csi-driver", fmt.Sprintf("-v=%d", d.Spec.LogLevel), "-logging-format=" + string(d.Spec.LogFormat), - "-mode=controller", - "-endpoint=unix:///csi/csi-controller.sock", - fmt.Sprintf("-registryEndpoint=tcp://0.0.0.0:%d", controllerServicePort), - "-nodeid=$(KUBE_NODE_NAME)", - "-caFile=/certs/ca.crt", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", + "-mode=webhooks", "-drivername=$(PMEM_CSI_DRIVER_NAME)", - fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort), + "-nodeSelector=" + nodeSelector.String(), + } + + if d.Spec.ControllerTLSSecret != "" { + args = append(args, + "-caFile=/certs/ca.crt", + "-certFile=/certs/tls.crt", + "-keyFile=/certs/tls.key", + fmt.Sprintf("-schedulerListen=:%d", schedulerPort), + ) } + + args = append(args, fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort)) + + return args } func (d *pmemCSIDeployment) getNodeDriverCommand() []string { @@ -1187,12 +1217,6 @@ func (d *pmemCSIDeployment) getNodeDriverCommand() []string { "-mode=node", "-endpoint=unix:///csi/csi.sock", "-nodeid=$(KUBE_NODE_NAME)", - fmt.Sprintf("-controllerEndpoint=tcp://$(KUBE_POD_IP):%d", nodeControllerPort), - // User controller service name(== deployment name) as registry endpoint. - fmt.Sprintf("-registryEndpoint=tcp://%s-controller:%d", d.GetHyphenedName(), controllerServicePort), - "-caFile=/certs/ca.crt", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", "-statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME)", "-drivername=$(PMEM_CSI_DRIVER_NAME)", fmt.Sprintf("-pmemPercentage=%d", d.Spec.PMEMPercentage), @@ -1202,55 +1226,47 @@ func (d *pmemCSIDeployment) getNodeDriverCommand() []string { func (d *pmemCSIDeployment) getControllerContainer() corev1.Container { true := true - return corev1.Container{ + c := corev1.Container{ Name: "pmem-driver", Image: d.Spec.Image, ImagePullPolicy: d.Spec.PullPolicy, Command: d.getControllerCommand(), Env: []corev1.EnvVar{ - { - Name: "KUBE_NODE_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, { Name: "TERMINATION_LOG_PATH", - Value: "/tmp/termination-log", + Value: "/dev/termination-log", }, { Name: "PMEM_CSI_DRIVER_NAME", Value: d.GetName(), }, { - Name: "GODEBUG", - Value: "x509ignoreCN=0", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "registry-cert", - MountPath: "/certs", - }, - { - Name: "plugin-socket-dir", - MountPath: "/csi", - }, - { - Name: "tmp-dir", - MountPath: "/tmp", + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, }, }, Ports: d.getMetricsPorts(controllerMetricsPort), Resources: *d.Spec.ControllerDriverResources, - TerminationMessagePath: "/tmp/termination-log", + TerminationMessagePath: "/dev/termination-log", SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, }, } + + if d.Spec.ControllerTLSSecret != "" { + c.VolumeMounts = append(c.VolumeMounts, + corev1.VolumeMount{ + Name: "webhook-cert", + MountPath: "/certs", + }) + } + + return c } func (d *pmemCSIDeployment) getNodeDriverContainer() corev1.Container { @@ -1272,15 +1288,6 @@ func (d *pmemCSIDeployment) getNodeDriverContainer() corev1.Container { }, }, }, - { - Name: "KUBE_POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "status.podIP", - }, - }, - }, { Name: "PMEM_CSI_DRIVER_NAME", Value: d.GetName(), @@ -1289,10 +1296,6 @@ func (d *pmemCSIDeployment) getNodeDriverContainer() corev1.Container { Name: "TERMINATION_LOG_PATH", Value: "/tmp/termination-log", }, - { - Name: "GODEBUG", - Value: "x509ignoreCN=0", - }, }, VolumeMounts: []corev1.VolumeMount{ { @@ -1305,10 +1308,6 @@ func (d *pmemCSIDeployment) getNodeDriverContainer() corev1.Container { MountPath: d.Spec.KubeletDir + "/pods", MountPropagation: &bidirectional, }, - { - Name: "node-cert", - MountPath: "/certs", - }, { Name: "dev-dir", MountPath: "/dev", @@ -1348,16 +1347,30 @@ func (d *pmemCSIDeployment) getProvisionerContainer() corev1.Container { ImagePullPolicy: d.Spec.PullPolicy, Args: []string{ fmt.Sprintf("-v=%d", d.Spec.LogLevel), - "--csi-address=/csi/csi-controller.sock", + "--csi-address=/csi/csi.sock", "--feature-gates=Topology=true", + "--node-deployment=true", "--strict-topology=true", + "--immediate-topology=false", + // TODO (?): make this configurable? "--timeout=5m", "--default-fstype=ext4", fmt.Sprintf("--metrics-address=:%d", provisionerMetricsPort), }, + Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { - Name: "plugin-socket-dir", + Name: "socket-dir", MountPath: "/csi", }, }, diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go index 8c90ac6de7..56b9d1bb75 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go @@ -21,10 +21,7 @@ import ( "github.com/intel/pmem-csi/pkg/version" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" @@ -67,14 +64,14 @@ func add(ctx context.Context, mgr manager.Manager, r *ReconcileDeployment) error UpdateFunc: func(e event.UpdateEvent) bool { r.reconcileMutex.Lock() defer r.reconcileMutex.Unlock() - l.V(3).Info("UPDATED", "object", logger.KObjWithType(e.MetaOld), "generation", e.MetaNew.GetGeneration()) - if e.MetaNew.GetDeletionTimestamp() != nil { + l.V(3).Info("UPDATED", "object", logger.KObjWithType(e.ObjectOld), "generation", e.ObjectNew.GetGeneration()) + if e.ObjectNew.GetDeletionTimestamp() != nil { // Deployment CR deleted, remove it's reference from cache. // Objects owned by it are automatically garbage collected. - r.deleteDeployment(e.MetaOld.GetName()) + r.deleteDeployment(e.ObjectOld.GetName()) return false } - if e.MetaOld.GetGeneration() == e.MetaNew.GetGeneration() { + if e.ObjectOld.GetGeneration() == e.ObjectNew.GetGeneration() { // No changes registered return false } @@ -99,10 +96,10 @@ func add(ctx context.Context, mgr manager.Manager, r *ReconcileDeployment) error DeleteFunc: func(e event.DeleteEvent) bool { r.reconcileMutex.Lock() defer r.reconcileMutex.Unlock() - l.V(3).Info("DELETED", "object", logger.KObjWithType(e.Meta)) + l.V(3).Info("DELETED", "object", logger.KObjWithType(e.Object)) // Deployment CR deleted, remove it's reference from cache. // Objects owned by it are automatically garbage collected. - r.deleteDeployment(e.Meta.GetName()) + r.deleteDeployment(e.Object.GetName()) // We already handled the event here, // so no more further reconcile required. return false @@ -122,22 +119,23 @@ func add(ctx context.Context, mgr manager.Manager, r *ReconcileDeployment) error // One exception is: If we fail to handle here, then we pass this // event to reconcile loop, where it should recognize these requests // and just requeue. Expecting that the failure is retried. - eventFunc := func(what string, meta metav1.Object, obj apiruntime.Object) bool { + eventFunc := func(what string, obj client.Object) bool { // TODO: // - check that this output is okay // - check why "go test" does not cover this code - l.V(3).Info(what, "object", logger.KObjWithType(meta)) + l.V(3).Info(what, "object", logger.KObjWithType(obj)) // Get the owned deployment - d, err := r.getDeploymentFor(meta) + d, err := r.getDeploymentFor(obj) if err != nil { - l.V(3).Info("not owned by any deployment", "object", logger.KObjWithType(meta)) + l.V(3).Info("not owned by any deployment", "object", logger.KObjWithType(obj)) // The owner might have deleted already // we can safely ignore this event return false } r.reconcileMutex.Lock() defer r.reconcileMutex.Unlock() - if err := d.handleEvent(ctx, meta, obj, r); err != nil { + // TODO (?): single parameter + if err := d.handleEvent(ctx, obj, obj, r); err != nil { l.Error(err, "while handling the event, requeuing the event") return true } @@ -145,14 +143,14 @@ func add(ctx context.Context, mgr manager.Manager, r *ReconcileDeployment) error } sop := predicate.Funcs{ DeleteFunc: func(e event.DeleteEvent) bool { - return eventFunc("DELETED", e.Meta, e.Object) + return eventFunc("DELETED", e.Object) }, UpdateFunc: func(e event.UpdateEvent) bool { - if e.MetaNew.GetDeletionTimestamp() != nil { + if e.ObjectNew.GetDeletionTimestamp() != nil { // We can handle this in delete handler return false } - return eventFunc("UPDATED", e.MetaOld, e.ObjectOld) + return eventFunc("UPDATED", e.ObjectOld) }, CreateFunc: func(e event.CreateEvent) bool { // Do not handle sub-object create events as the object was create by us. @@ -256,7 +254,7 @@ func NewReconcileDeployment(ctx context.Context, client client.Client, opts pmem // Note: // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. -func (r *ReconcileDeployment) Reconcile(request reconcile.Request) (reconcile.Result, error) { +func (r *ReconcileDeployment) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { var requeue bool var err error r.reconcileMutex.Lock() @@ -265,7 +263,7 @@ func (r *ReconcileDeployment) Reconcile(request reconcile.Request) (reconcile.Re requeueDelayOnError := 2 * time.Minute l := logger.Get(r.ctx).WithValues("deployment", request.NamespacedName.Name) - ctx := logger.Set(r.ctx, l) + ctx = logger.Set(ctx, l) // Fetch the Deployment instance deployment := &api.PmemCSIDeployment{} @@ -320,7 +318,7 @@ func (r *ReconcileDeployment) Reconcile(request reconcile.Request) (reconcile.Re l.V(3).Info("reconcile done", "duration", time.Since(startTime)) }() - d := &pmemCSIDeployment{dep, r.namespace, r.k8sVersion} + d := &pmemCSIDeployment{dep, r.namespace, r.k8sVersion, []byte{}} if err := d.reconcile(ctx, r); err != nil { l.Error(err, "reconcile failed") dep.Status.Phase = api.DeploymentPhaseFailed @@ -356,21 +354,14 @@ func (r *ReconcileDeployment) RemoveHook(h ReconcileHook) { } //Get tries to retrives the Kubernetes objects -func (r *ReconcileDeployment) Get(obj runtime.Object) error { - key, err := client.ObjectKeyFromObject(obj) - if err != nil { - return fmt.Errorf("internal error %T: %v", obj, err) - } +func (r *ReconcileDeployment) Get(obj client.Object) error { + key := client.ObjectKeyFromObject(obj) return r.client.Get(r.ctx, key, obj) } // Delete delete existing Kubernetes object -func (r *ReconcileDeployment) Delete(obj runtime.Object) error { - metaObj, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("internal error %T: %v", obj, err) - } - logger.Get(r.ctx).Info("deleting", "object", logger.KObjWithType(metaObj)) +func (r *ReconcileDeployment) Delete(obj client.Object) error { + logger.Get(r.ctx).Info("deleting", "object", logger.KObjWithType(obj)) return r.client.Delete(r.ctx, obj) } @@ -423,7 +414,7 @@ func (r *ReconcileDeployment) getDeploymentFor(obj metav1.Object) (*pmemCSIDeplo if err := deployment.EnsureDefaults(r.containerImage); err != nil { return nil, err } - return &pmemCSIDeployment{deployment, r.namespace, r.k8sVersion}, nil + return &pmemCSIDeployment{deployment, r.namespace, r.k8sVersion, []byte{}}, nil } } diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index f3fe47d3b3..37a8651127 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -8,7 +8,9 @@ package deployment_test import ( "context" "fmt" + "sort" "strings" + "sync" "testing" "time" @@ -23,7 +25,6 @@ import ( pmemcontroller "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment/testcases" - pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" "github.com/intel/pmem-csi/pkg/version" "github.com/intel/pmem-csi/test/e2e/operator/validate" @@ -46,6 +47,7 @@ import ( ) type pmemDeployment struct { + // input parameters for test name string deviceMode string logLevel uint16 @@ -55,8 +57,15 @@ type pmemDeployment struct { nodeCPU, nodeMemory string provisionerCPU, provisionerMemory string nodeRegistarCPU, nodeRegistrarMemory string - caCert, regCert, regKey, ncCert, ncKey []byte + controllerTLSSecret string + mutatePods api.MutatePods + schedulerNodePort int32 kubeletDir string + + objects []runtime.Object + + // expected result + expectFailure bool } func getDeployment(d *pmemDeployment) *api.PmemCSIDeployment { @@ -74,15 +83,19 @@ func getDeployment(d *pmemDeployment) *api.PmemCSIDeployment { // TODO (?): embed DeploymentSpec inside pmemDeployment instead of splitting it up into individual values. // The entire copying block below then collapses into a single line. - dep.Spec = api.DeploymentSpec{} + dep.Spec = api.DeploymentSpec{ + DeviceMode: api.DeviceMode(d.deviceMode), + LogLevel: d.logLevel, + LogFormat: api.LogFormat(d.logFormat), + Image: d.image, + PullPolicy: corev1.PullPolicy(d.pullPolicy), + ProvisionerImage: d.provisionerImage, + NodeRegistrarImage: d.registrarImage, + ControllerTLSSecret: d.controllerTLSSecret, + MutatePods: d.mutatePods, + SchedulerNodePort: d.schedulerNodePort, + } spec := &dep.Spec - spec.DeviceMode = api.DeviceMode(d.deviceMode) - spec.LogLevel = d.logLevel - spec.LogFormat = api.LogFormat(d.logFormat) - spec.Image = d.image - spec.PullPolicy = corev1.PullPolicy(d.pullPolicy) - spec.ProvisionerImage = d.provisionerImage - spec.NodeRegistrarImage = d.registrarImage if d.controllerCPU != "" || d.controllerMemory != "" { spec.ControllerDriverResources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -115,11 +128,6 @@ func getDeployment(d *pmemDeployment) *api.PmemCSIDeployment { }, } } - spec.CACert = d.caCert - spec.RegistryCert = d.regCert - spec.RegistryPrivateKey = d.regKey - spec.NodeControllerCert = d.ncCert - spec.NodeControllerPrivateKey = d.ncKey if d.kubeletDir != "" { spec.KubeletDir = d.kubeletDir } @@ -127,44 +135,44 @@ func getDeployment(d *pmemDeployment) *api.PmemCSIDeployment { return dep } -func testDeploymentPhase(t *testing.T, c client.Client, name string, expectedPhase api.DeploymentPhase) { +func (tc *testContext) testDeploymentPhase(name string, expectedPhase api.DeploymentPhase) { depObject := &api.PmemCSIDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } - err := c.Get(context.TODO(), namespacedNameWithOffset(t, 3, depObject), depObject) - require.NoError(t, err, "failed to retrive deployment object") - require.Equal(t, expectedPhase, depObject.Status.Phase, "Unexpected status phase") + err := tc.c.Get(tc.ctx, tc.namespacedNameWithOffset(3, depObject), depObject) + require.NoError(tc.t, err, "failed to retrive deployment object") + require.Equal(tc.t, expectedPhase, depObject.Status.Phase, "Unexpected status phase") } -func testReconcile(t *testing.T, rc reconcile.Reconciler, name string, expectErr bool, expectedRequeue bool) { +func (tc *testContext) testReconcile(name string, expectErr bool, expectedRequeue bool) { req := reconcile.Request{ NamespacedName: types.NamespacedName{ Name: name, }, } - resp, err := rc.Reconcile(req) + resp, err := tc.rc.Reconcile(tc.ctx, req) if expectErr { - require.Error(t, err, "expected reconcile failure") + require.Error(tc.t, err, "expected reconcile failure") } else { - require.NoError(t, err, "reconcile failed with error") + require.NoError(tc.t, err, "reconcile failed with error") } - require.Equal(t, expectedRequeue, resp.Requeue, "expected requeue reconcile") + require.Equal(tc.t, expectedRequeue, resp.Requeue, "expected requeue reconcile") } -func testReconcilePhase(t *testing.T, rc reconcile.Reconciler, c client.Client, name string, expectErr bool, expectedRequeue bool, expectedPhase api.DeploymentPhase) { - testReconcile(t, rc, name, expectErr, expectedRequeue) - testDeploymentPhase(t, c, name, expectedPhase) +func (tc *testContext) testReconcilePhase(name string, expectErr bool, expectedRequeue bool, expectedPhase api.DeploymentPhase) { + tc.testReconcile(name, expectErr, expectedRequeue) + tc.testDeploymentPhase(name, expectedPhase) } -func namespacedName(t *testing.T, obj runtime.Object) types.NamespacedName { - return namespacedNameWithOffset(t, 2, obj) +func (tc *testContext) namespacedName(t *testing.T, obj runtime.Object) types.NamespacedName { + return tc.namespacedNameWithOffset(2, obj) } -func namespacedNameWithOffset(t *testing.T, offset int, obj runtime.Object) types.NamespacedName { +func (tc *testContext) namespacedNameWithOffset(offset int, obj runtime.Object) types.NamespacedName { metaObj, err := meta.Accessor(obj) - require.NoError(t, err, "failed to get accessor") + require.NoError(tc.t, err, "failed to get accessor") return types.NamespacedName{Name: metaObj.GetName(), Namespace: metaObj.GetNamespace()} } @@ -215,17 +223,25 @@ type testContext struct { cs kubernetes.Interface rc reconcile.Reconciler evWatcher watch.Interface - events []*corev1.Event resourceVersions map[string]string k8sVersion version.Version + + eventsMutex sync.Mutex + events []corev1.Event } -func newTestContext(t *testing.T, k8sVersion version.Version) *testContext { +func newTestContext(t *testing.T, k8sVersion version.Version, initObjs ...runtime.Object) *testContext { + // Make a copy of the initial objects, just to be on the safe side. + var objs []runtime.Object + + for _, obj := range initObjs { + objs = append(objs, obj.DeepCopyObject()) + } ctx := logger.Set(context.Background(), testinglogger.New(t)) tc := &testContext{ ctx: ctx, t: t, - c: newTestClient(), + c: newTestClient(objs...), cs: cgfake.NewSimpleClientset(), resourceVersions: map[string]string{}, k8sVersion: k8sVersion, @@ -247,14 +263,9 @@ func (tc *testContext) ResetReconciler() { tc.rc = rc tc.UnsetEventWatcher() tc.evWatcher = rc.(*deployment.ReconcileDeployment).EventBroadcaster().StartEventWatcher(func(ev *corev1.Event) { - // Discard consecutive duplicate events, mimicking the EventAggregator behavior - if len(tc.events) != 0 { - lastEvent := tc.events[len(tc.events)-1] - if lastEvent.Reason == ev.Reason && lastEvent.InvolvedObject.UID == ev.InvolvedObject.UID { - return - } - } - tc.events = append(tc.events, ev) + tc.eventsMutex.Lock() + defer tc.eventsMutex.Unlock() + tc.events = append(tc.events, *ev) }) } @@ -264,13 +275,27 @@ func (tc *testContext) UnsetEventWatcher() { } } +func createSecret(name, namespace string, data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } +} + func TestDeploymentController(t *testing.T) { err := apis.AddToScheme(scheme.Scheme) require.NoError(t, err, "add api schema") testIt := func(t *testing.T, testK8sVersion version.Version) { - setup := func(t *testing.T) *testContext { - return newTestContext(t, testK8sVersion) + setup := func(t *testing.T, initObjs ...runtime.Object) *testContext { + return newTestContext(t, testK8sVersion, initObjs...) } teardown := func(tc *testContext) { @@ -279,15 +304,29 @@ func TestDeploymentController(t *testing.T) { validateEvents := func(tc *testContext, dep *api.PmemCSIDeployment, expectedEvents []string) { require.Eventually(tc.t, func() bool { - return len(tc.events) >= len(expectedEvents) - }, 30*time.Second, time.Second, "receive all expected events") - events := []string{} - for _, e := range tc.events { - if e.InvolvedObject.UID == dep.GetUID() { - events = append(events, e.Reason) + tc.eventsMutex.Lock() + defer tc.eventsMutex.Unlock() + + // Before comparing against expected + // events, we must sort by the "first + // seen", time stamp because events + // may get delivered out-of-order. + sort.Slice(tc.events, func(i, j int) bool { + return tc.events[i].FirstTimestamp.Before(&tc.events[j].FirstTimestamp) + }) + + // Then we need to filter out events for the + // right deployment and remove duplicates. + events := []string{} + for _, e := range tc.events { + if e.InvolvedObject.UID == dep.GetUID() && + (len(events) == 0 || events[len(events)-1] != e.Reason) { + events = append(events, e.Reason) + } } - } - require.ElementsMatch(tc.t, events, expectedEvents, "events must match") + require.Equal(tc.t, events, expectedEvents, "events must match") + return true + }, 30*time.Second, time.Second, "receive all expected events") } validateConditions := func(tc *testContext, name string, expected map[api.DeploymentConditionType]corev1.ConditionStatus) { @@ -324,30 +363,17 @@ func TestDeploymentController(t *testing.T) { t.Parallel() - t.Run("deployment with defaults", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - d := &pmemDeployment{ - name: "test-deployment", - } - - dep := getDeployment(d) - - err := tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, - }) - }) + dataOkay := map[string][]byte{ + api.TLSSecretCA: []byte("ca"), + api.TLSSecretKey: []byte("key"), + api.TLSSecretCert: []byte("cert"), + } - t.Run("deployment with explicit values", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - d := &pmemDeployment{ + cases := map[string]pmemDeployment{ + "deployment with defaults": pmemDeployment{ + name: "test-deployment", + }, + "deployment with explicit values": pmemDeployment{ name: "test-deployment", image: "test-driver:v0.0.0", provisionerImage: "test-provisioner-image:v0.0.0", @@ -360,20 +386,85 @@ func TestDeploymentController(t *testing.T) { nodeCPU: "1000m", nodeMemory: "500Mi", kubeletDir: "/some/directory", - } + }, + "invalid device mode": pmemDeployment{ + name: "test-driver-modes", + deviceMode: "foobar", + expectFailure: true, + }, + "LVM mode": pmemDeployment{ + name: "test-driver-modes", + deviceMode: "lvm", + }, + "direct mode": pmemDeployment{ + name: "test-driver-modes", + deviceMode: "direct", + }, + "with controller, no secret": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + expectFailure: true, + }, + "with controller, wrong secret content": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + objects: []runtime.Object{createSecret("controller-secret", testNamespace, nil)}, + expectFailure: true, + }, + "with controller, secret okay": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, + }, + "controller, no mutate": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + mutatePods: api.MutatePodsNever, + objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, + }, + "controller, try mutate": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + mutatePods: api.MutatePodsTry, + objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, + }, + "controller, always mutate": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + mutatePods: api.MutatePodsAlways, + objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, + }, + "controller, port 31000": pmemDeployment{ + name: "test-controller", + controllerTLSSecret: "controller-secret", + schedulerNodePort: 31000, + objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, + }, + } - dep := getDeployment(d) - err := tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") + for name, d := range cases { + d := d + t.Run(name, func(t *testing.T) { + tc := setup(t, d.objects...) + defer teardown(tc) + dep := getDeployment(&d) - // Reconcile now should change Phase to running - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, + err := tc.c.Create(tc.ctx, dep) + require.NoError(t, err, "failed to create deployment") + + if d.expectFailure { + tc.testReconcilePhase(d.name, true, true, api.DeploymentPhaseFailed) + validateEvents(tc, dep, []string{api.EventReasonNew, api.EventReasonFailed}) + validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{}) + } else { + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) + validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) + validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ + api.DriverDeployed: corev1.ConditionTrue, + }) + } }) - }) + } t.Run("multiple deployments", func(t *testing.T) { tc := setup(t) @@ -395,213 +486,17 @@ func TestDeploymentController(t *testing.T) { require.NoError(t, err, "failed to create deployment2") conditions := map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, } - testReconcilePhase(t, tc.rc, tc.c, d1.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d1.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep1, []string{api.EventReasonNew, api.EventReasonRunning}, false) validateConditions(tc, d1.name, conditions) - testReconcilePhase(t, tc.rc, tc.c, d2.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d2.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep2, []string{api.EventReasonNew, api.EventReasonRunning}, false) validateConditions(tc, d2.name, conditions) }) - t.Run("invalid device mode", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - d := &pmemDeployment{ - name: "test-driver-modes", - deviceMode: "foobar", - } - - dep := getDeployment(d) - - err := tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - // Deployment should failed with an error - testReconcilePhase(t, tc.rc, tc.c, d.name, true, true, api.DeploymentPhaseFailed) - validateEvents(tc, dep, []string{api.EventReasonNew, api.EventReasonFailed}) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{}) - }) - - t.Run("LVM mode", func(t *testing.T) { - tc := setup(t) - d := &pmemDeployment{ - name: "test-driver-modes", - deviceMode: "lvm", - } - - dep := getDeployment(d) - - err := tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, - }) - }) - - t.Run("direct mode", func(t *testing.T) { - tc := setup(t) - d := &pmemDeployment{ - name: "test-driver-modes", - deviceMode: "direct", - } - - dep := getDeployment(d) - - err := tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, - }) - }) - - t.Run("provided private keys", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - // Generate private key - regKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "Failed to generate a private key: %v", err) - - encodedKey := pmemtls.EncodeKey(regKey) - - d := &pmemDeployment{ - name: "test-deployment", - regKey: encodedKey, - } - dep := getDeployment(d) - err = tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - - // First deployment expected to be successful - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, - }) - }) - - t.Run("provided private keys and certificates", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - ca, err := pmemtls.NewCA(nil, nil) - require.NoError(t, err, "failed to instantiate CA") - - regKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - regCert, err := ca.GenerateCertificate("pmem-registry", regKey.Public()) - require.NoError(t, err, "failed to sign registry key") - - ncKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - ncCert, err := ca.GenerateCertificate("pmem-node-controller", ncKey.Public()) - require.NoError(t, err, "failed to sign node controller key") - - d := &pmemDeployment{ - name: "test-deployment", - caCert: ca.EncodedCertificate(), - regKey: pmemtls.EncodeKey(regKey), - regCert: pmemtls.EncodeCert(regCert), - ncKey: pmemtls.EncodeKey(ncKey), - ncCert: pmemtls.EncodeCert(ncCert), - } - dep := getDeployment(d) - err = tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - - // First deployment expected to be successful - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) - validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.CertsVerified: corev1.ConditionTrue, - api.DriverDeployed: corev1.ConditionTrue, - }) - }) - - t.Run("invalid private keys and certificates", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - ca, err := pmemtls.NewCA(nil, nil) - require.NoError(t, err, "faield to instantiate CA") - - regKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - regCert, err := ca.GenerateCertificate("invalid-registry", regKey.Public()) - require.NoError(t, err, "failed to sign registry key") - - ncKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - ncCert, err := ca.GenerateCertificate("invalid-node-controller", ncKey.Public()) - require.NoError(t, err, "failed to sign node key") - - d := &pmemDeployment{ - name: "test-deployment-cert-invalid", - caCert: ca.EncodedCertificate(), - regKey: pmemtls.EncodeKey(regKey), - regCert: pmemtls.EncodeCert(regCert), - ncKey: pmemtls.EncodeKey(ncKey), - ncCert: pmemtls.EncodeCert(ncCert), - } - dep := getDeployment(d) - err = tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - - testReconcilePhase(t, tc.rc, tc.c, d.name, true, true, api.DeploymentPhaseFailed) - validateEvents(tc, dep, []string{api.EventReasonNew, api.EventReasonFailed}) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsVerified: corev1.ConditionFalse, - api.DriverDeployed: corev1.ConditionFalse, - }) - }) - - t.Run("expired certificates", func(t *testing.T) { - tc := setup(t) - defer teardown(tc) - oneDayAgo := time.Now().Add(-24 * time.Hour) - oneMinuteAgo := time.Now().Add(-1 * time.Minute) - - ca, err := pmemtls.NewCA(nil, nil) - require.NoError(t, err, "faield to instantiate CA") - - regKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - regCert, err := ca.GenerateCertificateWithDuration("pmem-registry", oneDayAgo, oneMinuteAgo, regKey.Public()) - require.NoError(t, err, "failed to registry sign key") - - ncKey, err := pmemtls.NewPrivateKey() - require.NoError(t, err, "failed to generate a private key: %v", err) - ncCert, err := ca.GenerateCertificateWithDuration("pmem-node-controller", oneDayAgo, oneMinuteAgo, ncKey.Public()) - require.NoError(t, err, "failed to sign node controller key") - - d := &pmemDeployment{ - name: "test-deployment-cert-expired", - caCert: ca.EncodedCertificate(), - regKey: pmemtls.EncodeKey(regKey), - regCert: pmemtls.EncodeCert(regCert), - ncKey: pmemtls.EncodeKey(ncKey), - ncCert: pmemtls.EncodeCert(ncCert), - } - dep := getDeployment(d) - err = tc.c.Create(tc.ctx, dep) - require.NoError(t, err, "failed to create deployment") - - testReconcilePhase(t, tc.rc, tc.c, d.name, true, true, api.DeploymentPhaseFailed) - validateEvents(tc, dep, []string{api.EventReasonNew, api.EventReasonFailed}) - validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsVerified: corev1.ConditionFalse, - api.DriverDeployed: corev1.ConditionFalse, - }) - }) - t.Run("modified deployment under reconcile", func(t *testing.T) { tc := setup(t) defer teardown(tc) @@ -625,18 +520,17 @@ func TestDeploymentController(t *testing.T) { tc.rc.(*deployment.ReconcileDeployment).AddHook(&hook) conditions := map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, } - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) validateConditions(tc, d.name, conditions) tc.rc.(*deployment.ReconcileDeployment).RemoveHook(&hook) // Next reconcile phase should catch the deployment changes - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, updatedDep, []string{api.EventReasonNew, api.EventReasonRunning}, true) validateConditions(tc, d.name, conditions) }) @@ -653,24 +547,19 @@ func TestDeploymentController(t *testing.T) { // Assumption is that all the testcases are positive cases. conditions := map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, } - if yes, _ := dep.HaveCertificatesConfigured(); yes { - conditions[api.CertsVerified] = corev1.ConditionTrue - } - // When working with the fake client, we need to make up a UID. dep.UID = types.UID("fake-uid-" + dep.Name) err := tc.c.Create(tc.ctx, dep) require.NoError(t, err, "create deployment") - testReconcilePhase(t, tc.rc, tc.c, dep.Name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(dep.Name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) // Reconcile now should keep phase as running. - testReconcilePhase(t, tc.rc, tc.c, dep.Name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(dep.Name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) validateEvents(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}) validateConditions(tc, dep.Name, conditions) @@ -690,7 +579,7 @@ func TestDeploymentController(t *testing.T) { require.NoError(t, err, "update deployment") // Reconcile is expected to not fail. - testReconcilePhase(t, tc.rc, tc.c, dep.Name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(dep.Name, false, false, api.DeploymentPhaseRunning) // Recheck the container resources are updated validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, true) @@ -720,10 +609,9 @@ func TestDeploymentController(t *testing.T) { err := tc.c.Create(tc.ctx, dep) require.NoError(t, err, "failed to create deployment") - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) validateConditions(tc, d.name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, }) @@ -765,7 +653,7 @@ func TestDeploymentController(t *testing.T) { tc.ResetReconciler() // A fresh reconcile should delete the newly created above ConfigMap - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) err = tc.c.Get(tc.ctx, client.ObjectKey{Name: d.name}, dep) require.NoError(t, err, "get deployment") // It is debatable whether the operator should update all objects after @@ -810,11 +698,11 @@ func TestDeploymentController(t *testing.T) { tc.c.(*testClient).InjectPanicOn(nil) // mimic operator restart tc.ResetReconciler() - testReconcilePhase(t, tc.rc, tc.c, d.name, false, false, api.DeploymentPhaseRunning) + tc.testReconcilePhase(d.name, false, false, api.DeploymentPhaseRunning) validateDriver(tc, dep, []string{api.EventReasonNew, api.EventReasonRunning}, false) }() - tc.rc.Reconcile(req) + tc.rc.Reconcile(tc.ctx, req) } }) } @@ -834,6 +722,11 @@ func TestDeploymentController(t *testing.T) { } } +// patchMutex is used to serialize the patch operation because +// of concurrency issues in controller-runtime and/or json-iterator +// (https://github.com/intel/pmem-csi/issues/852). +var patchMutex sync.Mutex + type testClient struct { client.Client assertOn *schema.GroupVersionKind @@ -850,9 +743,15 @@ func (t *testClient) InjectPanicOn(gvk *schema.GroupVersionKind) { // Create adds given obj to its object tracking list. // It panics if the object type matches with the type of 'assertOn' // that was previously set using InjectPanicOn() -func (t *testClient) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error { +func (t *testClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { if t.assertOn != nil && obj.GetObjectKind().GroupVersionKind() == *t.assertOn { panic(fmt.Sprintf("assert: %v", obj.GetObjectKind())) } return t.Client.Create(ctx, obj, opts...) } + +func (t *testClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + patchMutex.Lock() + defer patchMutex.Unlock() + return t.Client.Patch(ctx, obj, patch, opts...) +} diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index c0562b7965..5e29efeb20 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -12,7 +12,6 @@ import ( "fmt" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" - pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -73,9 +72,6 @@ func UpdateTests() []UpdateTest { }, } }, - "TLS": func(d *api.PmemCSIDeployment) { - SetTLSOrDie(&d.Spec) - }, "logLevel": func(d *api.PmemCSIDeployment) { d.Spec.LogLevel++ }, @@ -166,7 +162,6 @@ func UpdateTests() []UpdateTest { }, }, } - SetTLSOrDie(&full.Spec) baseDeployments := map[string]api.PmemCSIDeployment{ "default deployment": { @@ -202,43 +197,3 @@ func UpdateTests() []UpdateTest { return tests } - -func SetTLSOrDie(spec *api.DeploymentSpec) { - err := SetTLS(spec) - if err != nil { - panic(err) - } -} - -func SetTLS(spec *api.DeploymentSpec) error { - ca, err := pmemtls.NewCA(nil, nil) - if err != nil { - return fmt.Errorf("instantiate CA: %v", err) - } - - regKey, err := pmemtls.NewPrivateKey() - if err != nil { - return fmt.Errorf("generate a private key: %v", err) - } - regCert, err := ca.GenerateCertificate("pmem-registry", regKey.Public()) - if err != nil { - return fmt.Errorf("sign registry key: %v", err) - } - - ncKey, err := pmemtls.NewPrivateKey() - if err != nil { - return fmt.Errorf("generate a private key: %v", err) - } - ncCert, err := ca.GenerateCertificate("pmem-node-controller", ncKey.Public()) - if err != nil { - return fmt.Errorf("sign node controller key: %v", err) - } - - spec.CACert = ca.EncodedCertificate() - spec.RegistryPrivateKey = pmemtls.EncodeKey(regKey) - spec.RegistryCert = pmemtls.EncodeCert(regCert) - spec.NodeControllerPrivateKey = pmemtls.EncodeKey(ncKey) - spec.NodeControllerCert = pmemtls.EncodeCert(ncCert) - - return nil -} diff --git a/pkg/pmem-csi-operator/pmem-tls/tls.go b/pkg/pmem-csi-operator/pmem-tls/tls.go deleted file mode 100644 index 1e694b66f2..0000000000 --- a/pkg/pmem-csi-operator/pmem-tls/tls.go +++ /dev/null @@ -1,251 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package pmemtls - -import ( - "crypto" - "errors" - "math" - "runtime" - "time" - - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "math/big" -) - -const ( - rasKeySize = 3072 -) - -// NewPrivateKey generate an rsa private key -func NewPrivateKey() (*rsa.PrivateKey, error) { - key, err := rsa.GenerateKey(rand.Reader, rasKeySize) - if err != nil { - return nil, err - } - - runtime.SetFinalizer(key, func(k *rsa.PrivateKey) { - // Zero key after usage - *k = rsa.PrivateKey{} - }) - return key, nil -} - -// EncodeKey returns PEM encoding of give private key -func EncodeKey(key *rsa.PrivateKey) []byte { - if key == nil { - return []byte{} - } - return pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(key), - }) -} - -// DecodeKey returns the decoded private key of given encodedKey -func DecodeKey(encodedKey []byte) (*rsa.PrivateKey, error) { - block, _ := pem.Decode(encodedKey) - - key, err := x509.ParsePKCS1PrivateKey(block.Bytes) - wipe(block.Bytes) - if err != nil { - return nil, err - } - - runtime.SetFinalizer(key, func(k *rsa.PrivateKey) { - // Zero key after usage - *k = rsa.PrivateKey{} - }) - - return key, nil -} - -// EncodeCert returns PEM encoding of given cert -func EncodeCert(cert *x509.Certificate) []byte { - if cert == nil { - return []byte{} - } - return pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Raw, - }) -} - -// DecodeCert return the decoded certificate of given encodedCert -func DecodeCert(encodedCert []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(encodedCert) - - cert, err := x509.ParseCertificate(block.Bytes) - wipe(block.Bytes) - - if err != nil { - return nil, err - } - runtime.SetFinalizer(cert, func(c *x509.Certificate) { - *c = x509.Certificate{} - }) - - return cert, nil -} - -func wipe(arr []byte) { - for i := range arr { - arr[i] = 0 - } -} - -// CA type representation for a self-signed certificate authority -type CA struct { - prKey *rsa.PrivateKey - cert *x509.Certificate -} - -// NewCA creates a new CA object for given CA certificate and private key. -// If both of caCert and key are nil, generates a new private key and -// a self-signed certificate -func NewCA(caCert *x509.Certificate, key *rsa.PrivateKey) (*CA, error) { - var err error - prKey := key - cert := caCert - if cert == nil { - if prKey == nil { - prKey, err = NewPrivateKey() - if err != nil { - return nil, err - } - } - - cert, err = NewCACertificate(prKey) - if err != nil { - return nil, err - } - } else if prKey == nil { - return nil, errors.New("certificate is provided but not the associated private key is missing") - } else { - requiredKeyUsages := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign - if cert.KeyUsage&requiredKeyUsages != requiredKeyUsages { - return nil, errors.New("provided certificates can not be used as CA certificate as" + - " is not usable for encrypting or signing other keys") - } - - if cert.IsCA != true { - return nil, errors.New("provided certificate is not a ca certificate") - } - } - - ca := &CA{ - prKey: prKey, - cert: cert, - } - return ca, nil -} - -// PrivateKey returns private key used -func (ca *CA) PrivateKey() *rsa.PrivateKey { - return ca.prKey -} - -// Certificate returns root ca certificate used -func (ca *CA) Certificate() *x509.Certificate { - return ca.cert -} - -// EncodedKey returns encoded private key used -func (ca *CA) EncodedKey() []byte { - return EncodeKey(ca.prKey) -} - -// EncodedCertificate returns encoded root ca certificate used -func (ca *CA) EncodedCertificate() []byte { - return EncodeCert(ca.cert) -} - -// GenerateCertificate returns a new certificate signed for given public key. -func (ca *CA) GenerateCertificate(cn string, key crypto.PublicKey) (*x509.Certificate, error) { - return ca.generateCertificate(cn, ca.cert.NotBefore, time.Now().Add(time.Hour*24*365), key) -} - -// GenerateCertificateWithDuration returns a new certificate signed for given public key. -// The duration of this certificate is with in the given notBefore and notAfter bounds. -// Intended use of this API is only by tests -func (ca *CA) GenerateCertificateWithDuration(cn string, notBefore, notAfter time.Time, key crypto.PublicKey) (*x509.Certificate, error) { - return ca.generateCertificate(cn, notAfter, notAfter, key) -} - -// NewCACertificate returns a self-signed certificate used as certificate authority -func NewCACertificate(key *rsa.PrivateKey) (*x509.Certificate, error) { - max := new(big.Int).SetInt64(math.MaxInt64) - serial, err := rand.Int(rand.Reader, max) - if err != nil { - return nil, err - } - tmpl := &x509.Certificate{ - Version: tls.VersionTLS12, - SerialNumber: serial, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365).UTC(), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - IsCA: true, - BasicConstraintsValid: true, - DNSNames: []string{"pmem-csi", "ca"}, - } - certBytes, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) - *tmpl = x509.Certificate{} - if err != nil { - return nil, err - } - - cert, err := x509.ParseCertificate(certBytes) - if err != nil { - return nil, err - } - - runtime.SetFinalizer(cert, func(c *x509.Certificate) { - *c = x509.Certificate{} - }) - - return cert, nil -} - -func (ca *CA) generateCertificate(cn string, notBefore, notAfter time.Time, key crypto.PublicKey) (*x509.Certificate, error) { - max := new(big.Int).SetInt64(math.MaxInt64) - serial, err := rand.Int(rand.Reader, max) - if err != nil { - return nil, err - } - - tmpl := &x509.Certificate{ - Version: tls.VersionTLS12, - SerialNumber: serial, - NotBefore: notBefore, - NotAfter: notAfter, - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - DNSNames: []string{cn}, - } - - certBytes, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, key, ca.prKey) - *tmpl = x509.Certificate{} - if err != nil { - return nil, err - } - - cert, err := x509.ParseCertificate(certBytes) - if err != nil { - return nil, err - } - - runtime.SetFinalizer(cert, func(c *x509.Certificate) { - *c = x509.Certificate{} - }) - - return cert, nil -} diff --git a/pkg/pmem-csi-operator/pmem-tls/tls_test.go b/pkg/pmem-csi-operator/pmem-tls/tls_test.go deleted file mode 100644 index 497f9ff919..0000000000 --- a/pkg/pmem-csi-operator/pmem-tls/tls_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -SPDX-License-Identifier: Apache-2.0 -*/ -package pmemtls_test - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "math" - "math/big" - "testing" - "time" - - pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" - "github.com/stretchr/testify/assert" -) - -func generateSelfSignedCertificate(key *rsa.PrivateKey, keyUsage x509.KeyUsage, isCA bool) (*x509.Certificate, error) { - max := new(big.Int).SetInt64(math.MaxInt64) - serial, err := rand.Int(rand.Reader, max) - if err != nil { - return nil, err - } - tmpl := &x509.Certificate{ - Version: tls.VersionTLS12, - SerialNumber: serial, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365), - KeyUsage: keyUsage, - IsCA: isCA, - BasicConstraintsValid: true, - Subject: pkix.Name{ - CommonName: "test root certificate authority", - }, - } - certBytes, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) - if err != nil { - return nil, err - } - - return x509.ParseCertificate(certBytes) -} - -func TestPmemTLS(t *testing.T) { - t.Run("key", func(t *testing.T) { - // create keys - key, err := pmemtls.NewPrivateKey() - assert.Empty(t, err, "Key creation failed with error: %v", err) - assert.NotEmpty(t, key, "nil key") - - // Encode-decode - bytes := pmemtls.EncodeKey(key) - assert.NotEqual(t, 0, len(bytes), "Zero length") - decodedKey, err := pmemtls.DecodeKey(bytes) - assert.Empty(t, err, "Failed to decode key: %v", err) - assert.Equal(t, key, decodedKey, "Mismatched key after decode: %+v", decodedKey) - }) - - t.Run("ca with defaults", func(t *testing.T) { - ca, err := pmemtls.NewCA(nil, nil) - - assert.Empty(t, err, "CA creation with defaults failed", err) - assert.NotEmpty(t, ca, "nil CA") - assert.NotEmpty(t, ca.PrivateKey(), "CA: empty private key") - assert.NotEmpty(t, ca.Certificate(), "CA: empty root certificate key") - }) - - t.Run("ca with invalid arguments", func(t *testing.T) { - cakey, err := pmemtls.NewPrivateKey() - assert.Empty(t, err, "failed to create new key") - - keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature - cacert, err := generateSelfSignedCertificate(cakey, keyUsage, true) - - _, err = pmemtls.NewCA(cacert, cakey) - assert.NotEmpty(t, err, "expected an error when no private key provided") - - keyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign - cacert, err = generateSelfSignedCertificate(cakey, keyUsage, false) - - _, err = pmemtls.NewCA(cacert, nil) - assert.NotEmpty(t, err, "expected an error when no private key provided") - - _, err = pmemtls.NewCA(cacert, cakey) - assert.NotEmpty(t, err, "expected an error when provided certificate is not for CA") - }) - - t.Run("ca with provided private key", func(t *testing.T) { - cakey, err := rsa.GenerateKey(rand.Reader, 1024) - assert.Empty(t, err, "failed to create new key") - - encKey := pmemtls.EncodeKey(cakey) - assert.NotEmpty(t, encKey, "Encoding key failed") - - ca, err := pmemtls.NewCA(nil, cakey) - assert.Empty(t, err, "CA creation with pre-provisioned key failed") - assert.NotEmpty(t, ca, "nil CA") - assert.Equal(t, cakey, ca.PrivateKey(), "CA: mismatched private key") - assert.Equal(t, encKey, ca.EncodedKey(), "CA: mismatched encoded key") - assert.NotEmpty(t, ca.Certificate(), "CA: empty root certificate key") - - prKey, err := pmemtls.NewPrivateKey() - assert.Empty(t, err, "Failed to create private key") - - cert, err := ca.GenerateCertificate("Some name", prKey.Public()) - assert.Empty(t, err, "Failed to sign certificate") - assert.NotEmpty(t, cert, "Generated certificate is empty") - }) - - t.Run("ca with provided certificate and key", func(t *testing.T) { - cakey, err := pmemtls.NewPrivateKey() - assert.Empty(t, err, "failed to create new key") - - keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign - cacert, err := generateSelfSignedCertificate(cakey, keyUsage, true) - - ca, err := pmemtls.NewCA(cacert, cakey) - assert.Empty(t, err, "CA creation with pre-provisioned key failed") - assert.NotEmpty(t, ca, "nil CA") - assert.Equal(t, cakey, ca.PrivateKey(), "CA: mismatched private key") - assert.Equal(t, cacert, ca.Certificate(), "CA: empty root certificate key") - - prKey, err := pmemtls.NewPrivateKey() - assert.Empty(t, err, "Failed to create private key") - - // CA signing truncates noano seconds - validity := time.Now().Add(time.Hour * 24 * 365).UTC().Truncate(time.Second) - - cert, err := ca.GenerateCertificate("test-cert", prKey.Public()) - assert.Empty(t, err, "Failed to sign certificate") - assert.NotEmpty(t, cert, "Generated certificate is empty") - - isValid := cert.NotAfter.Equal(validity) || cert.NotAfter.After(validity) - assert.Equal(t, isValid, true, "invalid certificate validity(%v) expected least %v", cert.NotAfter, validity) - - assert.Contains(t, cert.DNSNames, "test-cert", "mismatched common name") - }) -} diff --git a/pkg/pmem-device-manager/metrics.go b/pkg/pmem-device-manager/metrics.go new file mode 100644 index 0000000000..8e0597c1ff --- /dev/null +++ b/pkg/pmem-device-manager/metrics.go @@ -0,0 +1,88 @@ +/* +Copyright 2020 Intel Corporation. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package pmdmanager + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + pmemMaxDesc = prometheus.NewDesc( + "pmem_amount_max_volume_size", + "The size of the largest PMEM volume that can be created.", + nil, nil, + ) + pmemAvailableDesc = prometheus.NewDesc( + "pmem_amount_available", + "Remaining amount of PMEM on the host that can be used for new volumes.", + nil, nil, + ) + pmemManagedDesc = prometheus.NewDesc( + "pmem_amount_managed", + "Amount of PMEM on the host that is managed by PMEM-CSI.", + nil, nil, + ) + pmemTotalDesc = prometheus.NewDesc( + "pmem_amount_total", + "Total amount of PMEM on the host.", + nil, nil, + ) +) + +// NodeLabel is a label used for Prometheus which identifies the +// node that the controller talks to. +const NodeLabel = "node" + +// CapacityCollector is a wrapper around a PMEM device manager which +// takes GetCapacity values and turns them into metrics data. +type CapacityCollector struct { + PmemDeviceCapacity +} + +// MustRegister adds the collector to the registry, using labels to tag each sample with node and driver name. +func (cc CapacityCollector) MustRegister(reg prometheus.Registerer, nodeName, driverName string) { + labels := prometheus.Labels{ + NodeLabel: nodeName, + "driver_name": driverName, // same label name as in csi-lib-utils for CSI gRPC calls + } + prometheus.WrapRegistererWith(labels, reg).MustRegister(cc) +} + +// Describe implements prometheus.Collector.Describe. +func (cc CapacityCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(cc, ch) +} + +// Collect implements prometheus.Collector.Collect. +func (cc CapacityCollector) Collect(ch chan<- prometheus.Metric) { + capacity, err := cc.GetCapacity() + if err != nil { + return + } + ch <- prometheus.MustNewConstMetric( + pmemMaxDesc, + prometheus.GaugeValue, + float64(capacity.MaxVolumeSize), + ) + ch <- prometheus.MustNewConstMetric( + pmemAvailableDesc, + prometheus.GaugeValue, + float64(capacity.Available), + ) + ch <- prometheus.MustNewConstMetric( + pmemManagedDesc, + prometheus.GaugeValue, + float64(capacity.Managed), + ) + ch <- prometheus.MustNewConstMetric( + pmemTotalDesc, + prometheus.GaugeValue, + float64(capacity.Total), + ) +} + +var _ prometheus.Collector = CapacityCollector{} diff --git a/pkg/pmem-device-manager/pmd-manager.go b/pkg/pmem-device-manager/pmd-manager.go index ff4fe3b25f..ec816a6fe7 100644 --- a/pkg/pmem-device-manager/pmd-manager.go +++ b/pkg/pmem-device-manager/pmd-manager.go @@ -41,14 +41,25 @@ type Capacity struct { Total uint64 } +func (c Capacity) GetCapacity() (Capacity, error) { + return c, nil +} + +var _ PmemDeviceCapacity = Capacity{} + +// PmemDeviceCapacity interface just returns capacity information. +type PmemDeviceCapacity interface { + // GetCapacity returns information about local capacity. + GetCapacity() (Capacity, error) +} + //PmemDeviceManager interface to manage the PMEM block devices type PmemDeviceManager interface { + PmemDeviceCapacity + // GetName returns current device manager's operation mode GetMode() api.DeviceMode - // GetCapacity returns information about local capacity. - GetCapacity() (Capacity, error) - // CreateDevice creates a new block device with give name, size and namespace mode // Possible errors: ErrNotEnoughSpace, ErrDeviceExists CreateDevice(name string, size uint64) error diff --git a/pkg/pmem-registry/pmem-registry.pb.go b/pkg/pmem-registry/pmem-registry.pb.go deleted file mode 100644 index e0543f5ebf..0000000000 --- a/pkg/pmem-registry/pmem-registry.pb.go +++ /dev/null @@ -1,307 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: pmem-registry.proto - -package registry - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -//const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type RegisterControllerRequest struct { - // unique node id, usually id of the compute node in the cluster - // which has the nvdimm installed - NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - // Node controller's address that can be used for grpc.Dial to - // connect to the controller - Endpoint string `protobuf:"bytes,2,opt,name=endpoint,proto3" json:"endpoint,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RegisterControllerRequest) Reset() { *m = RegisterControllerRequest{} } -func (m *RegisterControllerRequest) String() string { return proto.CompactTextString(m) } -func (*RegisterControllerRequest) ProtoMessage() {} -func (*RegisterControllerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bfc8dd910f76aa6, []int{0} -} - -func (m *RegisterControllerRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RegisterControllerRequest.Unmarshal(m, b) -} -func (m *RegisterControllerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RegisterControllerRequest.Marshal(b, m, deterministic) -} -func (m *RegisterControllerRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_RegisterControllerRequest.Merge(m, src) -} -func (m *RegisterControllerRequest) XXX_Size() int { - return xxx_messageInfo_RegisterControllerRequest.Size(m) -} -func (m *RegisterControllerRequest) XXX_DiscardUnknown() { - xxx_messageInfo_RegisterControllerRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_RegisterControllerRequest proto.InternalMessageInfo - -func (m *RegisterControllerRequest) GetNodeId() string { - if m != nil { - return m.NodeId - } - return "" -} - -func (m *RegisterControllerRequest) GetEndpoint() string { - if m != nil { - return m.Endpoint - } - return "" -} - -type RegisterControllerReply struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RegisterControllerReply) Reset() { *m = RegisterControllerReply{} } -func (m *RegisterControllerReply) String() string { return proto.CompactTextString(m) } -func (*RegisterControllerReply) ProtoMessage() {} -func (*RegisterControllerReply) Descriptor() ([]byte, []int) { - return fileDescriptor_4bfc8dd910f76aa6, []int{1} -} - -func (m *RegisterControllerReply) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RegisterControllerReply.Unmarshal(m, b) -} -func (m *RegisterControllerReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RegisterControllerReply.Marshal(b, m, deterministic) -} -func (m *RegisterControllerReply) XXX_Merge(src proto.Message) { - xxx_messageInfo_RegisterControllerReply.Merge(m, src) -} -func (m *RegisterControllerReply) XXX_Size() int { - return xxx_messageInfo_RegisterControllerReply.Size(m) -} -func (m *RegisterControllerReply) XXX_DiscardUnknown() { - xxx_messageInfo_RegisterControllerReply.DiscardUnknown(m) -} - -var xxx_messageInfo_RegisterControllerReply proto.InternalMessageInfo - -type UnregisterControllerRequest struct { - // Id of the node controller to unregister from ControllerRegistry - NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *UnregisterControllerRequest) Reset() { *m = UnregisterControllerRequest{} } -func (m *UnregisterControllerRequest) String() string { return proto.CompactTextString(m) } -func (*UnregisterControllerRequest) ProtoMessage() {} -func (*UnregisterControllerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bfc8dd910f76aa6, []int{2} -} - -func (m *UnregisterControllerRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_UnregisterControllerRequest.Unmarshal(m, b) -} -func (m *UnregisterControllerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_UnregisterControllerRequest.Marshal(b, m, deterministic) -} -func (m *UnregisterControllerRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UnregisterControllerRequest.Merge(m, src) -} -func (m *UnregisterControllerRequest) XXX_Size() int { - return xxx_messageInfo_UnregisterControllerRequest.Size(m) -} -func (m *UnregisterControllerRequest) XXX_DiscardUnknown() { - xxx_messageInfo_UnregisterControllerRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_UnregisterControllerRequest proto.InternalMessageInfo - -func (m *UnregisterControllerRequest) GetNodeId() string { - if m != nil { - return m.NodeId - } - return "" -} - -type UnregisterControllerReply struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *UnregisterControllerReply) Reset() { *m = UnregisterControllerReply{} } -func (m *UnregisterControllerReply) String() string { return proto.CompactTextString(m) } -func (*UnregisterControllerReply) ProtoMessage() {} -func (*UnregisterControllerReply) Descriptor() ([]byte, []int) { - return fileDescriptor_4bfc8dd910f76aa6, []int{3} -} - -func (m *UnregisterControllerReply) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_UnregisterControllerReply.Unmarshal(m, b) -} -func (m *UnregisterControllerReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_UnregisterControllerReply.Marshal(b, m, deterministic) -} -func (m *UnregisterControllerReply) XXX_Merge(src proto.Message) { - xxx_messageInfo_UnregisterControllerReply.Merge(m, src) -} -func (m *UnregisterControllerReply) XXX_Size() int { - return xxx_messageInfo_UnregisterControllerReply.Size(m) -} -func (m *UnregisterControllerReply) XXX_DiscardUnknown() { - xxx_messageInfo_UnregisterControllerReply.DiscardUnknown(m) -} - -var xxx_messageInfo_UnregisterControllerReply proto.InternalMessageInfo - -func init() { - proto.RegisterType((*RegisterControllerRequest)(nil), "registry.v0.RegisterControllerRequest") - proto.RegisterType((*RegisterControllerReply)(nil), "registry.v0.RegisterControllerReply") - proto.RegisterType((*UnregisterControllerRequest)(nil), "registry.v0.UnregisterControllerRequest") - proto.RegisterType((*UnregisterControllerReply)(nil), "registry.v0.UnregisterControllerReply") -} - -func init() { proto.RegisterFile("pmem-registry.proto", fileDescriptor_4bfc8dd910f76aa6) } - -var fileDescriptor_4bfc8dd910f76aa6 = []byte{ - // 207 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2e, 0xc8, 0x4d, 0xcd, - 0xd5, 0x2d, 0x4a, 0x4d, 0xcf, 0x2c, 0x2e, 0x29, 0xaa, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, - 0xe2, 0x86, 0xf3, 0xcb, 0x0c, 0x94, 0x02, 0xb8, 0x24, 0x83, 0xc0, 0xdc, 0xd4, 0x22, 0xe7, 0xfc, - 0xbc, 0x92, 0xa2, 0xfc, 0x9c, 0x9c, 0xd4, 0xa2, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12, 0x21, - 0x71, 0x2e, 0xf6, 0xbc, 0xfc, 0x94, 0xd4, 0xf8, 0xcc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, - 0x20, 0x36, 0x10, 0xd7, 0x33, 0x45, 0x48, 0x8a, 0x8b, 0x23, 0x35, 0x2f, 0xa5, 0x20, 0x3f, 0x33, - 0xaf, 0x44, 0x82, 0x09, 0x2c, 0x03, 0xe7, 0x2b, 0x49, 0x72, 0x89, 0x63, 0x33, 0xb1, 0x20, 0xa7, - 0x52, 0xc9, 0x8c, 0x4b, 0x3a, 0x34, 0xaf, 0x88, 0x64, 0xeb, 0x94, 0xa4, 0xb9, 0x24, 0xb1, 0xeb, - 0x2b, 0xc8, 0xa9, 0x34, 0xba, 0xc3, 0xc8, 0xc5, 0x11, 0x04, 0xf5, 0x91, 0x50, 0x0a, 0x97, 0x10, - 0xa6, 0xe5, 0x42, 0x6a, 0x7a, 0x48, 0x5e, 0xd6, 0xc3, 0xe9, 0x5f, 0x29, 0x15, 0x82, 0xea, 0x40, - 0xbe, 0x60, 0x10, 0xca, 0xe2, 0x12, 0xc1, 0xe6, 0x1e, 0x21, 0x0d, 0x14, 0xfd, 0x78, 0xbc, 0x2a, - 0xa5, 0x46, 0x84, 0x4a, 0xb0, 0x5d, 0x4e, 0x5c, 0x51, 0x1c, 0x30, 0xa5, 0x49, 0x6c, 0xe0, 0x08, - 0x34, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x06, 0x3f, 0x1d, 0x9b, 0xd7, 0x01, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// RegistryClient is the client API for Registry service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type RegistryClient interface { - RegisterController(ctx context.Context, in *RegisterControllerRequest, opts ...grpc.CallOption) (*RegisterControllerReply, error) - UnregisterController(ctx context.Context, in *UnregisterControllerRequest, opts ...grpc.CallOption) (*UnregisterControllerReply, error) -} - -type registryClient struct { - cc *grpc.ClientConn -} - -func NewRegistryClient(cc *grpc.ClientConn) RegistryClient { - return ®istryClient{cc} -} - -func (c *registryClient) RegisterController(ctx context.Context, in *RegisterControllerRequest, opts ...grpc.CallOption) (*RegisterControllerReply, error) { - out := new(RegisterControllerReply) - err := c.cc.Invoke(ctx, "/registry.v0.Registry/RegisterController", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *registryClient) UnregisterController(ctx context.Context, in *UnregisterControllerRequest, opts ...grpc.CallOption) (*UnregisterControllerReply, error) { - out := new(UnregisterControllerReply) - err := c.cc.Invoke(ctx, "/registry.v0.Registry/UnregisterController", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// RegistryServer is the server API for Registry service. -type RegistryServer interface { - RegisterController(context.Context, *RegisterControllerRequest) (*RegisterControllerReply, error) - UnregisterController(context.Context, *UnregisterControllerRequest) (*UnregisterControllerReply, error) -} - -func RegisterRegistryServer(s *grpc.Server, srv RegistryServer) { - s.RegisterService(&_Registry_serviceDesc, srv) -} - -func _Registry_RegisterController_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(RegisterControllerRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(RegistryServer).RegisterController(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/registry.v0.Registry/RegisterController", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(RegistryServer).RegisterController(ctx, req.(*RegisterControllerRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Registry_UnregisterController_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(UnregisterControllerRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(RegistryServer).UnregisterController(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/registry.v0.Registry/UnregisterController", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(RegistryServer).UnregisterController(ctx, req.(*UnregisterControllerRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _Registry_serviceDesc = grpc.ServiceDesc{ - ServiceName: "registry.v0.Registry", - HandlerType: (*RegistryServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "RegisterController", - Handler: _Registry_RegisterController_Handler, - }, - { - MethodName: "UnregisterController", - Handler: _Registry_UnregisterController_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "pmem-registry.proto", -} diff --git a/pkg/pmem-registry/pmem-registry.proto b/pkg/pmem-registry/pmem-registry.proto deleted file mode 100644 index a8444660ba..0000000000 --- a/pkg/pmem-registry/pmem-registry.proto +++ /dev/null @@ -1,31 +0,0 @@ -syntax = "proto3"; -package registry.v0; -option go_package = "registry"; - - -service Registry { - rpc RegisterController(RegisterControllerRequest) returns (RegisterControllerReply) {} - rpc UnregisterController(UnregisterControllerRequest) returns (UnregisterControllerReply) {} -} - -message RegisterControllerRequest { - // unique node id, usually id of the compute node in the cluster - // which has the nvdimm installed - string node_id = 1; - // Node controller's address that can be used for grpc.Dial to - // connect to the controller - string endpoint = 2; -} - -message RegisterControllerReply { - // empty -} - -message UnregisterControllerRequest { - // Id of the node controller to unregister from ControllerRegistry - string node_id = 1; -} - -message UnregisterControllerReply { - // empty -} diff --git a/pkg/registryserver/registryserver.go b/pkg/registryserver/registryserver.go deleted file mode 100644 index f7adac2093..0000000000 --- a/pkg/registryserver/registryserver.go +++ /dev/null @@ -1,233 +0,0 @@ -package registryserver - -import ( - "crypto/tls" - "fmt" - "sync" - "time" - - pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" - registry "github.com/intel/pmem-csi/pkg/pmem-registry" - "github.com/kubernetes-csi/csi-lib-utils/metrics" - "github.com/prometheus/client_golang/prometheus" - "golang.org/x/net/context" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/klog/v2" - "k8s.io/utils/keymutex" -) - -// NodeLabel is a label used for Prometheus which identifies the -// node that the controller talks to. -const NodeLabel = "node" - -// RegistryListener is an interface for registry server change listeners -// All the callbacks are called once after updating the in-memory -// registry data. -type RegistryListener interface { - // OnNodeAdded is called by RegistryServer whenever a new node controller is registered - // or node controller updated its endpoint. - // In case of error, the node registration would fail and removed from registry. - OnNodeAdded(ctx context.Context, node *NodeInfo) error - // OnNodeDeleted is called by RegistryServer whenever a node controller unregistered. - // Callback implementations has to note that by the time this method is called, - // the NodeInfo for that node have already removed from in-memory registry. - OnNodeDeleted(ctx context.Context, node *NodeInfo) -} - -type RegistryServer struct { - // mutex is used to protect concurrent access of RegistryServer's - // data(nodeClients) - mutex sync.Mutex - // rpcMutex is used to avoid concurrent RPC(RegisterController, UnregisterController) - // requests from the same node - rpcMutex keymutex.KeyMutex - clientTLSConfig *tls.Config - nodeClients map[string]*NodeInfo - listeners map[RegistryListener]struct{} - - // All nodes share the same metrics manager, but log samples - // with a different "pmem_csi_node" label and thus get their - // own histogram. - cmm metrics.CSIMetricsManager -} - -type NodeInfo struct { - //NodeID controller node id - NodeID string - //Endpoint node controller endpoint - Endpoint string -} - -var ( - pmemNodes = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "pmem_nodes", - Help: "The number of PMEM-CSI nodes registered in the controller.", - }, - ) -) - -func init() { - prometheus.MustRegister(pmemNodes) -} - -func New(tlsConfig *tls.Config, driverName string) *RegistryServer { - return &RegistryServer{ - rpcMutex: keymutex.NewHashed(-1), - clientTLSConfig: tlsConfig, - nodeClients: map[string]*NodeInfo{}, - listeners: map[RegistryListener]struct{}{}, - cmm: metrics.NewCSIMetricsManagerWithOptions(driverName, - metrics.WithProcessStartTime(false), - metrics.WithSubsystem("pmem_csi_controller"), - metrics.WithLabelNames(NodeLabel), - ), - } -} - -func (rs *RegistryServer) GetMetricsGatherer() prometheus.Gatherer { - return rs.cmm.GetRegistry() -} - -func (rs *RegistryServer) RegisterService(rpcServer *grpc.Server) { - registry.RegisterRegistryServer(rpcServer, rs) -} - -//GetNodeController returns the node controller info for given nodeID, error if not found -func (rs *RegistryServer) GetNodeController(nodeID string) (NodeInfo, error) { - rs.mutex.Lock() - defer rs.mutex.Unlock() - - if node, ok := rs.nodeClients[nodeID]; ok { - return *node, nil - } - - return NodeInfo{}, fmt.Errorf("No node registered with id: %v", nodeID) -} - -// ConnectToNodeController initiates a connection to controller running at nodeId -func (rs *RegistryServer) ConnectToNodeController(nodeId string) (*grpc.ClientConn, error) { - nodeInfo, err := rs.GetNodeController(nodeId) - if err != nil { - return nil, err - } - - klog.V(3).Infof("Connecting to node controller: %s", nodeInfo.Endpoint) - - return pmemgrpc.Connect(nodeInfo.Endpoint, rs.clientTLSConfig, - grpc.WithUnaryInterceptor(func( - ctx context.Context, - method string, - req, reply interface{}, - cc *grpc.ClientConn, - invoker grpc.UnaryInvoker, - opts ...grpc.CallOption) error { - start := time.Now() - err := invoker(ctx, method, req, reply, cc, opts...) - duration := time.Since(start) - cmmv, err2 := rs.cmm.WithLabelValues( - map[string]string{NodeLabel: nodeId}, - ) - if err2 != nil { - klog.Errorf("CSI call metrics: set label %s value: %v", NodeLabel, err2) - } else { - cmmv.RecordMetrics( - method, /* operationName */ - err, /* operationErr */ - duration, /* operationDuration */ - ) - } - return err - }), - ) -} - -func (rs *RegistryServer) AddListener(l RegistryListener) { - rs.listeners[l] = struct{}{} -} - -func (rs *RegistryServer) RegisterController(ctx context.Context, req *registry.RegisterControllerRequest) (*registry.RegisterControllerReply, error) { - if req.GetNodeId() == "" { - return nil, status.Error(codes.InvalidArgument, "Missing NodeId parameter") - } - - if req.GetEndpoint() == "" { - return nil, status.Error(codes.InvalidArgument, "Missing endpoint address") - } - - rs.rpcMutex.LockKey(req.NodeId) - defer rs.rpcMutex.UnlockKey(req.NodeId) - - klog.V(3).Infof("Registering node: %s, endpoint: %s", req.NodeId, req.Endpoint) - - node := &NodeInfo{ - NodeID: req.NodeId, - Endpoint: req.Endpoint, - } - - rs.mutex.Lock() - n, found := rs.nodeClients[req.NodeId] - if found { - if n.Endpoint != req.Endpoint { - found = false - } - } - rs.nodeClients[req.NodeId] = node - pmemNodes.Set(float64(len(rs.nodeClients))) - rs.mutex.Unlock() - - if !found { - for l := range rs.listeners { - if err := l.OnNodeAdded(ctx, node); err != nil { - rs.mutex.Lock() - delete(rs.nodeClients, req.NodeId) - pmemNodes.Set(float64(len(rs.nodeClients))) - rs.mutex.Unlock() - return nil, fmt.Errorf("failed to register node: %w", err) - } - } - } - - return ®istry.RegisterControllerReply{}, nil -} - -func (rs *RegistryServer) UnregisterController(ctx context.Context, req *registry.UnregisterControllerRequest) (*registry.UnregisterControllerReply, error) { - if req.GetNodeId() == "" { - return nil, status.Error(codes.InvalidArgument, "Missing NodeId parameter") - } - - rs.rpcMutex.LockKey(req.NodeId) - defer rs.rpcMutex.UnlockKey(req.NodeId) - - rs.mutex.Lock() - node, ok := rs.nodeClients[req.NodeId] - delete(rs.nodeClients, req.NodeId) - pmemNodes.Set(float64(len(rs.nodeClients))) - rs.mutex.Unlock() - - if ok { - for l := range rs.listeners { - l.OnNodeDeleted(ctx, node) - } - klog.V(3).Infof("Unregistered node: %s", req.NodeId) - } else { - klog.V(3).Infof("No node registered with id '%s'", req.NodeId) - } - - return ®istry.UnregisterControllerReply{}, nil -} - -// NodeClients returns a new map which contains a copy of all currently known node clients. -// It is safe to use concurrently with the other methods. -func (rs *RegistryServer) NodeClients() map[string]*NodeInfo { - rs.mutex.Lock() - defer rs.mutex.Unlock() - - copy := map[string]*NodeInfo{} - for key, value := range rs.nodeClients { - copy[key] = value - } - return copy -} diff --git a/pkg/registryserver/registryserver_test.go b/pkg/registryserver/registryserver_test.go deleted file mode 100644 index d27e15bc93..0000000000 --- a/pkg/registryserver/registryserver_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package registryserver_test - -import ( - "crypto/tls" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" - pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" - registry "github.com/intel/pmem-csi/pkg/pmem-registry" - "github.com/intel/pmem-csi/pkg/registryserver" - "golang.org/x/net/context" - "google.golang.org/grpc" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestPmemRegistry(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Registry Suite") -} - -var tmpDir string - -var _ = BeforeSuite(func() { - var err error - tmpDir, err = ioutil.TempDir("", "pmem-test-") - Expect(err).NotTo(HaveOccurred()) -}) - -var _ = AfterSuite(func() { - os.RemoveAll(tmpDir) -}) - -var _ = Describe("pmem registry", func() { - - registryServerSocketFile := filepath.Join(tmpDir, "pmem-registry.sock") - registryServerEndpoint := "unix://" + registryServerSocketFile - - var ( - tlsConfig *tls.Config - nbServer *grpcserver.NonBlockingGRPCServer - registryClientConn *grpc.ClientConn - registryClient registry.RegistryClient - registryServer *registryserver.RegistryServer - ) - - BeforeEach(func() { - var err error - - registryServer = registryserver.New(nil, "pmem-csi.intel.com") - - caFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/ca.pem") - certFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-registry.pem") - keyFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-registry-key.pem") - tlsConfig, err = pmemgrpc.LoadServerTLS(caFile, certFile, keyFile, "pmem-node-controller") - Expect(err).NotTo(HaveOccurred()) - - nbServer = grpcserver.NewNonBlockingGRPCServer() - err = nbServer.Start(registryServerEndpoint, tlsConfig, nil, registryServer) - Expect(err).NotTo(HaveOccurred()) - _, err = os.Stat(registryServerSocketFile) - Expect(err).NotTo(HaveOccurred()) - - // set up node controller client - nodeCertFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-node-controller.pem") - nodeCertKey := os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-node-controller-key.pem") - tlsConfig, err = pmemgrpc.LoadClientTLS(caFile, nodeCertFile, nodeCertKey, "pmem-registry") - Expect(err).NotTo(HaveOccurred()) - - registryClientConn, err = pmemgrpc.Connect(registryServerEndpoint, tlsConfig) - Expect(err).NotTo(HaveOccurred()) - registryClient = registry.NewRegistryClient(registryClientConn) - }) - - AfterEach(func() { - if registryServer != nil { - nbServer.ForceStop() - nbServer.Wait() - } - os.Remove(registryServerSocketFile) - if registryClientConn != nil { - registryClientConn.Close() - } - }) - - Context("Registry API", func() { - controllerServerSocketFile := filepath.Join(tmpDir, "pmem-controller.sock") - controllerServerEndpoint := "unix://" + controllerServerSocketFile - var ( - nodeId = "pmem-test" - registerReq = registry.RegisterControllerRequest{ - NodeId: nodeId, - Endpoint: controllerServerEndpoint, - } - - unregisterReq = registry.UnregisterControllerRequest{ - NodeId: nodeId, - } - ) - - It("Register node controller", func() { - Expect(registryClient).ShouldNot(BeNil()) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := registryClient.RegisterController(ctx, ®isterReq) - Expect(err).NotTo(HaveOccurred()) - - _, err = registryServer.GetNodeController(nodeId) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Registration should fail", func() { - Expect(registryClient).ShouldNot(BeNil()) - - l := listener{} - - registryServer.AddListener(l) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := registryClient.RegisterController(ctx, ®isterReq) - Expect(err).To(HaveOccurred()) - - _, err = registryServer.GetNodeController(nodeId) - Expect(err).To(HaveOccurred()) - }) - - It("Unregister node controller", func() { - Expect(registryClient).ShouldNot(BeNil()) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := registryClient.RegisterController(ctx, ®isterReq) - Expect(err).NotTo(HaveOccurred()) - - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err = registryClient.UnregisterController(ctx, &unregisterReq) - Expect(err).NotTo(HaveOccurred()) - - _, err = registryServer.GetNodeController(nodeId) - Expect(err).To(HaveOccurred()) - }) - - It("Unregister non existing node controller", func() { - Expect(registryClient).ShouldNot(BeNil()) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := registryClient.UnregisterController(ctx, &unregisterReq) - Expect(err).NotTo(HaveOccurred()) - }) - }) - - Context("Registry Security", func() { - var ( - evilEndpoint = "unix:///tmp/pmem-evil.sock" - ca = os.ExpandEnv("${TEST_WORK}/pmem-ca/ca.pem") - cert = os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-node-controller.pem") - key = os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-node-controller-key.pem") - wrongCert = os.ExpandEnv("${TEST_WORK}/pmem-ca/wrong-node-controller.pem") - wrongKey = os.ExpandEnv("${TEST_WORK}/pmem-ca/wrong-node-controller-key.pem") - - evilCA = os.ExpandEnv("${TEST_WORK}/evil-ca/ca.pem") - evilCert = os.ExpandEnv("${TEST_WORK}/evil-ca/pmem-node-controller.pem") - evilKey = os.ExpandEnv("${TEST_WORK}/evil-ca/pmem-node-controller-key.pem") - ) - - // gRPC returns all kinds of errors when TLS fails. - badConnectionRE := "authentication handshake failed: remote error: tls: bad certificate|all SubConns are in TransientFailure|rpc error: code = Unavailable" - - // This covers different scenarios for connections to the registry. - cases := []struct { - name, ca, cert, key, peerName, errorRE string - }{ - // The exact error for the server side depends on whether TLS 1.3 is active (https://golang.org/doc/go1.12#tls_1_3). - // It looks like error detection is less precise in that case. - {"registry should detect man-in-the-middle", ca, evilCert, evilKey, "pmem-registry", - badConnectionRE, - }, - {"client should detect man-in-the-middle", evilCA, evilCert, evilKey, "pmem-registry", "transport: authentication handshake failed: x509: certificate signed by unknown authority"}, - {"client should detect wrong peer", ca, cert, key, "unknown-registry", "transport: authentication handshake failed: x509: certificate is valid for pmem-csi-scheduler, pmem-csi-scheduler.default, pmem-csi-scheduler.default.svc, pmem-csi-metrics, pmem-csi-metrics.default, pmem-csi-metrics.default.svc, pmem-registry, not unknown-registry"}, - {"server should detect wrong peer", ca, wrongCert, wrongKey, "pmem-registry", - badConnectionRE, - }, - } - - for _, c := range cases { - c := c - It(c.name, func() { - tlsConfig, err := pmemgrpc.LoadClientTLS(c.ca, c.cert, c.key, c.peerName) - Expect(err).NotTo(HaveOccurred()) - clientConn, err := pmemgrpc.Connect(registryServerEndpoint, tlsConfig) - Expect(err).NotTo(HaveOccurred()) - client := registry.NewRegistryClient(clientConn) - - req := registry.RegisterControllerRequest{ - NodeId: "pmem-evil", - Endpoint: evilEndpoint, - } - - _, err = client.RegisterController(context.Background(), &req) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(MatchRegexp(c.errorRE)) - }) - } - }) - -}) - -type listener struct{} - -func (l listener) OnNodeAdded(ctx context.Context, node *registryserver.NodeInfo) error { - return fmt.Errorf("failed") -} - -func (l listener) OnNodeDeleted(ctx context.Context, node *registryserver.NodeInfo) { -} diff --git a/pkg/scheduler/capacity.go b/pkg/scheduler/capacity.go index 01a6c74b87..fe4efe38b8 100644 --- a/pkg/scheduler/capacity.go +++ b/pkg/scheduler/capacity.go @@ -7,38 +7,132 @@ SPDX-License-Identifier: Apache-2.0 package scheduler import ( - "context" + "errors" "fmt" + "io" + "net/http" - "github.com/container-storage-interface/spec/lib/go/csi" - - "github.com/intel/pmem-csi/pkg/registryserver" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + corelistersv1 "k8s.io/client-go/listers/core/v1" ) -type capacity struct { - rs *registryserver.RegistryServer +type capacityFromMetrics struct { + namespace string + driverName string + podLister corelistersv1.PodLister + client http.Client +} + +func CapacityViaMetrics(namespace, driverName string, podLister corelistersv1.PodLister) Capacity { + return &capacityFromMetrics{ + namespace: namespace, + driverName: driverName, + podLister: podLister, + } } -func CapacityViaRegistry(rs *registryserver.RegistryServer) Capacity { - return capacity{rs} +// All PMEM-CSI deployments must use these labels for the node driver pods. +func (c *capacityFromMetrics) pmemCSINodeSelector() labels.Selector { + return labels.Set{ + "app.kubernetes.io/part-of": "pmem-csi", + "app.kubernetes.io/component": "node", + "app.kubernetes.io/instance": c.driverName, + }.AsSelector() } -// NodeCapacity implements the necessary method for the NodeCapacity interface based -// on a registry server. -func (c capacity) NodeCapacity(nodeName string) (int64, error) { - conn, err := c.rs.ConnectToNodeController(nodeName) +// NodeCapacity implements the necessary method for the NodeCapacity interface by +// looking up pods in the namespace which run on the node (usually one) +// and retrieving metrics data from them. The driver name is checked to allow +// more than one driver instance per node (unlikely). +func (c *capacityFromMetrics) NodeCapacity(nodeName string) (int64, error) { + pods, err := c.podLister.List(c.pmemCSINodeSelector()) if err != nil { - return 0, fmt.Errorf("connect to PMEM-CSI on node %q: %v", nodeName, err) + return 0, fmt.Errorf("list PMEM-CSI node pods: %v", err) } - defer conn.Close() + for _, pod := range pods { + if pod.Spec.NodeName != nodeName || + pod.Namespace != c.namespace { + continue + } + url := metricsURL(pod) + if url == "" { + continue + } + capacity, err := c.retrieveMaxVolumeSize(url) + switch err { + case wrongPod: + continue + case nil: + return capacity, nil + default: + return 0, fmt.Errorf("get metrics from pod %s via %s: %v", pod.Name, url, err) + } + } + + // Node not known or no metrics. + return 0, nil +} + +var wrongPod = errors.New("wrong driver pod") - csiClient := csi.NewControllerClient(conn) - // We assume here that storage class parameters do not matter. - resp, err := csiClient.GetCapacity(context.Background(), &csi.GetCapacityRequest{}) +func (c *capacityFromMetrics) retrieveMaxVolumeSize(url string) (int64, error) { + // TODO (?): negotiate encoding (https://pkg.go.dev/github.com/prometheus/common/expfmt#Negotiate) + resp, err := c.client.Get(url) if err != nil { - // We cause an abort of scheduling by treating this as error. - // A less drastic reaction would be to filter out the node. - return 0, fmt.Errorf("get capacity from node %q: %v", nodeName, err) + return 0, err + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("bad HTTP response status: %s", resp.Status) + } + decoder := expfmt.NewDecoder(resp.Body, expfmt.ResponseFormat(resp.Header)) + if err != nil { + return 0, fmt.Errorf("read response: %v", err) + } + var metrics dto.MetricFamily + + for { + err := decoder.Decode(&metrics) + if err != nil { + if errors.Is(err, io.EOF) { + // If we get here without finding what we look for, we must be talking + // to the wrong pod and should keep looking. + return 0, wrongPod + } + return 0, fmt.Errorf("decode response: %v", err) + } + if metrics.GetName() == "pmem_amount_max_volume_size" { + for _, metric := range metrics.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == "driver_name" && + label.GetValue() != c.driverName { + return 0, wrongPod + } + } + // "driver_name" was not present yet in PMEM-CSI 0.8.0, so + // we cannot fail when it is missing. + gauge := metric.GetGauge() + if gauge == nil { + return 0, fmt.Errorf("unexpected metric type for pmem_amount_max_volume_size: %s", metrics.GetType()) + } + return int64(gauge.GetValue()), nil + } + } + } +} + +func metricsURL(pod *corev1.Pod) string { + for _, container := range pod.Spec.Containers { + if container.Name == "pmem-driver" { + for _, containerPort := range container.Ports { + if containerPort.Name == "metrics" { + return fmt.Sprintf("http://%s:%d/metrics", pod.Status.PodIP, containerPort.ContainerPort) + } + } + return "" + } } - return resp.AvailableCapacity, nil + return "" } diff --git a/pkg/scheduler/capacity_test.go b/pkg/scheduler/capacity_test.go new file mode 100644 index 0000000000..a53fd3b1b8 --- /dev/null +++ b/pkg/scheduler/capacity_test.go @@ -0,0 +1,207 @@ +/* +Copyright 2020 Intel Corp. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package scheduler + +import ( + "net" + "net/http" + "strconv" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelistersv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + pmdmanager "github.com/intel/pmem-csi/pkg/pmem-device-manager" +) + +type node struct { + name, namespace string + capacity pmdmanager.Capacity + driverName string + noMetrics bool +} + +func (n node) createPMEMPod(port int) *corev1.Pod { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pmem-csi-node-" + n.name, + Namespace: n.namespace, + Labels: map[string]string{ + "app.kubernetes.io/part-of": "pmem-csi", + "app.kubernetes.io/component": "node", + "app.kubernetes.io/instance": n.driverName, + }, + }, + Spec: corev1.PodSpec{ + NodeName: n.name, + Containers: []corev1.Container{ + { + Name: "pmem-driver", + }, + }, + }, + Status: corev1.PodStatus{ + PodIP: "127.0.0.1", // address we listen on + }, + } + if port != 0 { + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: int32(port), + }, + } + } + return &pod +} + +func TestCapacityFromMetrics(t *testing.T) { + cap := pmdmanager.Capacity{ + MaxVolumeSize: 1000, + Available: 2000, + Managed: 3000, + Total: 4000, + } + capSmall := pmdmanager.Capacity{ + MaxVolumeSize: 1, + Available: 2, + Managed: 3, + Total: 4, + } + testcases := map[string]struct { + nodes []node + node string + namespace string + driverName string + expected int64 + expectError bool + }{ + "one node": { + nodes: []node{ + { + name: "foobar", + capacity: cap, + }, + }, + node: "foobar", + expected: 1000, + }, + "no such node": { + nodes: []node{ + { + name: "foo", + capacity: pmdmanager.Capacity{MaxVolumeSize: 1000}, + }, + }, + node: "bar", + }, + "no driver": { + node: "foobar", + }, + "wrong driver": { + nodes: []node{ + { + name: "foobar", + driverName: "AAA", + capacity: cap, + }, + }, + node: "foobar", + driverName: "BBB", + }, + "wrong namespace": { + nodes: []node{ + { + name: "foobar", + namespace: "default", + capacity: cap, + }, + }, + node: "foobar", + namespace: "pmem-csi", + }, + "multiple drivers": { + nodes: []node{ + { + name: "foobar", + driverName: "AAA", + capacity: capSmall, + }, + { + name: "foobar", + driverName: "BBB", + capacity: cap, + }, + }, + node: "foobar", + driverName: "BBB", + expected: 1000, + }, + "metrics handler missing": { + nodes: []node{ + { + name: "foobar", + capacity: cap, + noMetrics: true, + }, + }, + node: "foobar", + expectError: true, + }, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + + // We need one metrics server per node. + for _, node := range tc.nodes { + mux := http.NewServeMux() + registry := prometheus.NewPedanticRegistry() + collector := pmdmanager.CapacityCollector{PmemDeviceCapacity: node.capacity} + collector.MustRegister(registry, node.name, node.driverName) + if !node.noMetrics { + mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) + } + listen := "127.0.0.1:" + listener, err := net.Listen("tcp", listen) + require.NoError(t, err, "listen") + tcpListener := listener.(*net.TCPListener) + server := http.Server{ + Handler: mux, + } + go server.Serve(listener) + defer server.Close() + + // Now fake a PMEM-CSI pod on that node. + _, portStr, err := net.SplitHostPort(tcpListener.Addr().String()) + require.NoError(t, err, "split listen address") + port, err := strconv.Atoi(portStr) + require.NoError(t, err, "parse listen port") + pod := node.createPMEMPod(port) + if pod != nil { + podIndexer.Add(pod) + } + } + podLister := corelistersv1.NewPodLister(podIndexer) + c := CapacityViaMetrics(tc.namespace, tc.driverName, podLister) + actual, err := c.NodeCapacity(tc.node) + if tc.expectError { + t.Logf("got error %v", err) + require.Error(t, err, "NodeCapacity should have failed") + } else { + require.NoError(t, err, "NodeCapacity should have succeeded") + require.Equal(t, tc.expected, actual, "capacity") + } + }) + } +} diff --git a/pkg/scheduler/mutate_pod.go b/pkg/scheduler/mutate_pod.go index fea3ef3481..62d0b1f17e 100644 --- a/pkg/scheduler/mutate_pod.go +++ b/pkg/scheduler/mutate_pod.go @@ -27,8 +27,9 @@ import ( ) const ( - // Resource is the resource that will trigger the scheduler extender. - Resource = "pmem-csi.intel.com/scheduler" + // resourceSuffix is the part which gets added to the CSI driver name to + // create the extended resource name that will trigger the scheduler extender. + resourceSuffix = "/scheduler" ) // Handle implements admission.Handler interface. @@ -70,14 +71,15 @@ func (s scheduler) Handle(ctx context.Context, req admission.Request) admission. ctnr := &pod.Spec.Containers[0] quantity := resource.NewQuantity(1, resource.DecimalSI) + resource := corev1.ResourceName(s.driverName + resourceSuffix) if ctnr.Resources.Requests == nil { ctnr.Resources.Requests = corev1.ResourceList{} } - ctnr.Resources.Requests[Resource] = *quantity + ctnr.Resources.Requests[resource] = *quantity if ctnr.Resources.Limits == nil { ctnr.Resources.Limits = corev1.ResourceList{} } - ctnr.Resources.Limits[Resource] = *quantity + ctnr.Resources.Limits[resource] = *quantity marshaledPod, err := json.Marshal(pod) if err != nil { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 51299b32af..a8e0b930c4 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -12,7 +12,6 @@ import ( "fmt" "net/http" "sort" - "strings" "sync" "github.com/go-logr/logr" @@ -178,17 +177,16 @@ func (s *scheduler) doFilter(args schedulerapi.ExtenderArgs) (*schedulerapi.Exte nodeName := nodeName waitgroup.Add(1) go func() { - fits, failReasons, err := s.nodeHasEnoughCapacity(required, nodeName) + err := s.nodeHasEnoughCapacity(required, nodeName) + mutex.Lock() defer mutex.Unlock() defer waitgroup.Done() - switch { - case fits: + switch err { + case nil: filteredNodes = append(filteredNodes, nodeName) - case failReasons != nil: - failedNodes[nodeName] = strings.Join(failReasons, ",") - case err != nil: - failedNodes[nodeName] = fmt.Sprintf("checking for capacity: %v", err) + default: + failedNodes[nodeName] = err.Error() } }() } @@ -274,23 +272,22 @@ func (s *scheduler) requiredStorage(pod *v1.Pod) (int64, error) { return total, nil } -// nodeHasEnoughCapacity determines whether a node has enough storage available. It either returns -// true if yes, a list of explanations why not, or an error if checking failed. -func (s *scheduler) nodeHasEnoughCapacity(required int64, nodeName string) (bool, []string, error) { +// nodeHasEnoughCapacity determines whether a node has enough storage available. It returns +// an error if not, otherwise nil. +func (s *scheduler) nodeHasEnoughCapacity(required int64, nodeName string) error { available, err := s.capacity.NodeCapacity(nodeName) if err != nil { - return false, nil, fmt.Errorf("retrieve capacity: %v", err) + return fmt.Errorf("retrieve capacity: %v", err) } if available < required { - return false, []string{fmt.Sprintf("only %vB of PMEM available, need %vB", + return fmt.Errorf("only %vB of PMEM available, need %vB", resource.NewQuantity(available, resource.BinarySI), - resource.NewQuantity(required, resource.BinarySI)), - }, nil + resource.NewQuantity(required, resource.BinarySI)) } // Success! - return true, nil, nil + return nil } func listNodeNames(nodes []v1.Node) []string { diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 785e3596a2..e47bb360a2 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionv1 "k8s.io/api/admission/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -345,7 +345,7 @@ func TestScheduler(t *testing.T) { }, nodes: []string{nodeA}, expectedFailures: map[string]string{ - nodeA: "checking for capacity: retrieve capacity: node node-A unknown", + nodeA: "retrieve capacity: node node-A unknown", }, }, "one volume, one node, enough capacity": { @@ -670,7 +670,7 @@ func TestMutatePod(t *testing.T) { obj, err := json.Marshal(pod) require.NoError(t, err, "encode pod") req := admission.Request{ - AdmissionRequest: admissionv1beta1.AdmissionRequest{ + AdmissionRequest: admissionv1.AdmissionRequest{ Namespace: "default", Object: runtime.RawExtension{ Raw: obj, diff --git a/pkg/types/types.go b/pkg/types/types.go new file mode 100644 index 0000000000..f73f7b8dc0 --- /dev/null +++ b/pkg/types/types.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 Intel Corporation. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package types contains some type definitions that are used in +// various places. +package types + +import ( + "bytes" + "encoding/json" + "strings" +) + +// NodeSelector is a set of unique keys and their values. +type NodeSelector map[string]string + +// Set converts a JSON representation into a NodeSelector. +func (n *NodeSelector) Set(value string) error { + // Decoding into a plain map yields better error messages: + // "cannot unmarshal string into Go value of type types.NodeSelector" + // vs. + // "cannot unmarshal string into Go value of type map[string]string" + var m map[string]string + if err := json.NewDecoder(bytes.NewBufferString(value)).Decode(&m); err != nil { + return err + } + *n = m + return nil +} + +// String converts into the JSON representation expected by Set. +func (n *NodeSelector) String() string { + var value bytes.Buffer + if err := json.NewEncoder(&value).Encode(n); err != nil { + panic(err) + } + return strings.TrimSpace(value.String()) +} + +// MatchesLabels returns true if all key/value pairs in the selector +// are set in the labels. +func (n *NodeSelector) MatchesLabels(labels map[string]string) bool { + for key, value := range *n { + if labels[key] != value { + return false + } + } + return true +} diff --git a/runtime-deps.csv b/runtime-deps.csv index 3db762ca49..d89ea6d0da 100644 --- a/runtime-deps.csv +++ b/runtime-deps.csv @@ -1,8 +1,10 @@ -Go,https://github.com/golang/go,9051 +Go,https://golang.org/,11382 client_golang,https://github.com/prometheus/client_golang, github.com/operator-framework/operator-lib -golang-protobuf,https://github.com/golang/protobuf, google uuid,https://github.com/google/uuid, grpc-go,https://github.com/grpc/grpc-go, -kubernetes,https://github.com/kubernetes/kubernetes,9641 +kubernetes,https://github.com/kubernetes/kubernetes,12141 kubernetes-sigs/controller-runtime,https://github.com/kubernetes-sigs/controller-runtime, +kubernetes-sigs/sig-storage-lib-external-provisioner,https://github.com/kubernetes-sigs/sig-storage-lib-external-provisioner, +prometheus client_model,https://github.com/prometheus/client_model, +prometheus_common,https://github.com/prometheus/common, diff --git a/test/cmd/watch-pvs/watch-pvs.go b/test/cmd/watch-pvs/watch-pvs.go new file mode 100644 index 0000000000..21c8f0c727 --- /dev/null +++ b/test/cmd/watch-pvs/watch-pvs.go @@ -0,0 +1,86 @@ +/* +Copyright 2017 The Kubernetes Authors. +Copyright 2020 Intel Coporation. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package main + +import ( + "context" + "os" + "time" + + "github.com/google/go-cmp/cmp" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +func toYAML(obj interface{}) string { + out, err := yaml.Marshal(obj) + if err != nil { + klog.Fatalf("marshal %+q: %v", obj, err) + } + return string(out) +} + +func main() { + ctx := context.Background() + + // get the KUBECONFIG from env if specified (useful for local/debug cluster) + kubeconfigEnv := os.Getenv("KUBECONFIG") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigEnv) + if err != nil { + klog.Fatalf("Failed to create config from KUBECONFIG=%s: %v", kubeconfigEnv, err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("Failed to create client: %v", err) + } + + factory := informers.NewSharedInformerFactory(clientset, time.Hour) + claimInformer := factory.Core().V1().PersistentVolumeClaims().Informer() + volumeInformer := factory.Core().V1().PersistentVolumes().Informer() + + claimHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + klog.Infof("PVC added:\n%s\n", toYAML(obj)) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + klog.Infof("PVC updated:\n%s\n%s\n", + toYAML(newObj), + cmp.Diff(oldObj, newObj), + ) + }, + DeleteFunc: func(obj interface{}) { + klog.Infof("PVC deleted:\n%s\n", toYAML(obj)) + }, + } + claimInformer.AddEventHandlerWithResyncPeriod(claimHandler, time.Hour) + + volumeHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + klog.Infof("PV added:\n%s\n", toYAML(obj)) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + klog.Infof("PV updated:\n%s\n%s\n", + toYAML(newObj), + cmp.Diff(oldObj, newObj), + ) + }, + DeleteFunc: func(obj interface{}) { + klog.Infof("PV deleted:\n%s\n", toYAML(obj)) + }, + } + volumeInformer.AddEventHandlerWithResyncPeriod(volumeHandler, time.Hour) + + factory.Start(ctx.Done()) + for { + } +} diff --git a/test/delete-deployment.sh b/test/delete-deployment.sh new file mode 100755 index 0000000000..a8fb679497 --- /dev/null +++ b/test/delete-deployment.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -o errexit + +TEST_DIRECTORY=${TEST_DIRECTORY:-$(dirname $(readlink -f $0))} +source ${TEST_CONFIG:-${TEST_DIRECTORY}/test-config.sh} + +CLUSTER=${CLUSTER:-pmem-govm} +REPO_DIRECTORY="${REPO_DIRECTORY:-$(dirname $(dirname $(readlink -f $0)))}" +CLUSTER_DIRECTORY="${CLUSTER_DIRECTORY:-${REPO_DIRECTORY}/_work/${CLUSTER}}" +SSH="${CLUSTER_DIRECTORY}/ssh.0" +KUBECTL="${SSH} kubectl" # Always use the kubectl installed in the cluster. + +kinds=" + deployments + replicasets + statefulsets + daemonsets + + clusterrolebindings + clusterroles + crd + csidrivers + mutatingwebhookconfigurations + pods + rolebindings + roles + serviceaccounts + services + storageclasses +" +for kind in $kinds; do + echo -n "$kind: " + ${KUBECTL} delete --all-namespaces -l pmem-csi.intel.com/deployment $kind + ${KUBECTL} delete --all-namespaces -l app.kubernetes.io/part-of=pmem-csi $kind +done diff --git a/test/e2e/deploy/cluster.go b/test/e2e/deploy/cluster.go index 702df18789..f8aac6c0f8 100644 --- a/test/e2e/deploy/cluster.go +++ b/test/e2e/deploy/cluster.go @@ -15,6 +15,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" @@ -90,33 +91,32 @@ func (c *Cluster) WaitForServicePort(serviceName, namespace string) int { return port } -// GetAppInstance looks for a pod with a certain app label and a specific host or pod IP. +// GetAppInstance looks for a pod with certain labels and a specific host or pod IP. // The IP may also be empty. -func (c *Cluster) GetAppInstance(ctx context.Context, app, ip, namespace string) (*v1.Pod, error) { - pods, err := c.cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) +func (c *Cluster) GetAppInstance(ctx context.Context, appLabels labels.Set, ip, namespace string) (*v1.Pod, error) { + pods, err := c.cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: appLabels.String()}) if err != nil { return nil, err } for _, p := range pods.Items { - if p.Labels["app"] == app && - (ip == "" || p.Status.HostIP == ip || p.Status.PodIP == ip) { + if ip == "" || p.Status.HostIP == ip || p.Status.PodIP == ip { return &p, nil } } - return nil, fmt.Errorf("no app %q in namespace %q with IP %q found", app, namespace, ip) + return nil, fmt.Errorf("no app %s in namespace %q with IP %q found", appLabels, namespace, ip) } // WaitForAppInstance waits for a running pod which matches the app // label, optional host or pod IP, and namespace. -func (c *Cluster) WaitForAppInstance(app, ip, namespace string) *v1.Pod { +func (c *Cluster) WaitForAppInstance(appLabels labels.Set, ip, namespace string) *v1.Pod { var pod *v1.Pod Eventually(func() bool { var err error ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - pod, err = c.GetAppInstance(ctx, app, ip, namespace) + pod, err = c.GetAppInstance(ctx, appLabels, ip, namespace) return err == nil && pod.Status.Phase == v1.PodRunning - }, "3m").Should(BeTrue(), "%s app running on host %s in '%s' namespace", app, ip, namespace) + }, "3m").Should(BeTrue(), "%s app running on host %s in '%s' namespace", appLabels, ip, namespace) return pod } diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index e00bac0ed7..c976b82dc3 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -18,20 +18,24 @@ import ( "os/exec" "reflect" "regexp" + "strconv" "strings" "time" "github.com/prometheus/common/expfmt" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/skipper" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemexec "github.com/intel/pmem-csi/pkg/exec" + "github.com/intel/pmem-csi/test/test-config" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -76,7 +80,7 @@ func WaitForOperator(c *Cluster, namespace string) *v1.Pod { // TODO(avalluri): At later point of time we should add readiness support // for the operator. Then we can query directly the operator if its ready. // As intrem solution we are just checking Pod.Status. - operator := c.WaitForAppInstance("pmem-csi-operator", "", namespace) + operator := c.WaitForAppInstance(labels.Set{"app": "pmem-csi-operator"}, "", namespace) ginkgo.By("Operator is ready!") return operator } @@ -85,7 +89,7 @@ func WaitForOperator(c *Cluster, namespace string) *v1.Pod { // is ready else fails with exception. func WaitForOLM(c *Cluster, namespace string) *v1.Pod { ginkgo.By("Waiting if the OLM deployment is ready...") - olm := c.WaitForAppInstance("olm-operator", "", namespace) + olm := c.WaitForAppInstance(labels.Set{"app": "olm-operator"}, "", namespace) ginkgo.By("OLM is ready!") return olm } @@ -95,9 +99,7 @@ func WaitForOLM(c *Cluster, namespace string) *v1.Pod { // - controller service is up and running // - all nodes have registered // - for testing deployments: TCP CSI endpoints are ready -// -// "name" is the common prefix used for objects of the deployment. -func WaitForPMEMDriver(c *Cluster, name string, d *Deployment) (metricsURL string) { +func WaitForPMEMDriver(c *Cluster, d *Deployment) (metricsURL string) { ticker := time.NewTicker(time.Second) defer ticker.Stop() info := time.NewTicker(time.Minute) @@ -105,6 +107,17 @@ func WaitForPMEMDriver(c *Cluster, name string, d *Deployment) (metricsURL strin deadline, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() + // "name" is the common prefix used for objects of the deployment. + // If we are testing against an older version of PMEM-CSI, then it is always "pmem-csi". + // PMEM-CSI 0.9.0 derives it from the CSI driver name. + var name string + switch d.Version { + case "0.7", "0.8": + name = "pmem-csi" + default: + name = strings.ReplaceAll(d.DriverName, ".", "-") + } + if waitForPMEMDriverTimedOut { // Abort early. skipper.Skipf("installing PMEM-CSI driver during previous test was too slow") @@ -131,71 +144,119 @@ func WaitForPMEMDriver(c *Cluster, name string, d *Deployment) (metricsURL strin deadline, cancel := context.WithTimeout(deadline, timeout) defer cancel() - // The controller service must be defined. - port, err := c.GetServicePort(deadline, name+"-metrics", d.Namespace) - if err != nil { - return fmt.Errorf("get port for service %s-metrics in namespace %s: %v", name, d.Namespace, err) - } + if d.HasController { + // The controller service must be defined. + port, err := c.GetServicePort(deadline, name+"-metrics", d.Namespace) + if err != nil { + return fmt.Errorf("get port for service %s-metrics in namespace %s: %v", name, d.Namespace, err) + } - // We can connect to it and get metrics data. - scheme := "http" - if d.Version == "0.7" { - scheme = "https" - } - metricsURL = fmt.Sprintf("%s://%s:%d/metrics", scheme, c.NodeIP(0), port) - client := &http.Client{ - Transport: &tr, - Timeout: timeout, - } - resp, err := client.Get(metricsURL) - if err != nil { - return fmt.Errorf("get controller metrics: %v", err) - } - if resp.StatusCode != 200 { - body, _ := ioutil.ReadAll(resp.Body) - suffix := "" - if len(body) > 0 { - suffix = "\n" + string(body) + // We can connect to it and get metrics data. + scheme := "http" + if d.Version == "0.7" { + scheme = "https" + } + metricsURL = fmt.Sprintf("%s://%s:%d/metrics", scheme, c.NodeIP(0), port) + client := &http.Client{ + Transport: &tr, + Timeout: timeout, + } + resp, err := client.Get(metricsURL) + if err != nil { + return fmt.Errorf("get controller metrics: %v", err) + } + if resp.StatusCode != 200 { + body, _ := ioutil.ReadAll(resp.Body) + suffix := "" + if len(body) > 0 { + suffix = "\n" + string(body) + } + return fmt.Errorf("HTTP GET %s failed: %d%s", metricsURL, resp.StatusCode, suffix) } - return fmt.Errorf("HTTP GET %s failed: %d%s", metricsURL, resp.StatusCode, suffix) - } - // Parse and check number of connected nodes. Dump the - // version number while we are at it. - parser := expfmt.TextParser{} - metrics, err := parser.TextToMetricFamilies(resp.Body) - if err != nil { - return fmt.Errorf("parse metrics response: %v", err) - } - buildInfo, ok := metrics["build_info"] - if !ok { - return fmt.Errorf("expected build_info not found in metrics: %v", metrics) - } - if len(buildInfo.Metric) != 1 { - return fmt.Errorf("expected build_info to have one metric, got: %v", buildInfo.Metric) - } - buildMetric := buildInfo.Metric[0] - if len(buildMetric.Label) != 1 { - return fmt.Errorf("expected build_info to have one label, got: %v", buildMetric.Label) - } - label := buildMetric.Label[0] - if *label.Name != "version" { - return fmt.Errorf("expected build_info to contain a version label, got: %s", *label.Name) - } - version = *label.Value + // Check metrics data. + parser := expfmt.TextParser{} + metrics, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return fmt.Errorf("parse metrics response: %v", err) + } + buildInfo, ok := metrics["build_info"] + if !ok { + return fmt.Errorf("expected build_info not found in metrics: %v", metrics) + } + if len(buildInfo.Metric) != 1 { + return fmt.Errorf("expected build_info to have one metric, got: %v", buildInfo.Metric) + } + buildMetric := buildInfo.Metric[0] + if len(buildMetric.Label) != 1 { + return fmt.Errorf("expected build_info to have one label, got: %v", buildMetric.Label) + } + label := buildMetric.Label[0] + if *label.Name != "version" { + return fmt.Errorf("expected build_info to contain a version label, got: %s", *label.Name) + } + version = *label.Value + + // With the older, centralized provisioning we + // can also check that the controller knows + // about all nodes. + switch d.Version { + case "0.7", "0.8": + pmemNodes, ok := metrics["pmem_nodes"] + if !ok { + return fmt.Errorf("expected pmem_nodes not found in metrics: %v", metrics) + } - pmemNodes, ok := metrics["pmem_nodes"] - if !ok { - return fmt.Errorf("expected pmem_nodes not found in metrics: %v", metrics) + if len(pmemNodes.Metric) != 1 { + return fmt.Errorf("expected pmem_nodes to have one metric, got: %v", pmemNodes.Metric) + } + nodesMetric := pmemNodes.Metric[0] + actualNodes := int(*nodesMetric.Gauge.Value) + if actualNodes != c.NumNodes()-1 { + return fmt.Errorf("only %d of %d nodes have registered", actualNodes, c.NumNodes()-1) + } + } } - if len(pmemNodes.Metric) != 1 { - return fmt.Errorf("expected pmem_nodes to have one metric, got: %v", pmemNodes.Metric) - } - nodesMetric := pmemNodes.Metric[0] - actualNodes := int(*nodesMetric.Gauge.Value) - if actualNodes != c.NumNodes()-1 { - return fmt.Errorf("only %d of %d nodes have registered", actualNodes, c.NumNodes()-1) + // Check status of every node driver. This is crucial for 0.9.0 + // because this is no longer covered by the controller + // metrics check that was used previously. + switch d.Version { + case "0.7", "0.8": + // No need to test and doesn't have the necessary labels. + default: + pods, err := c.cs.CoreV1().Pods(d.Namespace).List(context.Background(), + metav1.ListOptions{ + LabelSelector: labels.Set{ + "app.kubernetes.io/instance": d.DriverName, + "app.kubernetes.io/component": "node", + }.String(), + }, + ) + if err != nil { + return fmt.Errorf("list node pods: %v", err) + } + if len(pods.Items) != c.NumNodes()-1 { + return fmt.Errorf("only %d of %d node driver pods exist", len(pods.Items), c.NumNodes()-1) + } + for _, pod := range pods.Items { + if !podIsReady(pod.Status) { + return fmt.Errorf("node driver pod %s on node %s is not ready", pod.Name, pod.Spec.NodeName) + } + csiNode, err := c.cs.StorageV1().CSINodes().Get(context.Background(), + pod.Spec.NodeName, + metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get CSINode %s: %v", pod.Spec.NodeName, err) + } + if !driverHasRegistered(*csiNode, d.DriverName) { + return fmt.Errorf("PMEM-CSI driver %s not added to CSINode %+v yet", d.DriverName, csiNode) + } + + // It would be nice to check the metrics endpoint here, but reaching it from outside + // the cluster relies on port forwarding (tricky to set up) or some way to run curl + // inside the cluster (expensive because we would need to start a pod for it). + } } // Done for normal deployments. @@ -260,6 +321,27 @@ func WaitForPMEMDriver(c *Cluster, name string, d *Deployment) (metricsURL strin } } +func podIsReady(podStatus v1.PodStatus) bool { + if podStatus.Phase != v1.PodRunning { + return false + } + for _, condition := range podStatus.Conditions { + if condition.Type == v1.ContainersReady { + return condition.Status == v1.ConditionTrue + } + } + return false +} + +func driverHasRegistered(csiNode storagev1.CSINode, driverName string) bool { + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == driverName { + return true + } + } + return false +} + // https://github.com/containerd/containerd/issues/4068 var containerdTaskError = regexp.MustCompile(`failed to (start|create) containerd task`) @@ -361,9 +443,8 @@ func RemoveObjects(c *Cluster, deployment *Deployment) error { } } - // We intentionally delete statefulset last because that is - // how FindDeployment will find it again if we don't manage to - // delete the entire deployment. Here we just scale it down + // We intentionally delete statefulset last because + // findDriver checks for it. Here we just scale it down // to trigger pod deletion. if list, err := c.cs.AppsV1().StatefulSets("").List(context.Background(), filter); !failure(err) { for _, object := range list.Items { @@ -518,17 +599,22 @@ func RemoveObjects(c *Cluster, deployment *Deployment) error { // Deployment contains some information about a some deployed PMEM-CSI component(s). // Those components can be a full driver installation and/or just the operator. type Deployment struct { - // HasDriver is true if the driver itself is running. The - // driver is reacting to the usual pmem-csi.intel.com driver - // name. + // HasDriver is true if the driver itself is running. HasDriver bool + // The CSI driver name that the driver is using. Usually + // pmem-csi.intel.com. + DriverName string + // HasOperator is true if the operator is running. HasOperator bool // HasOLM is true if the OLM(OperatorLifecycleManager) is running. HasOLM bool + // HasController is true if the controller part with the webhooks is enabled. + HasController bool + // Mode is the driver mode of the deployment. Mode api.DeviceMode @@ -596,19 +682,21 @@ func FindDeployment(c *Cluster) (*Deployment, error) { if operator != nil && driver != nil && operator.Name() != driver.Name() { return nil, fmt.Errorf("found two different deployments: %s and %s", operator.Name(), driver.Name()) } - if operator != nil { - return operator, nil - } + // findDriver is able to discover some additional information, so return that result + // if we have both. if driver != nil { return driver, nil } + if operator != nil { + return operator, nil + } return nil, nil } var imageVersion = regexp.MustCompile(`pmem-csi-driver(?:-test)?:v(\d+\.\d+)`) func findDriver(c *Cluster) (*Deployment, error) { - list, err := c.cs.AppsV1().StatefulSets("").List(context.Background(), metav1.ListOptions{LabelSelector: deploymentLabel}) + list, err := c.cs.AppsV1().DaemonSets("").List(context.Background(), metav1.ListOptions{LabelSelector: deploymentLabel}) if err != nil { return nil, err } @@ -623,6 +711,21 @@ func findDriver(c *Cluster) (*Deployment, error) { } deployment.Namespace = list.Items[0].Namespace + drivers, err := c.cs.StorageV1beta1().CSIDrivers().List(context.Background(), metav1.ListOptions{LabelSelector: deploymentLabel}) + if err != nil { + return nil, err + } + if len(drivers.Items) != 1 { + return nil, fmt.Errorf("expected one CSIDriver info, got: %v", drivers) + } + deployment.DriverName = drivers.Items[0].Name + + controllers, err := c.cs.AppsV1().StatefulSets("").List(context.Background(), metav1.ListOptions{LabelSelector: deploymentLabel}) + if err != nil { + return nil, fmt.Errorf("checking for StatefulSet: %v", err) + } + deployment.HasController = len(controllers.Items) > 0 + // Derive the version from the image tag. The annotation doesn't include it. // If the version matches what we are currently testing, then we skip // the version (i.e. "current version" == "no explicit version"). @@ -685,24 +788,33 @@ var allDeployments = []string{ "direct-testing", "direct-production", "operator", + // Uses second.pmem-csi.intel.com as driver name. "operator-lvm-production", - "operator-direct-production", // Uses kube-system, to ensure that deployment in a namespace also works. - "olm", // operator installed by OLM + // Uses kube-system, to ensure that deployment in a namespace also works, + // and *no* controller. + "operator-direct-production", + "olm", // operator installed by OLM } var deploymentRE = regexp.MustCompile(`^(operator|olm)?-?(\w*)?-?(testing|production)?-?([0-9\.]*)$`) // Parse the deployment name and sets fields accordingly. func Parse(deploymentName string) (*Deployment, error) { deployment := &Deployment{ - Namespace: "default", + Namespace: "default", + DriverName: "pmem-csi.intel.com", + HasController: true, } - if deploymentName == "operator" { + switch deploymentName { + case "operator": // Run the operator tests in a dedicated namespace // to cover the non-default namespace usecase deployment.Namespace = "operator-test" - } - if deploymentName == "operator-direct-production" { + case "operator-direct-production": deployment.Namespace = "kube-system" + // No secret available in that namespace. + deployment.HasController = false + case "operator-lvm-production": + deployment.DriverName = "second.pmem-csi.intel.com" } matches := deploymentRE.FindStringSubmatch(deploymentName) @@ -800,7 +912,7 @@ func EnsureDeploymentNow(f *framework.Framework, deployment *Deployment) { framework.Logf("reusing existing %s PMEM-CSI components", deployment.Name()) // Do some sanity checks on the running deployment before the test. if deployment.HasDriver { - WaitForPMEMDriver(c, "pmem-csi", deployment) + WaitForPMEMDriver(c, deployment) CheckPMEMDriver(c, deployment) } if deployment.HasOperator { @@ -902,8 +1014,10 @@ func EnsureDeploymentNow(f *framework.Framework, deployment *Deployment) { } cmd := exec.Command("test/setup-deployment.sh") cmd.Dir = root + flavor := "" env = append(env, "REPO_ROOT="+root, + "TEST_KUBERNETES_FLAVOR="+flavor, "TEST_DEPLOYMENT_QUIET=quiet", "TEST_DEPLOYMENTMODE="+deployment.DeploymentMode(), "TEST_DRIVER_NAMESPACE="+deployment.Namespace, @@ -916,7 +1030,7 @@ func EnsureDeploymentNow(f *framework.Framework, deployment *Deployment) { // We check for a running driver the same way at the moment, by directly // looking at the driver state. Long-term we want the operator to do that // checking itself. - WaitForPMEMDriver(c, "pmem-csi", deployment) + WaitForPMEMDriver(c, deployment) CheckPMEMDriver(c, deployment) } } @@ -924,7 +1038,7 @@ func EnsureDeploymentNow(f *framework.Framework, deployment *Deployment) { // GetDriverDeployment returns the spec for the driver deployment that is used // for deployments like operator-lvm-production. func (d *Deployment) GetDriverDeployment() api.PmemCSIDeployment { - return api.PmemCSIDeployment{ + dep := api.PmemCSIDeployment{ // TypeMeta is needed because // DefaultUnstructuredConverter does not add it for us. Is there a better way? TypeMeta: metav1.TypeMeta{ @@ -932,7 +1046,7 @@ func (d *Deployment) GetDriverDeployment() api.PmemCSIDeployment { Kind: "PmemCSIDeployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: "pmem-csi", + Name: d.DriverName, Labels: map[string]string{ deploymentLabel: d.Label(), }, @@ -952,6 +1066,21 @@ func (d *Deployment) GetDriverDeployment() api.PmemCSIDeployment { }, }, } + + if d.HasController { + // The controller is enabled, using a secret that must have + // been prepared beforehand. + dep.Spec.ControllerTLSSecret = strings.ReplaceAll(d.DriverName, ".", "-") + "-controller-secret" + dep.Spec.MutatePods = api.MutatePodsAlways + portStr := testconfig.GetOrFail("TEST_SCHEDULER_EXTENDER_NODE_PORT") + port, err := strconv.ParseInt(portStr, 10, 32) + if err != nil { + panic(fmt.Errorf("not an int32: TEST_SCHEDULER_EXTENDER_NODE_PORT=%q: %v", portStr, err)) + } + dep.Spec.SchedulerNodePort = int32(port) + } + + return dep } // DeleteAllPods deletes all currently running pods that belong to the deployment. @@ -987,13 +1116,8 @@ func LookupCSIAddresses(c *Cluster, namespace string) (nodeAddress, controllerAd // node service will fail. nodeAddress = c.NodeServiceAddress(1, SocatPort) - // The cluster controller service can be reached via - // any node, what matters is the service port. - port, err := c.GetServicePort(context.Background(), "pmem-csi-controller-testing", namespace) - if err != nil { - return "", "", fmt.Errorf("get PMEM-CSI controller service port: %v", err) - } - controllerAddress = c.NodeServiceAddress(0, port) + // Also use that same node as controller. + controllerAddress = nodeAddress return } diff --git a/test/e2e/driver/driver.go b/test/e2e/driver/driver.go index f626ccb14a..008ae88d0b 100644 --- a/test/e2e/driver/driver.go +++ b/test/e2e/driver/driver.go @@ -50,6 +50,11 @@ type DynamicDriver interface { WithParameters(parameters map[string]string) DynamicDriver } +// CSIDriver exposes the CSI driver name, something that is normally hidden. +type CSIDriver interface { + GetCSIDriverName(config *testsuites.PerTestConfig) string +} + func New(name, csiDriverName string, fsTypes []string, scManifests map[string]string) testsuites.TestDriver { if fsTypes == nil { fsTypes = []string{"", "ext4", "xfs"} diff --git a/test/e2e/gotests/gotests.go b/test/e2e/gotests/gotests.go index 0db7cfc98b..122504216f 100644 --- a/test/e2e/gotests/gotests.go +++ b/test/e2e/gotests/gotests.go @@ -55,7 +55,7 @@ func runGoTest(f *framework.Framework, pkg string) { err = build.Run() framework.ExpectNoError(err, "compile test program for %s", pkg) - label := labels.SelectorFromSet(labels.Set(map[string]string{"app": "pmem-csi-node"})) + label := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "pmem-csi-node"})) pods, err := f.ClientSet.CoreV1().Pods("default").List(context.Background(), metav1.ListOptions{LabelSelector: label.String()}) framework.ExpectNoError(err, "list PMEM-CSI pods") Expect(pods.Items).NotTo(BeEmpty(), "have PMEM-CSI pods") @@ -65,7 +65,7 @@ func runGoTest(f *framework.Framework, pkg string) { pod.RunInPod(f, root, []string{"_work/test.test", "_work/evil-ca", "_work/pmem-ca", "deploy/crd/"}, "if _work/test.test -h 2>&1 | grep -q ginkgo; then "+ - "TEST_WORK=_work REPO_ROOT=. _work/test.test -test.v -ginkgo.v; else "+ - "TEST_WORK=_work REPO_ROOT=. _work/test.test -test.v; fi", + "TEST_WORK=_work REPO_ROOT=. _work/test.test -ginkgo.v; else "+ + "TEST_WORK=_work REPO_ROOT=. _work/test.test; fi", pmem.Namespace, pmem.Name, "pmem-driver") } diff --git a/test/e2e/metrics/metrics.go b/test/e2e/metrics/metrics.go index 952f09cc95..b3352ef632 100644 --- a/test/e2e/metrics/metrics.go +++ b/test/e2e/metrics/metrics.go @@ -50,7 +50,7 @@ var _ = deploy.Describe("direct-testing", "direct-testing-metrics", "", func(d * BeforeEach(func() { cluster, err := deploy.NewCluster(f.ClientSet, f.DynamicClient) framework.ExpectNoError(err, "get cluster information") - metricsURL = deploy.WaitForPMEMDriver(cluster, "pmem-csi", d) + metricsURL = deploy.WaitForPMEMDriver(cluster, d) }) It("works", func() { @@ -67,7 +67,7 @@ var _ = deploy.Describe("direct-testing", "direct-testing-metrics", "", func(d * // "testing" deployment. We just need to find one of // those pods... socatPods, err := f.ClientSet.CoreV1().Pods(d.Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app in ( pmem-csi-node-testing )", + LabelSelector: "app.kubernetes.io/name in ( pmem-csi-node-testing )", }) framework.ExpectNoError(err, "list socat pods") Expect(pods.Items).NotTo(BeEmpty(), "at least one socat pod should be running") @@ -115,16 +115,13 @@ Accept: */* if strings.HasPrefix(container.Name, "pmem") { Expect(stdout).To(ContainSubstring("go_threads "), name) Expect(stdout).To(ContainSubstring("process_open_fds "), name) - Expect(stdout).To(ContainSubstring("csi_plugin_operations_seconds "), name) - if strings.HasPrefix(pod.Name, "pmem-csi-controller") { - Expect(stdout).To(ContainSubstring("pmem_nodes "), name) - Expect(stdout).To(ContainSubstring("pmem_csi_controller_operations_seconds "), name) - } else { + if !strings.Contains(pod.Name, "controller") { + // Only the node driver implements CSI and manages volumes. + Expect(stdout).To(ContainSubstring("csi_plugin_operations_seconds "), name) Expect(stdout).To(ContainSubstring("pmem_amount_available "), name) Expect(stdout).To(ContainSubstring("pmem_amount_managed "), name) Expect(stdout).To(ContainSubstring("pmem_amount_max_volume_size "), name) Expect(stdout).To(ContainSubstring("pmem_amount_total "), name) - Expect(stdout).To(ContainSubstring("pmem_csi_node_operations_seconds "), name) } } else { Expect(stdout).To(ContainSubstring("csi_sidecar_operations_seconds "), name) diff --git a/test/e2e/operator/deployment_api.go b/test/e2e/operator/deployment_api.go index c1355d760f..5e053445e4 100644 --- a/test/e2e/operator/deployment_api.go +++ b/test/e2e/operator/deployment_api.go @@ -21,17 +21,17 @@ import ( "github.com/intel/pmem-csi/test/e2e/deploy" "github.com/intel/pmem-csi/test/e2e/operator/validate" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -174,15 +174,13 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { }, 2*time.Minute, time.Second, what, ": ", expected) } - ensureObjectRecovered := func(obj apiruntime.Object) { - meta, err := meta.Accessor(obj) - Expect(err).ShouldNot(HaveOccurred(), "get meta object") - framework.Logf("Waiting for deleted object recovered %T/%s", obj, meta.GetName()) - key := runtime.ObjectKey{Name: meta.GetName(), Namespace: meta.GetNamespace()} + ensureObjectRecovered := func(obj runtime.Object) { + framework.Logf("Waiting for deleted object recovered %T/%s", obj, obj.GetName()) + key := runtime.ObjectKey{Name: obj.GetName(), Namespace: obj.GetNamespace()} Eventually(func() error { return client.Get(context.TODO(), key, obj) }, "2m", "1s").ShouldNot(HaveOccurred(), "failed to recover object") - framework.Logf("Object %T/%s recovered", obj, meta.GetName()) + framework.Logf("Object %T/%s recovered", obj, obj.GetName()) } Context("deployment", func() { @@ -232,7 +230,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { defer deploy.DeleteDeploymentCR(f, deployment.Name) validateDriver(deployment) validateConditions(deployment.Name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, }) validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonRunning}) @@ -295,6 +292,7 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { deployment.Spec.Image = "" deployment.Spec.PMEMPercentage = 50 deployment.Spec.LogFormat = format + deployment.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) @@ -355,7 +353,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { corev1.ResourceMemory: resource.MustParse("200Mi"), }, } - testcases.SetTLSOrDie(spec) deployment = deploy.UpdateDeploymentCR(f, deployment) @@ -387,14 +384,10 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { It("shall be able to use custom CA certificates", func() { deployment := getDeployment("test-deployment-with-certificates") - testcases.SetTLSOrDie(&deployment.Spec) - deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) validateDriver(deployment, true) validateConditions(deployment.Name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, - api.CertsVerified: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, }) validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonRunning}) @@ -408,7 +401,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { defer deploy.DeleteDeploymentCR(f, deployment.Name) validateDriver(deployment, true) validateConditions(deployment.Name, map[api.DeploymentConditionType]corev1.ConditionStatus{ - api.CertsReady: corev1.ConditionTrue, api.DriverDeployed: corev1.ConditionTrue, }) @@ -430,43 +422,41 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { It("shall recover from conflicts", func() { deployment := getDeployment("test-recover-from-conflicts") - sec := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, + deployment.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" + se := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: deployment.GetHyphenedName() + "-registry-secrets", + Name: deployment.GetHyphenedName() + "-scheduler", Namespace: d.Namespace, }, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{ - "ca.crt": []byte("fake ca"), - "tls.key": []byte("fake key"), - "tls.crt": []byte("fake crt"), + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": "foobar"}, + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + {Port: 433}, + }, }, } - deleteSecret := func(name string) { + deleteService := func() { Eventually(func() error { - err := f.ClientSet.CoreV1().Secrets(d.Namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) - deploy.LogError(err, "Delete secret error: %v, will retry...", err) + err := f.ClientSet.CoreV1().Services(d.Namespace).Delete(context.Background(), se.Name, metav1.DeleteOptions{}) + deploy.LogError(err, "Delete service error: %v, will retry...", err) if errors.IsNotFound(err) { return nil } return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "delete secret %q", name) + }, "3m", "1s").ShouldNot(HaveOccurred(), "delete service %s", se.Name) } Eventually(func() error { - _, err := f.ClientSet.CoreV1().Secrets(d.Namespace).Create(context.Background(), sec, metav1.CreateOptions{}) - deploy.LogError(err, "create secret error: %v, will retry...", err) + _, err := f.ClientSet.CoreV1().Services(d.Namespace).Create(context.Background(), se, metav1.CreateOptions{}) + deploy.LogError(err, "create service error: %v, will retry...", err) return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "create secret %q", sec.Name) - defer deleteSecret(sec.Name) + }, "3m", "1s").ShouldNot(HaveOccurred(), "create service %s", se.Name) + defer deleteService() deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) - // The deployment should fail to create required secret(s) as it already + // The deployment should fail to create the required service as it already // exists and is owned by others. Eventually(func() bool { out := deploy.GetDeploymentCR(f, deployment.Name) @@ -474,8 +464,54 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { }, "3m", "1s").Should(BeTrue(), "deployment should fail %q", deployment.Name) validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonFailed}) - // Deleting the existing secret should make the deployment succeed. - deleteSecret(sec.Name) + // Deleting the existing service should make the deployment succeed. + deleteService() + validateDriver(deployment, true) + validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonRunning}) + }) + + It("shall recover from missing secret", func() { + deployment := getDeployment("test-recover-from-missing-secret") + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-controller-secret", + Namespace: d.Namespace, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + "ca.crt": []byte("fake ca"), + "tls.key": []byte("fake key"), + "tls.crt": []byte("fake crt"), + }, + } + deployment.Spec.ControllerTLSSecret = sec.Name + deployment = deploy.CreateDeploymentCR(f, deployment) + defer deploy.DeleteDeploymentCR(f, deployment.Name) + + // The deployment should fail because the required secret is missing. + Eventually(func() bool { + out := deploy.GetDeploymentCR(f, deployment.Name) + return out.Status.Phase == api.DeploymentPhaseFailed + }, "3m", "1s").Should(BeTrue(), "deployment should fail %q", deployment.Name) + validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonFailed}) + + // Creating the secret should make the deployment succeed. + deleteSecret := func() { + Eventually(func() error { + err := f.ClientSet.CoreV1().Secrets(d.Namespace).Delete(context.Background(), sec.Name, metav1.DeleteOptions{}) + deploy.LogError(err, "Delete secret error: %v, will retry...", err) + if errors.IsNotFound(err) { + return nil + } + return err + }, "3m", "1s").ShouldNot(HaveOccurred(), "delete secret %s", sec.Name) + } + Eventually(func() error { + _, err := f.ClientSet.CoreV1().Secrets(d.Namespace).Create(context.Background(), sec, metav1.CreateOptions{}) + deploy.LogError(err, "create secret error: %v, will retry...", err) + return err + }, "3m", "1s").ShouldNot(HaveOccurred(), "create secret %s", sec.Name) + defer deleteSecret() validateDriver(deployment, true) validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonRunning}) }) @@ -591,9 +627,10 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) - deploy.WaitForPMEMDriver(c, deployment.Name, + deploy.WaitForPMEMDriver(c, &deploy.Deployment{ - Namespace: d.Namespace, + Namespace: d.Namespace, + DriverName: driverName, }) validateDriver(deployment, true) @@ -690,87 +727,111 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { Context("recover", func() { Context("deleted sub-resources", func() { - tests := map[string]func(*api.PmemCSIDeployment) apiruntime.Object{ - "registry secret": func(dep *api.PmemCSIDeployment) apiruntime.Object { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: dep.RegistrySecretName(), Namespace: d.Namespace, - }, + tests := map[string]func(*api.PmemCSIDeployment) runtime.Object{ + "provisioner service account": func(dep *api.PmemCSIDeployment) runtime.Object { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerServiceAccountName(), Namespace: d.Namespace}, } }, - "node secret": func(dep *api.PmemCSIDeployment) apiruntime.Object { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: dep.NodeSecretName(), Namespace: d.Namespace}, + "webhooks service account": func(dep *api.PmemCSIDeployment) runtime.Object { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksServiceAccountName(), Namespace: d.Namespace}, } }, - "service account": func(dep *api.PmemCSIDeployment) apiruntime.Object { - return &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: dep.ServiceAccountName(), Namespace: d.Namespace}, + "scheduler service": func(dep *api.PmemCSIDeployment) runtime.Object { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: dep.SchedulerServiceName(), Namespace: d.Namespace}, } }, - "controller service": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "controller service": func(dep *api.PmemCSIDeployment) runtime.Object { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: dep.ControllerServiceName(), Namespace: d.Namespace}, } }, - "metrics service": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "metrics service": func(dep *api.PmemCSIDeployment) runtime.Object { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: dep.MetricsServiceName(), Namespace: d.Namespace}, } }, - "provisioner role": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "webhooks role": func(dep *api.PmemCSIDeployment) runtime.Object { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksRoleName(), Namespace: d.Namespace}, + } + }, + "webhooks role binding": func(dep *api.PmemCSIDeployment) runtime.Object { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksRoleBindingName(), Namespace: d.Namespace}, + } + }, + "webhooks cluster role": func(dep *api.PmemCSIDeployment) runtime.Object { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksClusterRoleName()}, + } + }, + "webhooks cluster role binding": func(dep *api.PmemCSIDeployment) runtime.Object { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksClusterRoleBindingName()}, + } + }, + "mutating webhook config": func(dep *api.PmemCSIDeployment) runtime.Object { + return &admissionregistrationv1beta1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: dep.MutatingWebhookName()}, + } + }, + "provisioner role": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerRoleName(), Namespace: d.Namespace}, } }, - "provisioner role binding": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "provisioner role binding": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerRoleBindingName(), Namespace: d.Namespace}, } }, - "provisioner cluster role": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "provisioner cluster role": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerClusterRoleName()}, } }, - "provisioner cluster role binding": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "provisioner cluster role binding": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerClusterRoleBindingName()}, } }, - "csi driver": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "csi driver": func(dep *api.PmemCSIDeployment) runtime.Object { return &storagev1beta1.CSIDriver{ ObjectMeta: metav1.ObjectMeta{Name: dep.GetName()}, } }, - "controller driver": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "controller driver": func(dep *api.PmemCSIDeployment) runtime.Object { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{Name: dep.ControllerDriverName(), Namespace: d.Namespace}, } }, - "node driver": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "node driver": func(dep *api.PmemCSIDeployment) runtime.Object { return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{Name: dep.NodeDriverName(), Namespace: d.Namespace}, } }, } - delete := func(obj apiruntime.Object) { - meta, err := meta.Accessor(obj) - Expect(err).ShouldNot(HaveOccurred(), "get meta object") + delete := func(obj runtime.Object) { Eventually(func() error { err := client.Delete(context.TODO(), obj) if err == nil || errors.IsNotFound(err) { return nil } return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "delete object '%T/%s", obj, meta.GetName()) - framework.Logf("Deleted object %T/%s", obj, meta.GetName()) + }, "3m", "1s").ShouldNot(HaveOccurred(), "delete object '%T/%s", obj, obj.GetName()) + framework.Logf("Deleted object %T/%s", obj, obj.GetName()) } for name, getter := range tests { name, getter := name, getter It(name, func() { + // Create a deployment with controller and webhook config. dep := getDeployment("recover-" + strings.ReplaceAll(name, " ", "-")) + dep.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" + dep.Spec.MutatePods = api.MutatePodsAlways deployment := deploy.CreateDeploymentCR(f, dep) defer deploy.DeleteDeploymentCR(f, dep.Name) validateDriver(deployment) @@ -784,8 +845,8 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { }) Context("conflicting update", func() { - tests := map[string]func(dep *api.PmemCSIDeployment) apiruntime.Object{ - "controller": func(dep *api.PmemCSIDeployment) apiruntime.Object { + tests := map[string]func(dep *api.PmemCSIDeployment) runtime.Object{ + "controller": func(dep *api.PmemCSIDeployment) runtime.Object { obj := &appsv1.StatefulSet{} key := runtime.ObjectKey{Name: dep.ControllerDriverName(), Namespace: d.Namespace} EventuallyWithOffset(1, func() error { @@ -800,7 +861,7 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { } return obj }, - "node driver": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "node driver": func(dep *api.PmemCSIDeployment) runtime.Object { obj := &appsv1.DaemonSet{} key := runtime.ObjectKey{Name: dep.NodeDriverName(), Namespace: d.Namespace} EventuallyWithOffset(1, func() error { @@ -815,7 +876,7 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { } return obj }, - "metrics service": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "metrics service": func(dep *api.PmemCSIDeployment) runtime.Object { obj := &corev1.Service{} key := runtime.ObjectKey{Name: dep.MetricsServiceName(), Namespace: d.Namespace} EventuallyWithOffset(1, func() error { @@ -831,7 +892,7 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { } return obj }, - "controller service": func(dep *api.PmemCSIDeployment) apiruntime.Object { + "controller service": func(dep *api.PmemCSIDeployment) runtime.Object { obj := &corev1.Service{} key := runtime.ObjectKey{Name: dep.ControllerServiceName(), Namespace: d.Namespace} EventuallyWithOffset(1, func() error { @@ -853,6 +914,7 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { name, mutate := name, mutate It(name, func() { dep := getDeployment("recover-" + strings.ReplaceAll(name, " ", "-")) + dep.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" deployment := deploy.CreateDeploymentCR(f, dep) defer deploy.DeleteDeploymentCR(f, dep.Name) validateDriver(deployment) @@ -1015,7 +1077,7 @@ func stopOperator(c *deploy.Cluster, d *deploy.Deployment) error { Eventually(func() bool { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - _, err := c.GetAppInstance(ctx, "pmem-csi-operator", "", d.Namespace) + _, err := c.GetAppInstance(ctx, labels.Set{"app": "pmem-csi-operator"}, "", d.Namespace) deploy.LogError(err, "get operator error: %v, will retry...", err) return err != nil && strings.HasPrefix(err.Error(), "no app") }, "3m", "1s").Should(BeTrue(), "delete operator pod") @@ -1057,7 +1119,10 @@ func switchDeploymentMode(c *deploy.Cluster, f *framework.Framework, depName, ns for i := 1; i < c.NumNodes(); i++ { Eventually(func() error { - pod, err := c.GetAppInstance(context.Background(), depName+"-node", c.NodeIP(i), ns) + pod, err := c.GetAppInstance(context.Background(), + labels.Set{"app.kubernetes.io/name": "pmem-csi-node", + "app.kubernetes.io/instance": depName}, + c.NodeIP(i), ns) if err != nil { return err } @@ -1082,9 +1147,10 @@ func switchDeploymentMode(c *deploy.Cluster, f *framework.Framework, depName, ns }, "3m", "1s").Should(BeTrue(), "Pod restart '%s'", pod) } - deploy.WaitForPMEMDriver(c, depName, + deploy.WaitForPMEMDriver(c, &deploy.Deployment{ - Namespace: ns, + Namespace: ns, + DriverName: depName, }) return deployment diff --git a/test/e2e/operator/driver.go b/test/e2e/operator/driver.go index acdfb115b3..bd72286b6a 100644 --- a/test/e2e/operator/driver.go +++ b/test/e2e/operator/driver.go @@ -14,7 +14,9 @@ import ( "github.com/intel/pmem-csi/test/e2e/deploy" "github.com/intel/pmem-csi/test/e2e/driver" "github.com/intel/pmem-csi/test/e2e/operator/validate" + "github.com/intel/pmem-csi/test/e2e/storage" "github.com/intel/pmem-csi/test/e2e/storage/dax" + "github.com/intel/pmem-csi/test/e2e/storage/scheduler" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testsuites" @@ -57,6 +59,14 @@ var _ = deploy.DescribeForSome("driver", func(d *deploy.Deployment) bool { var csiTestSuites = []func() testsuites.TestSuite{ dax.InitDaxTestSuite, } + if d.HasController { + // Scheduler tests depend on the webhooks in the controller. + csiTestSuites = append(csiTestSuites, scheduler.InitSchedulerTestSuite) + } testsuites.DefineTestSuite(csiTestDriver, csiTestSuites) + + // Late binding must work, regardless of the driver name and whether we have + // a scheduler extender. + storage.DefineLateBindingTests(d) }) diff --git a/test/e2e/operator/validate/validate.go b/test/e2e/operator/validate/validate.go index 14801cdaa2..5bb0c38426 100644 --- a/test/e2e/operator/validate/validate.go +++ b/test/e2e/operator/validate/validate.go @@ -147,25 +147,6 @@ func DriverDeployment(client client.Client, k8sver version.Version, namespace st } expected := findObject(expectedObjects, actual) if expected == nil { - if actual.GetKind() == "Secret" { - // Custom comparison against expected - // content of secrets, which aren't - // part of the reference objects. - switch actual.GetName() { - case deployment.GetHyphenedName() + "-registry-secrets": - diffs = append(diffs, compareSecrets(actual, - deployment.Spec.CACert, - deployment.Spec.RegistryPrivateKey, - deployment.Spec.RegistryCert)...) - continue - case deployment.GetHyphenedName() + "-node-secrets": - diffs = append(diffs, compareSecrets(actual, - deployment.Spec.CACert, - deployment.Spec.NodeControllerPrivateKey, - deployment.Spec.NodeControllerCert)...) - continue - } - } diffs = append(diffs, fmt.Sprintf("unexpected object was deployed: %s", prettyPrintObjectID(actual))) continue } @@ -215,14 +196,7 @@ func DriverDeployment(client client.Client, k8sver version.Version, namespace st } } } - gvk := schema.GroupVersionKind{ - Kind: "Secret", - Version: "v1", - } - for _, expected := range append(expectedObjects, - // Content doesn't matter, we just want to be sure they exist. - createObject(gvk, deployment.GetHyphenedName()+"-registry-secrets", namespace), - createObject(gvk, deployment.GetHyphenedName()+"-node-secrets", namespace)) { + for _, expected := range expectedObjects { if findObject(objects, expected) == nil { diffs = append(diffs, fmt.Sprintf("expected object was not deployed: %v", prettyPrintObjectID(expected))) } @@ -315,6 +289,10 @@ func parseDefaultValues() map[string]interface{} { imagePullPolicy: IfNotPresent ports: protocol: TCP + env: + valueFrom: + fieldRef: + apiVersion: v1 volumes: secret: defaultMode: 420` @@ -340,6 +318,20 @@ StatefulSet:` + defaultsApps + ` CSIDriver: spec: storageCapacity: false +MutatingWebhookConfiguration: + webhooks: + clientConfig: + caBundle: ignore # content varies, correctness is validated during E2E testing + service: + port: 443 + admissionReviewVersions: + - v1beta1 + matchPolicy: Exact + reinvocationPolicy: Never + rules: + scope: "*" + sideEffects: Unknown + timeoutSeconds: 30 ` err := yaml.UnmarshalStrict([]byte(defaultsYAML), &defaults) @@ -505,7 +497,7 @@ func listAllDeployedObjects(c client.Client, deployment api.PmemCSIDeployment, n // Test client does not support differentiating cluster-scoped objects // and the query fails when fetch those object by setting the namespace- switch list.GetKind() { - case "CSIDriverList", "ClusterRoleList", "ClusterRoleBindingList": + case "CSIDriverList", "ClusterRoleList", "ClusterRoleBindingList", "MutatingWebhookConfigurationList": opts = &client.ListOptions{} } // Filtering by owner doesn't work, so we have to use brute-force and look at all diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 514fc2a84d..43aa3b8723 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -29,17 +29,19 @@ import ( "github.com/intel/pmem-csi/test/e2e/storage/dax" "github.com/intel/pmem-csi/test/e2e/storage/scheduler" "github.com/intel/pmem-csi/test/e2e/versionskew" + "github.com/intel/pmem-csi/test/test-config" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/storage/podlogs" "k8s.io/kubernetes/test/e2e/storage/testsuites" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var ( @@ -48,7 +50,7 @@ var ( ) var _ = deploy.DescribeForAll("E2E", func(d *deploy.Deployment) { - csiTestDriver := driver.New(d.Name(), "pmem-csi.intel.com", nil, nil) + csiTestDriver := driver.New(d.Name(), d.DriverName, nil, nil) // List of testSuites to be added below. var csiTestSuites = []func() testsuites.TestSuite{ @@ -70,20 +72,35 @@ var _ = deploy.DescribeForAll("E2E", func(d *deploy.Deployment) { } testsuites.DefineTestSuite(csiTestDriver, csiTestSuites) + DefineLateBindingTests(d) + DefineKataTests(d) +}) + +func DefineLateBindingTests(d *deploy.Deployment) { + f := framework.NewDefaultFramework("latebinding") Context("late binding", func() { var ( - storageClassLateBindingName = "pmem-csi-sc-late-binding" // from deploy/common/pmem-storageclass-late-binding.yaml - claim v1.PersistentVolumeClaim + cleanup func() + sc *storagev1.StorageClass + claim v1.PersistentVolumeClaim ) - f := framework.NewDefaultFramework("latebinding") + BeforeEach(func() { - // Check whether storage class exists before trying to use it. - _, err := f.ClientSet.StorageV1().StorageClasses().Get(context.Background(), storageClassLateBindingName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - skipper.Skipf("storage class %s not found, late binding not supported", storageClassLateBindingName) + csiTestDriver := driver.New(d.Name(), d.DriverName, nil, nil) + config, cl := csiTestDriver.PrepareTest(f) + cleanup = cl + sc = csiTestDriver.(testsuites.DynamicPVTestDriver).GetDynamicProvisionStorageClass(config, "ext4") + lateBindingMode := storagev1.VolumeBindingWaitForFirstConsumer + sc.VolumeBindingMode = &lateBindingMode + + // Create or replace storage class. + err := f.ClientSet.StorageV1().StorageClasses().Delete(context.Background(), sc.Name, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + framework.ExpectNoError(err, "delete old storage class %s", sc.Name) } - framework.ExpectNoError(err, "get storage class %s", storageClassLateBindingName) + _, err = f.ClientSet.StorageV1().StorageClasses().Create(context.Background(), sc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create storage class %s", sc.Name) claim = v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -99,19 +116,43 @@ var _ = deploy.DescribeForAll("E2E", func(d *deploy.Deployment) { v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Mi"), }, }, - StorageClassName: &storageClassLateBindingName, + StorageClassName: &sc.Name, }, } }) + AfterEach(func() { + err := f.ClientSet.StorageV1().StorageClasses().Delete(context.Background(), sc.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "delete old storage class %s", sc.Name) + if cleanup != nil { + cleanup() + } + }) + It("works", func() { TestDynamicLateBindingProvisioning(f.ClientSet, &claim, "latebinding") }) - // This test is pending because it triggers volumes leaks - // in PMEM-CSI (https://github.com/intel/pmem-csi/issues/823). - // We need to fix those before enabling the test again. - PIt("stress test", func() { + It("unsets unsuitable selected node", func() { + nodes, err := f.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + framework.ExpectNoError(err, "list nodes") + selectedNode := "" + nodeLabelName, nodeLabelValue := testconfig.GetNodeLabelOrFail() + for _, node := range nodes.Items { + if node.Labels[nodeLabelName] != nodeLabelValue { + selectedNode = node.Name + break + } + } + Expect(selectedNode).NotTo(BeEmpty(), "have a node without PMEM-CSI") + claim.Annotations = map[string]string{ + "volume.kubernetes.io/selected-node": selectedNode, + "volume.beta.kubernetes.io/storage-provisioner": d.DriverName, + } + TestDynamicLateBindingProvisioning(f.ClientSet, &claim, "latebinding") + }) + + It("stress test [Slow]", func() { // We cannot test directly whether pod and // volume were created on the same node by // chance or because the code enforces it. @@ -156,7 +197,9 @@ var _ = deploy.DescribeForAll("E2E", func(d *deploy.Deployment) { wg.Wait() }) }) +} +func DefineKataTests(d *deploy.Deployment) { // Also run some limited tests with Kata Containers, using different // storage classes than usual. kataDriver := driver.New(d.Name()+"-pmem-csi-kata", "pmem-csi.intel.com", @@ -171,4 +214,4 @@ var _ = deploy.DescribeForAll("E2E", func(d *deploy.Deployment) { dax.InitDaxTestSuite, }) }) -}) +} diff --git a/test/e2e/storage/sanity.go b/test/e2e/storage/sanity.go index 9ed5ef96e9..1dcca10e84 100644 --- a/test/e2e/storage/sanity.go +++ b/test/e2e/storage/sanity.go @@ -24,7 +24,6 @@ import ( "os" "os/exec" "regexp" - "sort" "strings" "sync" "sync/atomic" @@ -47,7 +46,6 @@ import ( clientset "k8s.io/client-go/kubernetes" clientexec "k8s.io/client-go/util/exec" "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/framework/skipper" testutils "k8s.io/kubernetes/test/utils" @@ -130,7 +128,11 @@ var _ = deploy.DescribeForSome("sanity", func(d *deploy.Deployment) bool { if socat != nil { return socat } - socat = cluster.WaitForAppInstance("pmem-csi-node-testing", cluster.NodeIP(1), d.Namespace) + socat = cluster.WaitForAppInstance(labels.Set{ + "app.kubernetes.io/component": "node-testing", + "app.kubernetes.io/part-of": "pmem-csi", + }, + cluster.NodeIP(1), d.Namespace) return socat } @@ -333,7 +335,7 @@ var _ = deploy.DescribeForSome("sanity", func(d *deploy.Deployment) bool { v.namePrefix = "mount-volume" pods, err := WaitForPodsWithLabelRunningReady(f.ClientSet, d.Namespace, - labels.Set{"app": "pmem-csi-node"}.AsSelector(), cluster.NumNodes()-1, time.Minute) + labels.Set{"app.kubernetes.io/name": "pmem-csi-node"}.AsSelector(), cluster.NumNodes()-1, time.Minute) framework.ExpectNoError(err, "All node drivers are not ready") name, vol := v.create(22*1024*1024, nodeID) @@ -366,7 +368,7 @@ var _ = deploy.DescribeForSome("sanity", func(d *deploy.Deployment) bool { It("capacity is restored after controller restart", func() { By("Fetching pmem-csi-controller pod name") pods, err := WaitForPodsWithLabelRunningReady(f.ClientSet, d.Namespace, - labels.Set{"app": "pmem-csi-controller"}.AsSelector(), 1 /* one replica */, time.Minute) + labels.Set{"app.kubernetes.io/name": "pmem-csi-controller"}.AsSelector(), 1 /* one replica */, time.Minute) framework.ExpectNoError(err, "PMEM-CSI controller running with one replica") controllerNode := pods.Items[0].Spec.NodeName canRestartNode(controllerNode) @@ -379,7 +381,7 @@ var _ = deploy.DescribeForSome("sanity", func(d *deploy.Deployment) bool { restartNode(f.ClientSet, controllerNode, sc) _, err = WaitForPodsWithLabelRunningReady(f.ClientSet, d.Namespace, - labels.Set{"app": "pmem-csi-controller"}.AsSelector(), 1 /* one replica */, 5*time.Minute) + labels.Set{"app.kubernetes.io/name": "pmem-csi-controller"}.AsSelector(), 1 /* one replica */, 5*time.Minute) framework.ExpectNoError(err, "PMEM-CSI controller running again with one replica") By("waiting for full capacity") @@ -710,64 +712,6 @@ var _ = deploy.DescribeForSome("sanity", func(d *deploy.Deployment) bool { v.remove(vol, volName) }) - It("supports cache volumes", func() { - v.namePrefix = "cache" - - // Create a cache volume with as many instances as nodes. - sc.Config.TestVolumeParameters = map[string]string{ - "persistencyModel": "cache", - "cacheSize": fmt.Sprintf("%d", len(nodes)), - } - sizeInBytes := int64(33 * 1024 * 1024) - volName, vol := v.create(sizeInBytes, "") - sc.Config.TestVolumeParameters = map[string]string{} - var expectedTopology []*csi.Topology - // These node names are sorted. - readyNodes, err := e2enode.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "get schedulable nodes") - for _, node := range readyNodes.Items { - if node.Labels["storage"] != "pmem" { - continue - } - expectedTopology = append(expectedTopology, &csi.Topology{ - Segments: map[string]string{ - "pmem-csi.intel.com/node": node.Name, - }, - }) - } - // vol.AccessibleTopology isn't, so we have to sort before comparing. - sort.Slice(vol.AccessibleTopology, func(i, j int) bool { - return strings.Compare( - vol.AccessibleTopology[i].Segments["pmem-csi.intel.com/node"], - vol.AccessibleTopology[j].Segments["pmem-csi.intel.com/node"], - ) < 0 - }) - Expect(vol.AccessibleTopology).To(Equal(expectedTopology), "cache volume topology") - - // Each node now should have one additional volume, - // and its size should match the requested one. - for nodeName, node := range nodes { - currentVolumes, err := node.cc.ListVolumes(context.Background(), &csi.ListVolumesRequest{}) - framework.ExpectNoError(err, "list volumes on node %s via %s", nodeName) - Expect(len(currentVolumes.Entries)).To(Equal(len(node.volumes)+1), "one additional volume on node %s", nodeName) - for _, e := range currentVolumes.Entries { - if e.Volume.VolumeId == vol.VolumeId { - Expect(e.Volume.CapacityBytes).To(Equal(sizeInBytes), "additional volume size on node %s(%s)", nodeName, node.host) - break - } - } - } - - v.remove(vol, volName) - - // Now those volumes are gone again. - for nodeName, node := range nodes { - currentVolumes, err := node.cc.ListVolumes(context.Background(), &csi.ListVolumesRequest{}) - framework.ExpectNoError(err, "list volumes on node %s", nodeName) - Expect(len(currentVolumes.Entries)).To(Equal(len(node.volumes)), "same volumes as before on node %s", nodeName) - } - }) - Context("ephemeral volumes", func() { doit := func(withFlag bool, repeatCalls int) { targetPath := sc.TargetPath + "/ephemeral" diff --git a/test/e2e/storage/scheduler/scheduler.go b/test/e2e/storage/scheduler/scheduler.go index a1d6e3839b..357589a65c 100644 --- a/test/e2e/storage/scheduler/scheduler.go +++ b/test/e2e/storage/scheduler/scheduler.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" + e2edriver "github.com/intel/pmem-csi/test/e2e/driver" "github.com/intel/pmem-csi/test/e2e/ephemeral" . "github.com/onsi/ginkgo" @@ -109,12 +110,15 @@ func (p *schedulerTestSuite) DefineTests(driver testsuites.TestDriver, pattern t init() defer cleanup() - l.testSchedulerInPod(f, l.resource.Pattern.VolType, l.resource.VolSource, l.config) + driverName := driver.(e2edriver.CSIDriver).GetCSIDriverName(l.config) + + l.testSchedulerInPod(f, driverName, l.resource.Pattern.VolType, l.resource.VolSource, l.config) }) } func (l local) testSchedulerInPod( f *framework.Framework, + driverName string, volumeType testpatterns.TestVolType, source *v1.VolumeSource, config *testsuites.PerTestConfig) { @@ -163,12 +167,14 @@ func (l local) testSchedulerInPod( podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) }() + resourceName := v1.ResourceName(driverName + "/scheduler") + Expect(createdPod.Spec.Containers[0].Resources).NotTo(BeNil(), "pod resources") Expect(createdPod.Spec.Containers[0].Resources.Requests).NotTo(BeNil(), "pod resource requests") - _, ok := createdPod.Spec.Containers[0].Resources.Requests["pmem-csi.intel.com/scheduler"] + _, ok := createdPod.Spec.Containers[0].Resources.Requests[resourceName] Expect(ok).To(BeTrue(), "PMEM-CSI extended resource request") Expect(createdPod.Spec.Containers[0].Resources.Limits).NotTo(BeNil(), "pod resource requests") - _, ok = createdPod.Spec.Containers[0].Resources.Requests["pmem-csi.intel.com/scheduler"] + _, ok = createdPod.Spec.Containers[0].Resources.Requests[resourceName] Expect(ok).To(BeTrue(), "PMEM-CSI extended resource limit") podErr := e2epod.WaitForPodRunningInNamespace(f.ClientSet, createdPod) diff --git a/test/e2e/tls/tls.go b/test/e2e/tls/tls.go index 6553f83616..6de023f66b 100644 --- a/test/e2e/tls/tls.go +++ b/test/e2e/tls/tls.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/framework/skipper" "github.com/intel/pmem-csi/test/e2e/deploy" pmempod "github.com/intel/pmem-csi/test/e2e/pod" @@ -36,7 +37,7 @@ var _ = deploy.DescribeForAll("TLS", func(d *deploy.Deployment) { var nodePod *v1.Pod BeforeEach(func() { // Find one node driver pod. - label := labels.SelectorFromSet(labels.Set(map[string]string{"app": "pmem-csi-node"})) + label := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "pmem-csi-node"})) pods, err := f.ClientSet.CoreV1().Pods("default").List(context.Background(), metav1.ListOptions{LabelSelector: label.String()}) framework.ExpectNoError(err, "list PMEM-CSI node pods") Expect(pods.Items).NotTo(BeEmpty(), "have PMEM-CSI node pods") @@ -45,7 +46,10 @@ var _ = deploy.DescribeForAll("TLS", func(d *deploy.Deployment) { Context("controller", func() { It("is secure", func() { - checkTLS(f, "pmem-csi-controller-0.pmem-csi-controller.default") + if !d.HasController { + skipper.Skipf("has no controller") + } + checkTLS(f, "pmem-csi-intel-com-controller-0.pmem-csi-intel-com-controller."+d.Namespace) }) }) Context("node", func() { diff --git a/test/e2e/versionskew/versionskew.go b/test/e2e/versionskew/versionskew.go index 94d07f3b09..829856f7ed 100644 --- a/test/e2e/versionskew/versionskew.go +++ b/test/e2e/versionskew/versionskew.go @@ -15,6 +15,7 @@ package versionskew import ( "context" "fmt" + "strings" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/skipper" @@ -39,8 +40,11 @@ import ( ) const ( + // TODO: remove this and all code using it when no longer testing against 0.8 + base_08 = "0.8" + // base is the release branch used for version skew testing. Empty if none. - base = "0.8" + base = base_08 ) func baseSupportsKubernetes(ver version.Version) bool { @@ -263,15 +267,17 @@ func (p *skewTestSuite) DefineTests(driver testsuites.TestDriver, pattern testpa } deploy.EnsureDeploymentNow(f, deployment) - // Work around volume leak (https://github.com/intel/pmem-csi/issues/733) by - // waiting for controller to know about all volumes. - switch pattern.VolType { - case testpatterns.CSIInlineVolume: - // One running pod -> one volume. - waitForVolumes(1) - default: - // Three stand-alone volumes. - waitForVolumes(3) + if strings.Contains(otherName, base_08) { + // Work around volume leak (https://github.com/intel/pmem-csi/issues/733) by + // waiting for controller to know about all volumes. + switch pattern.VolType { + case testpatterns.CSIInlineVolume: + // One running pod -> one volume. + waitForVolumes(1) + default: + // Three stand-alone volumes. + waitForVolumes(3) + } } // Use some other volume. @@ -309,6 +315,10 @@ func (p *skewTestSuite) DefineTests(driver testsuites.TestDriver, pattern testpa // and if there compatibility issues, then hopefully the direction // of the skew won't matter. It("controller [Slow]", func() { + if base == base_08 { + skipper.Skipf("current controller not compatible with PMEM-CSI 0.8") + } + withKataContainers := false c, err := deploy.NewCluster(f.ClientSet, f.DynamicClient) @@ -363,7 +373,7 @@ func (p *skewTestSuite) DefineTests(driver testsuites.TestDriver, pattern testpa framework.ExpectNoError(err, "get cluster information") mixedDeployment := *deployment mixedDeployment.Version = "" - deploy.WaitForPMEMDriver(c, "pmem-csi", &mixedDeployment) + deploy.WaitForPMEMDriver(c, &mixedDeployment) // This relies on FindDeployment getting the version number from the image. deployment, err = deploy.FindDeployment(c) diff --git a/test/setup-ca-kubernetes.sh b/test/setup-ca-kubernetes.sh index ae27cf9a09..d623e4afb9 100755 --- a/test/setup-ca-kubernetes.sh +++ b/test/setup-ca-kubernetes.sh @@ -16,8 +16,9 @@ tmpdir=`mktemp -d` trap 'rm -r $tmpdir' EXIT # Generate certificates. They are not going to be needed again and will -# be deleted together with the temp directory. -WORKDIR="$tmpdir" "$TEST_DIRECTORY/setup-ca.sh" +# be deleted together with the temp directory. Only the root CA is +# stored in a permanent location. +WORKDIR="$tmpdir" CA="$TEST_CA" NS="${TEST_DRIVER_NAMESPACE}" PREFIX="${TEST_DRIVER_PREFIX}" "$TEST_DIRECTORY/setup-ca.sh" # This reads a file and encodes it for use in a secret. read_key () { @@ -26,8 +27,13 @@ read_key () { # Read certificate files and turn them into Kubernetes secrets. # +# The "registry" part in the file and variable names is historic. +# PMEM-CSI < 0.9.0 used that certificate for the node registry +# and webhooks. PMEM-CSI >= 0.9.0 only uses it for the webhooks +# and no longer has such a registry. +# # -caFile (controller and all nodes) -CA=$(read_key "$tmpdir/ca.pem") +CA=$(read_key "${TEST_CA}.pem") # -certFile (controller) REGISTRY_CERT=$(read_key "$tmpdir/pmem-registry.pem") # -keyFile (controller) @@ -37,12 +43,13 @@ NODE_CERT=$(read_key "$tmpdir/pmem-node-controller.pem") # -keyFile (same for all nodes) NODE_KEY=$(read_key "$tmpdir/pmem-node-controller-key.pem") -kubectl get ns ${TEST_DRIVER_NAMESPACE} 2>/dev/null >/dev/null || kubectl create ns ${TEST_DRIVER_NAMESPACE} +${KUBECTL} get ns ${TEST_DRIVER_NAMESPACE} 2>/dev/null >/dev/null || ${KUBECTL} create ns ${TEST_DRIVER_NAMESPACE} ${KUBECTL} apply -f - <= 0.9.0. +# It still gets created to support downgrades. +apiVersion: v1 +kind: Secret +metadata: + name: pmem-csi-node-secrets + namespace: ${TEST_DRIVER_NAMESPACE} type: Opaque data: ca.crt: ${CA} diff --git a/test/setup-ca.sh b/test/setup-ca.sh index a756302409..3880faaf81 100755 --- a/test/setup-ca.sh +++ b/test/setup-ca.sh @@ -6,6 +6,7 @@ mkdir -p $WORKDIR cd $WORKDIR CA=${CA:="$WORKDIR/ca"} NS=${NS:-pmem-csi} +PREFIX=${PREFIX:-pmem-csi-intel-com} # Check for cfssl utilities. cfssl_found=1 @@ -36,15 +37,21 @@ DEFAULT_CNS="pmem-registry pmem-node-controller" CNS="${DEFAULT_CNS} ${EXTRA_CNS:=""}" for name in ${CNS}; do echo "Generating Certificate for '$name'(NS=$NS) ..." - </dev/null >/dev/null || ${KUBECTL} create ns ${TEST_DRIVER_NAMESPACE} - -${KUBECTL} apply -f - <'$tmpdir/my-deployment/scheduler-patch.yaml'" <'$tmpdir/my-deployment/lvm-parameters-patch.yaml'" <>'$tmpdir/my-deployment/node-label-patch.yaml'" <'$tmpdir/my-deployment/scheduler-patch.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <&2 "$path is missing." + echo >&2 "$paths are all missing." exit 1 ;; esac diff --git a/test/setup-kubernetes.sh b/test/setup-kubernetes.sh index 0adba0b19b..4703f0ba5b 100755 --- a/test/setup-kubernetes.sh +++ b/test/setup-kubernetes.sh @@ -82,6 +82,14 @@ list_gates () ( # filter, the extender is only going to be called for pods which # explicitly enable it and thus other pods (including PMEM-CSI # itself!) can be scheduled without it. +# +# In order to reach the scheduler extender, a fixed node port +# is used regardless of the driver name, so only one deployment +# can be active at once. In production this has to be solved +# differently. +# +# Usually the driver name will be "pmem-csi.intel.com", but for testing +# purposed we also configure a second extender. sudo mkdir -p /var/lib/scheduler/ sudo cp ca.crt /var/lib/scheduler/ @@ -118,6 +126,18 @@ EOF "name": "pmem-csi.intel.com/scheduler", "ignoredByScheduler": true }] + }, + { + "urlPrefix": "https://127.0.0.1:${TEST_SCHEDULER_EXTENDER_NODE_PORT}", + "filterVerb": "filter", + "prioritizeVerb": "prioritize", + "nodeCacheCapable": true, + "weight": 1, + "managedResources": + [{ + "name": "second.pmem-csi.intel.com/scheduler", + "ignoredByScheduler": true + }] }] } EOF @@ -139,6 +159,14 @@ extenders: managedResources: - name: pmem-csi.intel.com/scheduler ignoredByScheduler: true +- urlPrefix: https://127.0.0.1:${TEST_SCHEDULER_EXTENDER_NODE_PORT} + filterVerb: filter + prioritizeVerb: prioritize + nodeCacheCapable: true + weight: 1 + managedResources: + - name: second.pmem-csi.intel.com/scheduler + ignoredByScheduler: true EOF ;; esac @@ -165,6 +193,9 @@ apiServer: controllerManager: extraArgs: feature-gates: ${TEST_FEATURE_GATES} + # Let the kube-controller-manager run as fast as it can. + kube-api-burst: \"100000\" + kube-api-qps: \"100000\" scheduler: extraVolumes: - name: config diff --git a/test/start-operator.sh b/test/start-operator.sh index 7a9665b41f..2d2535f8aa 100755 --- a/test/start-operator.sh +++ b/test/start-operator.sh @@ -141,6 +141,10 @@ case $deploy_method in exit 1 ;; esac +# Set up TLS secrets in the TEST_OPERATOR_NAMESPACE, with the two different prefixes. +PATH="${REPO_DIRECTORY}/_work/bin:$PATH" TEST_DRIVER_NAMESPACE="${TEST_OPERATOR_NAMESPACE}" TEST_DRIVER_PREFIX=second-pmem-csi-intel-com ${TEST_DIRECTORY}/setup-ca-kubernetes.sh +PATH="${REPO_DIRECTORY}/_work/bin:$PATH" TEST_DRIVER_NAMESPACE="${TEST_OPERATOR_NAMESPACE}" ${TEST_DIRECTORY}/setup-ca-kubernetes.sh + cat <