From ef4d10e616b1fd1468a8b8ab038d36b5a8051f45 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Wed, 9 Oct 2024 11:58:19 +0530 Subject: [PATCH 1/6] Adding Integration tests for disabling SDC dependency on NFS --- env.sh | 5 + service/controller.go | 2 +- test/integration/features/integration.feature | 22 ++- test/integration/integration_test.go | 2 +- test/integration/run.sh | 10 +- test/integration/step_defs_test.go | 173 +++++++++++++++++- 6 files changed, 200 insertions(+), 14 deletions(-) diff --git a/env.sh b/env.sh index a3eac969..a4334456 100644 --- a/env.sh +++ b/env.sh @@ -24,6 +24,11 @@ export SDC_GUID=$(/bin/emc/scaleio/drv_cfg --query_guid) # Alternate GUID is for another system for testing expose volume to multiple hosts export ALT_GUID= +# Kubernetes variables +export KUBE_CONFIG=/root/.kube/config +export KUBE_NODE_NAME="" +export NODE_INTERFACES="worker1:InterfaceIP1, worker2:InterfaceIP2" + #Debug variables for goscaleio library export GOSCALEIO_SHOWHTTP="true" diff --git a/service/controller.go b/service/controller.go index 723ec4b2..157ea21a 100644 --- a/service/controller.go +++ b/service/controller.go @@ -1291,9 +1291,9 @@ func (s *service) ControllerPublishVolume( var ipAddresses []string ipAddresses, err = s.findNetworkInterfaceIPs() - Log.Printf("ControllerPublish - No network interfaces found, trying to get SDC IPs") if err != nil || len(ipAddresses) == 0 { + Log.Printf("ControllerPublish - No network interfaces found, trying to get SDC IPs") // get SDC IPs if Network Interface IPs not found ipAddresses, err = s.getSDCIPs(nodeID, systemID) if err != nil { diff --git a/test/integration/features/integration.feature b/test/integration/features/integration.feature index e0222c5c..df322a74 100644 --- a/test/integration/features/integration.feature +++ b/test/integration/features/integration.feature @@ -565,7 +565,25 @@ Feature: VxFlex OS CSI interface Then there are no errors Examples: | voltype | access | fstype | errormsg | - | "mount" | "single-writer" | "nfs" | "none" | + | "mount" | "single-writer" | "nfs" | "none" | + + Scenario Outline: Create publish, node-publish, node-unpublish, unpublish, and delete nfs volume without SDC dependency + Given a VxFlexOS service + And a nfs capability with voltype access fstype + And a nfs volume request "nfsinttestvol" "8" + When I call CreateVolume + And there are no errors + And when I call PublishVolume for nfs + And when I call NodePublishVolume for nfs + And there are no errors + And when I call NodeUnpublishVolume for nfs + And when I call UnpublishVolume for nfs + And there are no errors + And when I call DeleteVolume + Then there are no errors + Examples: + | voltype | access | fstype | errormsg | + | "mount" | "single-writer" | "nfs" | "none" | Scenario: Expand Nfs Volume Given a VxFlexOS service @@ -920,4 +938,4 @@ Scenario: Custom file system format options (mkfsFormatOption) | "mount" | "-L MyVolume -m 1" | "single-node-single-writer" | "ext4" | "none" | | "mount" | "-T largefile4" |"single-node-multi-writer" | "ext4" | "none" | | "mount" | ":-L MyVolume" | "single-writer" | "xfs" | "error performing private mount" | - | "mount" | "abc" | "single-node-single-writer" | "ext4" | "error performing private mount" | \ No newline at end of file + | "mount" | "abc" | "single-node-single-writer" | "ext4" | "error performing private mount" | diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 07a84836..880860eb 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { Format: "junit", Output: outputfile, Paths: []string{"features"}, - // Tags: "wip", + Tags: "wip", } exitVal := godog.TestSuite{ diff --git a/test/integration/run.sh b/test/integration/run.sh index f3336b89..9a93a06b 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -16,12 +16,12 @@ # on this system. This will make real calls to the SIO. # NOTE: you must run this as root, as the plugin cannot retrieve the SdcGUID without being root! -sh validate_http_unauthorized.sh -rc=$? -if [ $rc -ne 0 ]; then echo "failed http unauthorized test"; exit $rc; fi +#sh validate_http_unauthorized.sh +#rc=$? +#if [ $rc -ne 0 ]; then echo "failed http unauthorized test"; exit $rc; fi -rm -f unix.sock -source ../../env.sh +rm -f unix_sock +. ../../env.sh echo $SDC_GUID GOOS=linux CGO_ENABLED=0 GO111MODULE=on go test -v -coverprofile=c.linux.out -timeout 60m -coverpkg=github.com/dell/csi-vxflexos/service *test.go & if [ -f ./csi-sanity ] ; then diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index ec5707d9..1932ebd6 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -20,7 +20,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dell/csi-vxflexos/v2/service" "io" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/clientcmd" "log" "math" "net/http" @@ -45,11 +51,13 @@ import ( ) const ( - MaxRetries = 10 - RetrySleepTime = 10 * time.Second - SleepTime = 100 * time.Millisecond - Pool1 = "pool1" - NfsPool = "Env8-SP-SW_SSD-1" + MaxRetries = 10 + RetrySleepTime = 10 * time.Second + SleepTime = 100 * time.Millisecond + Pool1 = "poo11" + NfsPool = "Env8-SP-SW_SSD-1" + DriverConfigMap = "vxflexos-config-params" + DriverNamespace = "vxflexos" ) // ArrayConnectionData contains data required to connect to array @@ -215,6 +223,26 @@ func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) { return arrays, nil } +func (f *feature) GetNodeUID() (string, error) { + + kubeConfig := os.Getenv("KUBE_CONFIG") + config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return "", fmt.Errorf("error building kubeconfig: %s", err.Error()) + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return "", fmt.Errorf("error building kubernetes clientset: %s", err.Error()) + } + + node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), os.Getenv("KUBE_NODE_NAME"), v1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("error getting node: %s", err.Error()) + } + return string(node.UID), nil +} + func (f *feature) addError(err error) { f.errs = append(f.errs, err) } @@ -2170,6 +2198,21 @@ func (f *feature) aNfsVolumeRequest(name string, size int64) error { return nil } +func (f *feature) whenICallPublishVolumeForNfsWithoutSDC() error { + if f.createVolumeRequest == nil { + return nil + } + err := f.controllerPublishVolumeForNfsWithoutSDC(f.volID) + if err != nil { + fmt.Printf("ControllerPublishVolume %s:\n", err.Error()) + f.addError(err) + } else { + fmt.Printf("ControllerPublishVolume completed successfully\n") + } + time.Sleep(SleepTime) + return nil +} + func (f *feature) whenICallPublishVolumeForNfs(nodeIDEnvVar string) error { if f.createVolumeRequest == nil { return nil @@ -2185,6 +2228,64 @@ func (f *feature) whenICallPublishVolumeForNfs(nodeIDEnvVar string) error { return nil } +func (f *feature) controllerPublishVolumeForNfsWithoutSDC(id string) error { + if f.createVolumeRequest == nil { + return nil + } + req := f.getControllerPublishVolumeRequest() + req.VolumeId = id + req.NodeId, _ = f.GetNodeUID() + + clientSet := fake.NewSimpleClientset() + service.K8sClientset = clientSet + var configYAMLContent strings.Builder + + for _, iface := range strings.Split(os.Getenv("NODE_INTERFACES"), ",") { + interfaceData := strings.Split(strings.TrimSpace(iface), ":") + configYAMLContent.WriteString(fmt.Sprintf(" %s: %s\n", interfaceData[0], interfaceData[1])) + } + + configMapData := map[string]string{ + "driver-config-params.yaml": fmt.Sprintf(`interfaceNames: +%s`, configYAMLContent.String()), + } + + configMap := &apiv1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: DriverConfigMap, + Namespace: DriverNamespace, + }, + Data: configMapData, + } + + _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, v1.CreateOptions{}) + if err != nil { + fmt.Printf("Failed to create configMap: %v\n", err) + return err + } + + if f.arrays == nil { + fmt.Printf("Initialize ArrayConfig from %s:\n", configFile) + var err error + f.arrays, err = f.getArrayConfig() + if err != nil { + return errors.New("Get multi array config failed " + err.Error()) + } + } + + for _, a := range f.arrays { + req.VolumeContext = make(map[string]string) + req.VolumeContext["nasName"] = a.NasName + req.VolumeContext["fsType"] = "nfs" + ctx := context.Background() + client := csi.NewControllerClient(grpcClient) + _, err := client.ControllerPublishVolume(ctx, req) + return err + } + + return nil +} + func (f *feature) controllerPublishVolumeForNfs(id string, nodeIDEnvVar string) error { if f.createVolumeRequest == nil { return nil @@ -2242,6 +2343,21 @@ func (f *feature) getNodePublishVolumeRequestForNfs() *csi.NodePublishVolumeRequ return req } +func (f *feature) whenICallNodePublishVolumeForNfsWithoutSDC() error { + if f.createVolumeRequest == nil { + return nil + } + err := f.nodePublishVolumeForNfs(f.volID, "") + if err != nil { + fmt.Printf("NodePublishVolume failed: %s\n", err.Error()) + f.addError(err) + } else { + fmt.Printf("NodePublishVolume completed successfully\n") + } + time.Sleep(SleepTime) + return nil +} + //nolint:revive func (f *feature) whenICallNodePublishVolumeForNfs(arg1 string) error { if f.createVolumeRequest == nil { @@ -2280,6 +2396,21 @@ func (f *feature) nodePublishVolumeForNfs(id string, path string) error { return err } +func (f *feature) whenICallNodeUnpublishVolumeForNfsWithoutSDC() error { + if f.createVolumeRequest == nil { + return nil + } + err := f.nodeUnpublishVolumeForNfs(f.volID, f.nodePublishVolumeRequest.TargetPath) + if err != nil { + fmt.Printf("NodeUnpublishVolume failed: %s\n", err.Error()) + f.addError(err) + } else { + fmt.Printf("NodeUnpublishVolume completed successfully\n") + } + time.Sleep(SleepTime) + return nil +} + //nolint:revive func (f *feature) whenICallNodeUnpublishVolumeForNfs(arg1 string) error { if f.createVolumeRequest == nil { @@ -2307,6 +2438,21 @@ func (f *feature) nodeUnpublishVolumeForNfs(id string, path string) error { return err } +func (f *feature) whenICallUnpublishVolumeForNfsWithoutSDC() error { + if f.createVolumeRequest == nil { + return nil + } + err := f.controllerUnpublishVolumeForNfsWithoutSDC(f.publishVolumeRequest.VolumeId) + if err != nil { + fmt.Printf("ControllerUnpublishVolume failed: %s\n", err.Error()) + f.addError(err) + } else { + fmt.Printf("ControllerUnpublishVolume completed successfully\n") + } + time.Sleep(SleepTime) + return nil +} + func (f *feature) whenICallUnpublishVolumeForNfs(nodeIDEnvVar string) error { if f.createVolumeRequest == nil { return nil @@ -2322,6 +2468,19 @@ func (f *feature) whenICallUnpublishVolumeForNfs(nodeIDEnvVar string) error { return nil } +func (f *feature) controllerUnpublishVolumeForNfsWithoutSDC(id string) error { + if f.createVolumeRequest == nil { + return nil + } + req := new(csi.ControllerUnpublishVolumeRequest) + req.VolumeId = id + req.NodeId, _ = f.GetNodeUID() + ctx := context.Background() + client := csi.NewControllerClient(grpcClient) + _, err := client.ControllerUnpublishVolume(ctx, req) + return err +} + func (f *feature) controllerUnpublishVolumeForNfs(id string, nodeIDEnvVar string) error { if f.createVolumeRequest == nil { return nil @@ -2572,6 +2731,10 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^when I call NodePublishVolume for nfs "([^"]*)"$`, f.whenICallNodePublishVolumeForNfs) s.Step(`^when I call NodeUnpublishVolume for nfs "([^"]*)"$`, f.whenICallNodeUnpublishVolumeForNfs) s.Step(`^when I call UnpublishVolume for nfs "([^"]*)"$`, f.whenICallUnpublishVolumeForNfs) + s.Step(`^when I call PublishVolume for nfs$`, f.whenICallPublishVolumeForNfsWithoutSDC) + s.Step(`^when I call NodePublishVolume for nfs$`, f.whenICallNodePublishVolumeForNfsWithoutSDC) + s.Step(`^when I call NodeUnpublishVolume for nfs$`, f.whenICallNodeUnpublishVolumeForNfsWithoutSDC) + s.Step(`^when I call UnpublishVolume for nfs$`, f.whenICallUnpublishVolumeForNfsWithoutSDC) s.Step(`^when I call NfsExpandVolume to "([^"]*)"$`, f.whenICallNfsExpandVolumeTo) s.Step(`^I call ListFileSystemSnapshot$`, f.ICallListFileSystemSnapshot) s.Step(`^I call CreateSnapshotForFS$`, f.iCallCreateSnapshotForFS) From e1d40d2ad78e1240c402ac7d7e5cb751145cdfe2 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Wed, 9 Oct 2024 14:05:12 +0530 Subject: [PATCH 2/6] Adding integration tests for Volume Expansion --- env.sh | 2 +- go.mod | 1 + go.sum | 1 + test/integration/features/integration.feature | 46 ++++++++++++++++++- test/integration/step_defs_test.go | 32 ++++++++++++- 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/env.sh b/env.sh index a4334456..1d4ecdbd 100644 --- a/env.sh +++ b/env.sh @@ -27,7 +27,7 @@ export ALT_GUID= # Kubernetes variables export KUBE_CONFIG=/root/.kube/config export KUBE_NODE_NAME="" -export NODE_INTERFACES="worker1:InterfaceIP1, worker2:InterfaceIP2" +export NODE_INTERFACES="nodeName:interfaceName" #Debug variables for goscaleio library export GOSCALEIO_SHOWHTTP="true" diff --git a/go.mod b/go.mod index 80c03d55..e4247ffd 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( github.com/hashicorp/go-memdb v1.3.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect diff --git a/go.sum b/go.sum index cc1b92b2..5e8f9b06 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/test/integration/features/integration.feature b/test/integration/features/integration.feature index df322a74..0c4a4969 100644 --- a/test/integration/features/integration.feature +++ b/test/integration/features/integration.feature @@ -582,8 +582,8 @@ Feature: VxFlex OS CSI interface And when I call DeleteVolume Then there are no errors Examples: - | voltype | access | fstype | errormsg | - | "mount" | "single-writer" | "nfs" | "none" | + | voltype | access | fstype | + | "mount" | "single-writer" | "nfs" | Scenario: Expand Nfs Volume Given a VxFlexOS service @@ -606,6 +606,27 @@ Feature: VxFlex OS CSI interface And when I call DeleteVolume Then there are no errors + Scenario: Expand Nfs Volume without SDC dependency + Given a VxFlexOS service + And a nfs capability with voltype "mount" access "single-writer" fstype "nfs" + And a nfs volume request "nfsinttestvol2" "16" + When I call CreateVolume + And there are no errors + And when I call PublishVolume for nfs + And there are no errors + And when I call NodePublishVolume for nfs + And there are no errors + And when I call NfsExpandVolume to "20" + And there are no errors + And I call ListVolume + And a valid ListVolumeResponse is returned + And when I call NodeUnpublishVolume for nfs + And there are no errors + And when I call UnpublishVolume for nfs + And there are no errors + And when I call DeleteVolume + Then there are no errors + Scenario: NFS Create volume, create snapshot, delete volume Given a VxFlexOS service And a basic nfs volume request "nfsvolume1" "8" @@ -713,6 +734,27 @@ Feature: VxFlex OS CSI interface And when I call DeleteVolume Then there are no errors + Scenario: Expand Nfs Volume without SDC dependency with tree quota enabled + Given a VxFlexOS service + And a nfs capability with voltype "mount" access "single-writer" fstype "nfs" + And a basic nfs volume request with quota enabled volname "vol-quota" volsize "10" path "/nfs-quotakk" softlimit "80" graceperiod "86400" + When I call CreateVolume + And there are no errors + And when I call PublishVolume for nfs + And there are no errors + And when I call NodePublishVolume for nfs + And there are no errors + And when I call NfsExpandVolume to "15" + And there are no errors + And I call ListVolume + And a valid ListVolumeResponse is returned + And when I call NodeUnpublishVolume for nfs + And there are no errors + And when I call UnpublishVolume for nfs + And there are no errors + And when I call DeleteVolume + Then there are no errors + Scenario: Expand Nfs Volume with tree quota enabled given invalid volume size for exapnd volume Given a VxFlexOS service diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index 1932ebd6..ba144b72 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "log" "math" + "net" "net/http" "os" "os/exec" @@ -243,6 +244,29 @@ func (f *feature) GetNodeUID() (string, error) { return string(node.UID), nil } +func (f *feature) getIPAddressByInterface(interfaceName string) (string, error) { + interfaceObj, err := net.InterfaceByName(interfaceName) + if err != nil { + return "", err + } + + addrs, err := interfaceObj.Addrs() + if err != nil { + return "", err + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + if ipNet.IP.To4() != nil { + return ipNet.IP.String(), nil + } + } + return "", fmt.Errorf("no IPv4 address found for interface %s", interfaceName) +} + func (f *feature) addError(err error) { f.errs = append(f.errs, err) } @@ -2241,8 +2265,14 @@ func (f *feature) controllerPublishVolumeForNfsWithoutSDC(id string) error { var configYAMLContent strings.Builder for _, iface := range strings.Split(os.Getenv("NODE_INTERFACES"), ",") { + interfaceData := strings.Split(strings.TrimSpace(iface), ":") - configYAMLContent.WriteString(fmt.Sprintf(" %s: %s\n", interfaceData[0], interfaceData[1])) + interfaceIP, err := f.getIPAddressByInterface(interfaceData[1]) + if err != nil { + fmt.Printf("Error while getting IP address for interface %s: %v\n", interfaceData[1], err) + continue + } + configYAMLContent.WriteString(fmt.Sprintf(" %s: %s\n", interfaceData[0], interfaceIP)) } configMapData := map[string]string{ From a770eb26d15373ddbe461fc7ebb10ccaf4d27709 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Wed, 9 Oct 2024 14:38:09 +0530 Subject: [PATCH 3/6] Resolving liniting issues --- go.mod | 2 -- test/integration/integration_test.go | 2 +- test/integration/run.sh | 10 +++++----- test/integration/step_defs_test.go | 19 ++++++++++--------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index e4247ffd..2b2db86c 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,6 @@ module github.com/dell/csi-vxflexos/v2 go 1.22.0 -toolchain go1.22.5 - require ( github.com/akutz/memconn v0.1.0 github.com/apparentlymart/go-cidr v1.1.0 diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 880860eb..07a84836 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { Format: "junit", Output: outputfile, Paths: []string{"features"}, - Tags: "wip", + // Tags: "wip", } exitVal := godog.TestSuite{ diff --git a/test/integration/run.sh b/test/integration/run.sh index 9a93a06b..f3336b89 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -16,12 +16,12 @@ # on this system. This will make real calls to the SIO. # NOTE: you must run this as root, as the plugin cannot retrieve the SdcGUID without being root! -#sh validate_http_unauthorized.sh -#rc=$? -#if [ $rc -ne 0 ]; then echo "failed http unauthorized test"; exit $rc; fi +sh validate_http_unauthorized.sh +rc=$? +if [ $rc -ne 0 ]; then echo "failed http unauthorized test"; exit $rc; fi -rm -f unix_sock -. ../../env.sh +rm -f unix.sock +source ../../env.sh echo $SDC_GUID GOOS=linux CGO_ENABLED=0 GO111MODULE=on go test -v -coverprofile=c.linux.out -timeout 60m -coverpkg=github.com/dell/csi-vxflexos/service *test.go & if [ -f ./csi-sanity ] ; then diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index ba144b72..db5cb7e0 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -20,13 +20,7 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dell/csi-vxflexos/v2/service" "io" - apiv1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/clientcmd" "log" "math" "net" @@ -40,6 +34,14 @@ import ( "syscall" "time" + "github.com/dell/csi-vxflexos/v2/service" + + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/clientcmd" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -55,8 +57,8 @@ const ( MaxRetries = 10 RetrySleepTime = 10 * time.Second SleepTime = 100 * time.Millisecond - Pool1 = "poo11" - NfsPool = "Env8-SP-SW_SSD-1" + Pool1 = "SP-SW_SSD-1" + NfsPool = "SP-SW_SSD-1" DriverConfigMap = "vxflexos-config-params" DriverNamespace = "vxflexos" ) @@ -225,7 +227,6 @@ func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) { } func (f *feature) GetNodeUID() (string, error) { - kubeConfig := os.Getenv("KUBE_CONFIG") config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { From 0f082addb0c14c2315d1cab4729092f0aca160f8 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Thu, 10 Oct 2024 12:59:39 +0530 Subject: [PATCH 4/6] Changes for getting NodeUID --- env.sh | 4 +- service/controller.go | 2 + test/integration/run.sh | 4 +- test/integration/step_defs_test.go | 128 ++++++++++++++++++----------- 4 files changed, 87 insertions(+), 51 deletions(-) diff --git a/env.sh b/env.sh index 1d4ecdbd..dba10ff7 100644 --- a/env.sh +++ b/env.sh @@ -24,9 +24,7 @@ export SDC_GUID=$(/bin/emc/scaleio/drv_cfg --query_guid) # Alternate GUID is for another system for testing expose volume to multiple hosts export ALT_GUID= -# Kubernetes variables -export KUBE_CONFIG=/root/.kube/config -export KUBE_NODE_NAME="" +# Interface variables export NODE_INTERFACES="nodeName:interfaceName" #Debug variables for goscaleio library diff --git a/service/controller.go b/service/controller.go index 157ea21a..e8633503 100644 --- a/service/controller.go +++ b/service/controller.go @@ -1624,6 +1624,7 @@ func (s *service) ControllerUnpublishVolume( ipAddresses, err = s.findNetworkInterfaceIPs() if err != nil || len(ipAddresses) == 0 { + Log.Printf("ControllerUnPublish - No network interfaces found, trying to get SDC IPs") ipAddresses, err = s.getSDCIPs(nodeID, systemID) if err != nil { return nil, status.Errorf(codes.NotFound, "%s", err.Error()) @@ -1631,6 +1632,7 @@ func (s *service) ControllerUnpublishVolume( return nil, status.Errorf(codes.NotFound, "%s", "received empty sdcIPs") } } + Log.Printf("ControllerUnPublish - ipAddresses %v", ipAddresses) // unexport for NFS err = s.unexportFilesystem(ctx, req, adminClient, fs, req.GetVolumeId(), ipAddresses, nodeID) diff --git a/test/integration/run.sh b/test/integration/run.sh index f3336b89..754504c7 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -20,8 +20,8 @@ sh validate_http_unauthorized.sh rc=$? if [ $rc -ne 0 ]; then echo "failed http unauthorized test"; exit $rc; fi -rm -f unix.sock -source ../../env.sh +rm -f unix_sock +. ../../env.sh echo $SDC_GUID GOOS=linux CGO_ENABLED=0 GO111MODULE=on go test -v -coverprofile=c.linux.out -timeout 60m -coverpkg=github.com/dell/csi-vxflexos/service *test.go & if [ -f ./csi-sanity ] ; then diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index db5cb7e0..70a732f5 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -34,16 +34,15 @@ import ( "syscall" "time" + "github.com/dell/csi-vxflexos/v2/k8sutils" + "github.com/dell/csi-vxflexos/v2/service" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" apiv1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/clientcmd" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/cucumber/godog" @@ -57,8 +56,9 @@ const ( MaxRetries = 10 RetrySleepTime = 10 * time.Second SleepTime = 100 * time.Millisecond - Pool1 = "SP-SW_SSD-1" - NfsPool = "SP-SW_SSD-1" + Pool1 = "pool1" + NfsPool = "Env8-SP-SW_SSD-1" + NodeName = "node1" DriverConfigMap = "vxflexos-config-params" DriverNamespace = "vxflexos" ) @@ -226,21 +226,65 @@ func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) { return arrays, nil } -func (f *feature) GetNodeUID() (string, error) { - kubeConfig := os.Getenv("KUBE_CONFIG") - config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) - if err != nil { - return "", fmt.Errorf("error building kubeconfig: %s", err.Error()) +func (f *feature) createConfigMap() error { + var configYAMLContent strings.Builder + + for _, iface := range strings.Split(os.Getenv("NODE_INTERFACES"), ",") { + + interfaceData := strings.Split(strings.TrimSpace(iface), ":") + interfaceIP, err := f.getIPAddressByInterface(interfaceData[1]) + if err != nil { + fmt.Printf("Error while getting IP address for interface %s: %v\n", interfaceData[1], err) + continue + } + configYAMLContent.WriteString(fmt.Sprintf(" %s: %s\n", interfaceData[0], interfaceIP)) + } + + configMapData := map[string]string{ + "driver-config-params.yaml": fmt.Sprintf(`interfaceNames: +%s`, configYAMLContent.String()), } - clientSet, err := kubernetes.NewForConfig(config) + configMap := &apiv1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: DriverConfigMap, + Namespace: DriverNamespace, + }, + Data: configMapData, + } + + _, err := service.K8sClientset.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, v1.CreateOptions{}) if err != nil { - return "", fmt.Errorf("error building kubernetes clientset: %s", err.Error()) + fmt.Printf("Failed to create configMap: %v\n", err) + return err + } + return nil +} + +func (f *feature) setFakeNode() (*apiv1.Node, error) { + fakeNode := &apiv1.Node{ + ObjectMeta: v1.ObjectMeta{ + Name: NodeName, + Labels: map[string]string{"label1": "value1", "label2": "value2"}, + UID: "1aa4c285-d41b-4911-bf3e-621253bfbade", + }, + } + return service.K8sClientset.CoreV1().Nodes().Create(context.TODO(), fakeNode, v1.CreateOptions{}) +} + +func (f *feature) GetNodeUID() (string, error) { + if service.K8sClientset == nil { + err := k8sutils.CreateKubeClientSet() + if err != nil { + return "", fmt.Errorf("init client failed with error: %v", err) + } + service.K8sClientset = k8sutils.Clientset } - node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), os.Getenv("KUBE_NODE_NAME"), v1.GetOptions{}) + // access the API to fetch node object + node, err := service.K8sClientset.CoreV1().Nodes().Get(context.TODO(), NodeName, v1.GetOptions{}) if err != nil { - return "", fmt.Errorf("error getting node: %s", err.Error()) + return "", fmt.Errorf("unable to fetch the node details. Error: %v", err) } return string(node.UID), nil } @@ -2257,42 +2301,21 @@ func (f *feature) controllerPublishVolumeForNfsWithoutSDC(id string) error { if f.createVolumeRequest == nil { return nil } - req := f.getControllerPublishVolumeRequest() - req.VolumeId = id - req.NodeId, _ = f.GetNodeUID() clientSet := fake.NewSimpleClientset() service.K8sClientset = clientSet - var configYAMLContent strings.Builder - - for _, iface := range strings.Split(os.Getenv("NODE_INTERFACES"), ",") { - - interfaceData := strings.Split(strings.TrimSpace(iface), ":") - interfaceIP, err := f.getIPAddressByInterface(interfaceData[1]) - if err != nil { - fmt.Printf("Error while getting IP address for interface %s: %v\n", interfaceData[1], err) - continue - } - configYAMLContent.WriteString(fmt.Sprintf(" %s: %s\n", interfaceData[0], interfaceIP)) - } - - configMapData := map[string]string{ - "driver-config-params.yaml": fmt.Sprintf(`interfaceNames: -%s`, configYAMLContent.String()), + _, err := f.setFakeNode() + if err != nil { + return fmt.Errorf("setFakeNode failed with error: %v", err) } - configMap := &apiv1.ConfigMap{ - ObjectMeta: v1.ObjectMeta{ - Name: DriverConfigMap, - Namespace: DriverNamespace, - }, - Data: configMapData, - } + req := f.getControllerPublishVolumeRequest() + req.VolumeId = id + req.NodeId, _ = f.GetNodeUID() - _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, v1.CreateOptions{}) + err = f.createConfigMap() if err != nil { - fmt.Printf("Failed to create configMap: %v\n", err) - return err + return fmt.Errorf("createConfigMap failed with error: %v", err) } if f.arrays == nil { @@ -2503,12 +2526,25 @@ func (f *feature) controllerUnpublishVolumeForNfsWithoutSDC(id string) error { if f.createVolumeRequest == nil { return nil } + + clientSet := fake.NewSimpleClientset() + service.K8sClientset = clientSet + _, err := f.setFakeNode() + if err != nil { + return fmt.Errorf("setFakeNode failed with error: %v", err) + } + req := new(csi.ControllerUnpublishVolumeRequest) req.VolumeId = id req.NodeId, _ = f.GetNodeUID() + err = f.createConfigMap() + if err != nil { + return fmt.Errorf("createConfigMap failed with error: %v", err) + } + ctx := context.Background() client := csi.NewControllerClient(grpcClient) - _, err := client.ControllerUnpublishVolume(ctx, req) + _, err = client.ControllerUnpublishVolume(ctx, req) return err } From 184c0a7ed88696b81a07fb0124f4714b0ce36a84 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Thu, 10 Oct 2024 15:50:41 +0530 Subject: [PATCH 5/6] Changes to adding MDM in env.sh --- env.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/env.sh b/env.sh index dba10ff7..97e93c21 100644 --- a/env.sh +++ b/env.sh @@ -35,11 +35,14 @@ export GOSCALEIO_SHOWHTTP="true" #leave this variable blank. export ALT_SYSTEM_ID="" -MDM=`grep mdm ../../config.json | awk -F":" '{print $2}'` -for i in $MDM -do -IP=$i -IP=$(echo "$i" | sed "s/\"//g") -echo $IP - /opt/emc/scaleio/sdc/bin/drv_cfg --add_mdm --ip $IP -done +if /sbin/lsmod | grep -q scini; then + echo "scini module is present, Proceeding to add MDM..." + MDM=`grep mdm ../../config.json | awk -F":" '{print $2}'` + for i in $MDM + do + IP=$i + IP=$(echo "$i" | sed "s/\"//g") + echo "Adding MDM wth IP: $IP" + /opt/emc/scaleio/sdc/bin/drv_cfg --add_mdm --ip $IP + done +fi From 2c66609d51ca17c138f00c84b411787385ba2456 Mon Sep 17 00:00:00 2001 From: Harshita Pandey Date: Thu, 10 Oct 2024 21:14:22 +0530 Subject: [PATCH 6/6] Making Storage Pool configurable --- env.sh | 1 + test/integration/step_defs_test.go | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/env.sh b/env.sh index 97e93c21..8b102d36 100644 --- a/env.sh +++ b/env.sh @@ -20,6 +20,7 @@ export X_CSI_QUOTA_ENABLED="true" # Variables for using tests export CSI_ENDPOINT=`pwd`/unix_sock export STORAGE_POOL="" +export NFS_STORAGE_POOL="" export SDC_GUID=$(/bin/emc/scaleio/drv_cfg --query_guid) # Alternate GUID is for another system for testing expose volume to multiple hosts export ALT_GUID= diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index 70a732f5..180b456c 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -56,8 +56,6 @@ const ( MaxRetries = 10 RetrySleepTime = 10 * time.Second SleepTime = 100 * time.Millisecond - Pool1 = "pool1" - NfsPool = "Env8-SP-SW_SSD-1" NodeName = "node1" DriverConfigMap = "vxflexos-config-params" DriverNamespace = "vxflexos" @@ -335,8 +333,9 @@ func (f *feature) aVxFlexOSService() error { func (f *feature) aBasicBlockVolumeRequest(name string, size int64) error { req := new(csi.CreateVolumeRequest) + storagePool := os.Getenv("STORAGE_POOL") params := make(map[string]string) - params["storagepool"] = Pool1 + params["storagepool"] = storagePool params["thickprovisioning"] = "false" if len(f.anotherSystemID) > 0 { params["systemID"] = f.anotherSystemID @@ -368,6 +367,7 @@ func (f *feature) aBasicNfsVolumeRequest(name string, size int64) error { params := make(map[string]string) ctx := context.Background() + nfsPool := os.Getenv("NFS_STORAGE_POOL") fmt.Println("f.arrays,len", f.arrays, f.arrays) @@ -388,7 +388,7 @@ func (f *feature) aBasicNfsVolumeRequest(name string, size int64) error { } if val { - params["storagepool"] = NfsPool + params["storagepool"] = nfsPool params["thickprovisioning"] = "false" if os.Getenv("X_CSI_QUOTA_ENABLED") == "true" { params["isQuotaEnabled"] = "true" @@ -432,6 +432,7 @@ func (f *feature) aBasicNfsVolumeRequestWithSizeLessThan3Gi(name string, size in params := make(map[string]string) ctx := context.Background() + nfsPool := os.Getenv("NFS_STORAGE_POOL") fmt.Println("f.arrays,len", f.arrays, f.arrays) @@ -452,7 +453,7 @@ func (f *feature) aBasicNfsVolumeRequestWithSizeLessThan3Gi(name string, size in } if val { - params["storagepool"] = NfsPool + params["storagepool"] = nfsPool params["thickprovisioning"] = "false" if os.Getenv("X_CSI_QUOTA_ENABLED") == "true" { params["isQuotaEnabled"] = "true" @@ -496,6 +497,7 @@ func (f *feature) aNfsVolumeRequestWithQuota(volname string, volsize int64, path params := make(map[string]string) ctx := context.Background() + nfsPool := os.Getenv("NFS_STORAGE_POOL") fmt.Println("f.arrays,len", f.arrays, f.arrays) @@ -519,7 +521,7 @@ func (f *feature) aNfsVolumeRequestWithQuota(volname string, volsize int64, path if a.NasName != "" { params["nasName"] = a.NasName } - params["storagepool"] = NfsPool + params["storagepool"] = nfsPool params["thickprovisioning"] = "false" params["isQuotaEnabled"] = "true" params["softLimit"] = softlimit @@ -675,7 +677,8 @@ func (f *feature) aMountVolumeRequest(name string) error { func (f *feature) getMountVolumeRequest(name string) *csi.CreateVolumeRequest { req := new(csi.CreateVolumeRequest) params := make(map[string]string) - params["storagepool"] = Pool1 + storagePool := os.Getenv("STORAGE_POOL") + params["storagepool"] = storagePool if len(f.anotherSystemID) > 0 { params["systemID"] = f.anotherSystemID } @@ -804,7 +807,8 @@ func (f *feature) aCapabilityWithVoltypeAccessFstype(voltype, access, fstype str func (f *feature) aVolumeRequest(name string, size int64) error { req := new(csi.CreateVolumeRequest) params := make(map[string]string) - params["storagepool"] = Pool1 + storagePool := os.Getenv("STORAGE_POOL") + params["storagepool"] = storagePool params["thickprovisioning"] = "true" if len(f.anotherSystemID) > 0 { params["systemID"] = f.anotherSystemID @@ -2089,6 +2093,7 @@ func (f *feature) aBasicNfsVolumeRequestWithWrongNasName(name string, size int64 params := make(map[string]string) ctx := context.Background() + nfsPool := os.Getenv("NFS_STORAGE_POOL") fmt.Println("f.arrays,len", f.arrays, f.arrays) @@ -2115,7 +2120,7 @@ func (f *feature) aBasicNfsVolumeRequestWithWrongNasName(name string, size int64 params["nasName"] = wrongNasName } - params["storagepool"] = NfsPool + params["storagepool"] = nfsPool params["thickprovisioning"] = "false" if len(f.anotherSystemID) > 0 { params["systemID"] = f.anotherSystemID @@ -2215,6 +2220,7 @@ func (f *feature) aNfsCapabilityWithVoltypeAccessFstype(voltype, access, fstype func (f *feature) aNfsVolumeRequest(name string, size int64) error { ctx := context.Background() + nfsPool := os.Getenv("NFS_STORAGE_POOL") fmt.Println("f.arrays,len", f.arrays, f.arrays) @@ -2240,7 +2246,7 @@ func (f *feature) aNfsVolumeRequest(name string, size int64) error { if a.NasName != "" { params["nasName"] = a.NasName } - params["storagepool"] = NfsPool + params["storagepool"] = nfsPool params["thickprovisioning"] = "false" if os.Getenv("X_CSI_QUOTA_ENABLED") == "true" { params["isQuotaEnabled"] = "true"