From e6342428cff62a85a1421de69a97b353db3679ae Mon Sep 17 00:00:00 2001 From: evgenLevin <76552389+evgenLevin@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:55:18 -0400 Subject: [PATCH] cnf-network: backport parallel draining test to 4.15 (#137) --- go.mod | 2 +- go.sum | 4 +- .../core/network/internal/netenv/netenv.go | 10 + .../sriov/internal/sriovenv/sriovenv.go | 149 +++++++++ .../network/sriov/internal/tsparams/consts.go | 2 + .../network/sriov/tests/paralleldraining.go | 303 +++++++++++++++++ .../openshift-kni/eco-goinfra/pkg/pod/pod.go | 20 ++ .../eco-goinfra/pkg/sriov/poolconfig.go | 311 ++++++++++++++++++ .../eco-goinfra/pkg/sriov/poolconfiglist.go | 78 +++++ vendor/modules.txt | 2 +- 10 files changed, 877 insertions(+), 4 deletions(-) create mode 100644 tests/cnf/core/network/sriov/tests/paralleldraining.go create mode 100644 vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfig.go create mode 100644 vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfiglist.go diff --git a/go.mod b/go.mod index 7fe19e722..36dd72e16 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/nmstate/kubernetes-nmstate/api v0.0.0-20231116153922-80c6e01df02e github.com/onsi/ginkgo/v2 v2.15.0 github.com/onsi/gomega v1.31.1 - github.com/openshift-kni/eco-goinfra v0.0.0-20240806183830-e3d1d55fa491 // release-4.15 + github.com/openshift-kni/eco-goinfra v0.0.0-20240806191402-9800847af598 // release-4.15 github.com/openshift-kni/k8sreporter v1.0.5 github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible github.com/openshift/assisted-service/api v0.0.0 diff --git a/go.sum b/go.sum index eef857eae..4639d052d 100644 --- a/go.sum +++ b/go.sum @@ -510,8 +510,8 @@ github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/openshift-kni/cluster-group-upgrades-operator v0.0.0-20240227195723-b6e045729e4d h1:JsYQI7mtdh+x+Z/J3zfkLcK4eGtcWivikdxyYMWrjos= github.com/openshift-kni/cluster-group-upgrades-operator v0.0.0-20240227195723-b6e045729e4d/go.mod h1:zs70lcNtrMh1On3+e48IDg/bvoJER/IygEgo0bSnUuo= -github.com/openshift-kni/eco-goinfra v0.0.0-20240806183830-e3d1d55fa491 h1:YuoKfNHfh0MzFLadozclVaYyVCJw5OEfimJk7otpALk= -github.com/openshift-kni/eco-goinfra v0.0.0-20240806183830-e3d1d55fa491/go.mod h1:msival6kW06SsMcB5eWX5FcCFYG0ZQ36GNPz9pdKf3s= +github.com/openshift-kni/eco-goinfra v0.0.0-20240806191402-9800847af598 h1:KOi+i2hw9Nb8R/PzM5J5eMLrNHHW8MY5SkAlg4SnPWI= +github.com/openshift-kni/eco-goinfra v0.0.0-20240806191402-9800847af598/go.mod h1:msival6kW06SsMcB5eWX5FcCFYG0ZQ36GNPz9pdKf3s= github.com/openshift-kni/k8sreporter v1.0.5 h1:1GYBc/BTZyVoXilHef43v9A8BSzw700zAPZ6zsZvo6Y= github.com/openshift-kni/k8sreporter v1.0.5/go.mod h1:fg8HI9yxiKAi6UzR6NTtrmQmA2WKzUqmkRUHwQ1+Bj8= github.com/openshift-kni/lifecycle-agent v0.0.0-20240309022641-e2b836e0f2d3 h1:8xBtqDIor4vR8uczH6KHNJCJv359WW/+pw6xH7emHFc= diff --git a/tests/cnf/core/network/internal/netenv/netenv.go b/tests/cnf/core/network/internal/netenv/netenv.go index 002ef5952..0aceec296 100644 --- a/tests/cnf/core/network/internal/netenv/netenv.go +++ b/tests/cnf/core/network/internal/netenv/netenv.go @@ -77,3 +77,13 @@ func IsSriovDeployed(apiClient *clients.Settings, netConfig *netconfig.NetworkCo return nil } + +// MapFirstKeyValue returns the first key-value pair found in the input map. +// If the input map is empty, it returns empty strings. +func MapFirstKeyValue(inputMap map[string]string) (string, string) { + for key, value := range inputMap { + return key, value + } + + return "", "" +} diff --git a/tests/cnf/core/network/sriov/internal/sriovenv/sriovenv.go b/tests/cnf/core/network/sriov/internal/sriovenv/sriovenv.go index 2069be363..1a577628b 100644 --- a/tests/cnf/core/network/sriov/internal/sriovenv/sriovenv.go +++ b/tests/cnf/core/network/sriov/internal/sriovenv/sriovenv.go @@ -13,8 +13,10 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/sriov" sriovV1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/sriov/internal/tsparams" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -197,6 +199,153 @@ func ConfigureSriovMlnxFirmwareOnWorkers( return nil } +// CreatePodsAndRunTraffic creates test pods and verifies connectivity between them. +func CreatePodsAndRunTraffic( + clientNodeName string, + serverNodeName string, + sriovResNameClient string, + sriovResNameServer string, + clientMac string, + serverMac string, + clientIPs []string, + serverIPs []string) error { + glog.V(90).Infof("Creating test pods and checking ICMP connectivity between them") + + clientPod, _, err := createAndWaitTestPods( + clientNodeName, + serverNodeName, + sriovResNameClient, + sriovResNameServer, + clientMac, + serverMac, + clientIPs, + serverIPs) + + if err != nil { + glog.V(90).Infof("Failed to create test pods") + + return err + } + + return cmd.ICMPConnectivityCheck(clientPod, serverIPs) +} + +// RemoveSriovConfigurationAndWaitForSriovAndMCPStable removes all SR-IOV networks +// and policies in SR-IOV operator namespace. +func RemoveSriovConfigurationAndWaitForSriovAndMCPStable() error { + glog.V(90).Infof("Removing all SR-IOV networks and policies") + + err := RemoveAllSriovNetworks() + if err != nil { + glog.V(90).Infof("Failed to remove all SR-IOV networks") + + return err + } + + err = removeAllPoliciesAndWaitForSriovAndMCPStable() + if err != nil { + glog.V(90).Infof("Failed to remove all SR-IOV policies") + + return err + } + + return nil +} + +// RemoveAllSriovNetworks removes all SR-IOV networks. +func RemoveAllSriovNetworks() error { + glog.V(90).Infof("Removing all SR-IOV networks") + + sriovNs, err := namespace.Pull(APIClient, NetConfig.SriovOperatorNamespace) + if err != nil { + glog.V(90).Infof("Failed to pull SR-IOV operator namespace") + + return err + } + + err = sriovNs.CleanObjects( + netparam.DefaultTimeout, + sriov.GetSriovNetworksGVR()) + if err != nil { + glog.V(90).Infof("Failed to remove SR-IOV networks from SR-IOV operator namespace") + + return err + } + + return nil +} + +// removeAllPoliciesAndWaitForSriovAndMCPStable removes all SriovNetworkNodePolicies and waits until +// SR-IOV and MCP become stable. +func removeAllPoliciesAndWaitForSriovAndMCPStable() error { + glog.V(90).Infof("Deleting all SriovNetworkNodePolicies and waiting for SR-IOV and MCP become stable.") + + err := sriov.CleanAllNetworkNodePolicies(APIClient, NetConfig.SriovOperatorNamespace) + if err != nil { + return err + } + + return netenv.WaitForSriovAndMCPStable( + APIClient, tsparams.MCOWaitTimeout, time.Minute, NetConfig.CnfMcpLabel, NetConfig.SriovOperatorNamespace) +} + +// createAndWaitTestPods creates test pods and waits until they are in the ready state. +func createAndWaitTestPods( + clientNodeName string, + serverNodeName string, + sriovResNameClient string, + sriovResNameServer string, + clientMac string, + serverMac string, + clientIPs []string, + serverIPs []string) (client *pod.Builder, server *pod.Builder, err error) { + glog.V(90).Infof("Creating client pod with IPs %v, mac %s, SR-IOV resourceName %s"+ + " and server pod with IPs %v, mac %s, SR-IOV resourceName %s.", + clientIPs, clientMac, sriovResNameClient, serverIPs, serverMac, sriovResNameServer) + + clientPod, err := createAndWaitTestPodWithSecondaryNetwork("client", clientNodeName, + sriovResNameClient, clientMac, clientIPs) + if err != nil { + glog.V(90).Infof("Failed to create clientPod") + + return nil, nil, err + } + + serverPod, err := createAndWaitTestPodWithSecondaryNetwork("server", serverNodeName, + sriovResNameServer, serverMac, serverIPs) + if err != nil { + glog.V(90).Infof("Failed to create serverPod") + + return nil, nil, err + } + + return clientPod, serverPod, nil +} + +// createAndWaitTestPodWithSecondaryNetwork creates test pod with secondary network +// and waits until it is in the ready state. +func createAndWaitTestPodWithSecondaryNetwork( + podName string, + testNodeName string, + sriovResNameTest string, + testMac string, + testIPs []string) (*pod.Builder, error) { + glog.V(90).Infof("Creating a test pod name %s", podName) + + secNetwork := pod.StaticIPAnnotationWithMacAddress(sriovResNameTest, testIPs, testMac) + testPod, err := pod.NewBuilder(APIClient, podName, tsparams.TestNamespaceName, NetConfig.CnfNetTestContainer). + DefineOnNode(testNodeName).WithPrivilegedFlag(). + WithSecondaryNetwork(secNetwork).CreateAndWaitUntilRunning(netparam.DefaultTimeout) + + if err != nil { + glog.V(90).Infof("Failed to create pod %s with secondary network", podName) + + return nil, err + } + + return testPod, nil +} + func isVfCreated(sriovNodeState *sriov.NetworkNodeStateBuilder, vfNumber int, sriovInterfaceName string) error { sriovNumVfs, err := sriovNodeState.GetNumVFs(sriovInterfaceName) if err != nil { diff --git a/tests/cnf/core/network/sriov/internal/tsparams/consts.go b/tests/cnf/core/network/sriov/internal/tsparams/consts.go index c2a8bd5f4..1b40bbad9 100644 --- a/tests/cnf/core/network/sriov/internal/tsparams/consts.go +++ b/tests/cnf/core/network/sriov/internal/tsparams/consts.go @@ -7,4 +7,6 @@ const ( TestNamespaceName = "sriov-tests" // LabelExternallyManagedTestCases represents ExternallyManaged label that can be used for test cases selection. LabelExternallyManagedTestCases = "externallymanaged" + // LabelParallelDrainingTestCases represents parallel draining label that can be used for test cases selection. + LabelParallelDrainingTestCases = "paralleldraining" ) diff --git a/tests/cnf/core/network/sriov/tests/paralleldraining.go b/tests/cnf/core/network/sriov/tests/paralleldraining.go new file mode 100644 index 000000000..f82b0fc4c --- /dev/null +++ b/tests/cnf/core/network/sriov/tests/paralleldraining.go @@ -0,0 +1,303 @@ +package tests + +import ( + "fmt" + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift-kni/eco-goinfra/pkg/namespace" + "github.com/openshift-kni/eco-goinfra/pkg/nodes" + "github.com/openshift-kni/eco-goinfra/pkg/pod" + "github.com/openshift-kni/eco-goinfra/pkg/reportxml" + "github.com/openshift-kni/eco-goinfra/pkg/sriov" + + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" + . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/sriov/internal/sriovenv" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/sriov/internal/tsparams" +) + +const sriovAndResourceNameParallelDrain = "paralleldraining" + +var _ = Describe("ParallelDraining", Ordered, Label(tsparams.LabelParallelDrainingTestCases), + ContinueOnFailure, func() { + var ( + sriovInterfacesUnderTest []string + workerNodeList []*nodes.Builder + err error + poolConfigName = "pool1" + poolConfig2Name = "pool2" + testKey = "test" + testLabel1 = map[string]string{testKey: "label1"} + testLabel2 = map[string]string{testKey: "label2"} + ) + + BeforeAll(func() { + By("Validating SR-IOV interfaces") + workerNodeList, err = nodes.List(APIClient, + metav1.ListOptions{LabelSelector: labels.Set(NetConfig.WorkerLabelMap).String()}) + Expect(err).ToNot(HaveOccurred(), "Failed to discover worker nodes") + Expect(sriovenv.ValidateSriovInterfaces(workerNodeList, 1)).ToNot(HaveOccurred(), + "Failed to get required SR-IOV interfaces") + sriovInterfacesUnderTest, err = NetConfig.GetSriovInterfaces(1) + Expect(err).ToNot(HaveOccurred(), "Failed to retrieve SR-IOV interfaces for testing") + + By("Verifying if parallel draining tests can be executed on given cluster") + err = netenv.DoesClusterHasEnoughNodes(APIClient, NetConfig, 1, 2) + Expect(err).ToNot(HaveOccurred(), + "Cluster doesn't support parallel draining test cases - doesn't have enough nodes") + }) + BeforeEach(func() { + By("Configuring SR-IOV") + createSriovConfigurationParallelDrain(sriovInterfacesUnderTest[0]) + + By("Creating test pods and checking connectivity between the them") + err := sriovenv.CreatePodsAndRunTraffic(workerNodeList[0].Object.Name, workerNodeList[0].Object.Name, + sriovAndResourceNameParallelDrain, sriovAndResourceNameParallelDrain, + tsparams.ClientMacAddress, tsparams.ServerMacAddress, + []string{tsparams.ClientIPv4IPAddress}, []string{tsparams.ServerIPv4IPAddress}) + Expect(err).ToNot(HaveOccurred(), "Connectivity check between test pods failed") + + By("Adding pods with terminationGracePeriodSeconds on each worker node") + createPodWithVFOnEachWorker(workerNodeList) + }) + + AfterEach(func() { + removeLabelFromWorkersIfExists(workerNodeList, testLabel1) + + By("Removing SR-IOV configuration") + err := sriovenv.RemoveSriovConfigurationAndWaitForSriovAndMCPStable() + Expect(err).ToNot(HaveOccurred(), "Failed to remove SR-IOV configration") + + err = sriov.CleanAllPoolConfigs(APIClient, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to remove SriovNetworkPoolConfigs") + + By("Cleaning test namespace") + err = namespace.NewBuilder(APIClient, tsparams.TestNamespaceName).CleanObjects( + netparam.DefaultTimeout, pod.GetGVR()) + Expect(err).ToNot(HaveOccurred(), "Failed to clean test namespace") + }) + + It("without SriovNetworkPoolConfig", reportxml.ID("68640"), func() { + By("Removing test configuration to call draining mechanism") + removeTestConfiguration() + + By("Validating that nodes are drained one by one") + Eventually(isDrainingRunningAsExpected, time.Minute, tsparams.RetryInterval).WithArguments(1). + Should(BeTrue(), "draining runs not as expected") + + err = netenv.WaitForSriovStable(APIClient, tsparams.MCOWaitTimeout, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for stable cluster.") + }) + + It("without maxUnavailable field", reportxml.ID("68661"), func() { + By("Creating SriovNetworkPoolConfig without maxUnavailable field") + _, err = sriov.NewPoolConfigBuilder(APIClient, poolConfigName, NetConfig.SriovOperatorNamespace). + WithNodeSelector(NetConfig.WorkerLabelMap).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SriovNetworkPoolConfig without maxUnavailable field.") + + By("Removing test configuration to call draining mechanism") + removeTestConfiguration() + + By("Validating that nodes are drained all together") + Eventually(isDrainingRunningAsExpected, time.Minute, tsparams.RetryInterval).WithArguments(len(workerNodeList)). + Should(BeTrue(), "draining runs not as expected") + + err = netenv.WaitForSriovStable(APIClient, tsparams.MCOWaitTimeout, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for stable cluster.") + }) + + It("1 SriovNetworkPoolConfig: maxUnavailable value is 2", reportxml.ID("68662"), func() { + By("Validating that the cluster has more 2 worker nodes") + if len(workerNodeList) < 3 { + Skip(fmt.Sprintf("The cluster has less than 3 workers: %d", len(workerNodeList))) + } + + By("Creating SriovNetworkPoolConfig with maxUnavailable 2") + _, err = sriov.NewPoolConfigBuilder(APIClient, poolConfigName, NetConfig.SriovOperatorNamespace). + WithMaxUnavailable(intstr.FromInt32(2)). + WithNodeSelector(NetConfig.WorkerLabelMap).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SriovNetworkPoolConfig with maxUnavailable 2.") + + By("Removing test configuration to call draining mechanism") + removeTestConfiguration() + + By("Validating that nodes are drained by 2") + Eventually(isDrainingRunningAsExpected, time.Minute, tsparams.RetryInterval).WithArguments(2). + Should(BeTrue(), "draining runs not as expected") + + err = netenv.WaitForSriovStable(APIClient, tsparams.MCOWaitTimeout, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for stable cluster.") + }) + + It("2 SriovNetworkPoolConfigs", reportxml.ID("68663"), func() { + By("Validating that the cluster has more 2 worker nodes") + if len(workerNodeList) < 3 { + Skip(fmt.Sprintf("The cluster has less than 3 workers: %d", len(workerNodeList))) + } + + By("Labeling workers under test with the specified test label") + _, err = workerNodeList[0].WithNewLabel(netenv.MapFirstKeyValue(testLabel1)).Update() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to label worker %s with the test label %v", + workerNodeList[0].Object.Name, testLabel1)) + _, err = workerNodeList[1].WithNewLabel(netenv.MapFirstKeyValue(testLabel1)).Update() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to label worker %s with the test label %v", + workerNodeList[1].Object.Name, testLabel1)) + _, err = workerNodeList[2].WithNewLabel(netenv.MapFirstKeyValue(testLabel2)).Update() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to label worker %s with the test label %v", + workerNodeList[2].Object.Name, testLabel2)) + + By("Creating SriovNetworkPoolConfig with maxUnavailable 2") + _, err = sriov.NewPoolConfigBuilder(APIClient, poolConfigName, NetConfig.SriovOperatorNamespace). + WithMaxUnavailable(intstr.FromInt32(2)). + WithNodeSelector(testLabel1).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SriovNetworkPoolConfig with maxUnavailable 2") + + By("Creating SriovNetworkPoolConfig with maxUnavailable 0") + poolConfig2, err := sriov.NewPoolConfigBuilder(APIClient, poolConfig2Name, NetConfig.SriovOperatorNamespace). + WithMaxUnavailable(intstr.FromInt32(0)). + WithNodeSelector(testLabel2).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SriovNetworkPoolConfig with maxUnavailable 0") + + By("Removing test configuration to call draining mechanism") + removeTestConfiguration() + + By("Verifying that two workers are draining, and the third worker remains in an idle state permanently") + Eventually(isDrainingRunningAsExpected, time.Minute, tsparams.RetryInterval).WithArguments(2). + Should(BeTrue(), "draining runs not as expected") + + sriovNodeStateList, err := sriov.ListNetworkNodeState(APIClient, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to collect all SriovNetworkNodeStates") + + Consistently(func() bool { + err = sriovNodeStateList[2].Discover() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to discover the worker %s", + sriovNodeStateList[2].Objects.Name)) + + return sriovNodeStateList[2].Objects.Annotations["sriovnetwork.openshift.io/current-state"] == "Idle" && + sriovNodeStateList[2].Objects.Status.SyncStatus == "InProgress" + }, 2*time.Minute, tsparams.RetryInterval).Should(BeTrue(), + fmt.Sprintf("The third worker is not in an idle and InProgress states forever. His state is: %s,%s", + sriovNodeStateList[2].Objects.Status.SyncStatus, + sriovNodeStateList[2].Objects.Annotations["sriovnetwork.openshift.io/current-state"])) + + By("Removing the test labels from the workers") + removeLabelFromWorkersIfExists(workerNodeList, testLabel1) + + By("Removing SriovNetworkPoolConfig with maxUnavailable set to 0 and waiting for all workers to drain") + err = poolConfig2.Delete() + Expect(err).ToNot(HaveOccurred(), "Failed to remove SriovNetworkPoolConfig") + + err = netenv.WaitForSriovStable(APIClient, tsparams.MCOWaitTimeout, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for stable cluster.") + }) + + It("Draining does not remove non SR-IOV pod", reportxml.ID("68664"), func() { + By("Creating non SR-IOV pod on the first worker") + nonSriovPod, err := pod.NewBuilder( + APIClient, "nonsriov", tsparams.TestNamespaceName, NetConfig.CnfNetTestContainer). + DefineOnNode(workerNodeList[0].Object.Name). + CreateAndWaitUntilRunning(netparam.DefaultTimeout) + Expect(err).ToNot(HaveOccurred(), "Failed to create the non SR-IOV test pod") + + By("Creating SriovNetworkPoolConfig with 100% maxUnavailable field") + _, err = sriov.NewPoolConfigBuilder(APIClient, poolConfigName, NetConfig.SriovOperatorNamespace). + WithMaxUnavailable(intstr.FromString("100%")). + WithNodeSelector(NetConfig.WorkerLabelMap).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SriovNetworkPoolConfig with 100% maxUnavailable field") + + By("Removing test configuration to call draining mechanism") + removeTestConfiguration() + + By("Validating that all workers are drained simultaneously") + Eventually(isDrainingRunningAsExpected, time.Minute, tsparams.RetryInterval).WithArguments(len(workerNodeList)). + Should(BeTrue(), "draining runs not as expected") + + err = netenv.WaitForSriovStable(APIClient, tsparams.MCOWaitTimeout, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for stable cluster.") + + By("Checking that non SR-IOV pod is still on the first worker") + if !nonSriovPod.Exists() { + Fail("Non SR-IOV pod has been removed after the draining process") + } + }) + }) + +func createSriovConfigurationParallelDrain(sriovInterfaceName string) { + By("Creating SR-IOV policy") + + sriovPolicy := sriov.NewPolicyBuilder( + APIClient, + sriovAndResourceNameParallelDrain, + NetConfig.SriovOperatorNamespace, + sriovAndResourceNameParallelDrain, + 5, + []string{sriovInterfaceName}, NetConfig.WorkerLabelMap) + + err := sriovenv.CreateSriovPolicyAndWaitUntilItsApplied(sriovPolicy, tsparams.MCOWaitTimeout) + Expect(err).ToNot(HaveOccurred(), "Failed to configure SR-IOV policy") + + By("Creating SR-IOV network") + + _, err = sriov.NewNetworkBuilder(APIClient, sriovAndResourceNameParallelDrain, NetConfig.SriovOperatorNamespace, + tsparams.TestNamespaceName, sriovAndResourceNameParallelDrain).WithStaticIpam().WithMacAddressSupport(). + WithIPAddressSupport().Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create SR-IOV network") +} + +func removeTestConfiguration() { + err := sriovenv.RemoveAllSriovNetworks() + Expect(err).ToNot(HaveOccurred(), "Failed to clean all SR-IOV Networks") + err = sriov.CleanAllNetworkNodePolicies(APIClient, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to clean all SR-IOV policies") +} + +func isDrainingRunningAsExpected(expectedConcurrentDrains int) bool { + sriovNodeStateList, err := sriov.ListNetworkNodeState(APIClient, NetConfig.SriovOperatorNamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to collect all SriovNetworkNodeStates") + Expect(len(sriovNodeStateList)).ToNot(Equal(0), "SriovNetworkNodeStates list is empty") + + var inProgressWithDrainingCompleteCount int + + for _, sriovNodeState := range sriovNodeStateList { + // Check if syncStatus is "InProgress" and CurrentSyncState is "DrainComplete" + if sriovNodeState.Objects.Status.SyncStatus == "InProgress" && + sriovNodeState.Objects.Annotations["sriovnetwork.openshift.io/current-state"] == "Draining" { + inProgressWithDrainingCompleteCount++ + } + } + + return inProgressWithDrainingCompleteCount == expectedConcurrentDrains +} + +func createPodWithVFOnEachWorker(workerList []*nodes.Builder) { + for i, worker := range workerList { + // 192.168.0.1 and 192.168.0.2 addresses are busy by client and server pods + ipaddress := "192.168.0." + strconv.Itoa(i+3) + "/24" + secNetwork := pod.StaticIPAnnotation(sriovAndResourceNameParallelDrain, []string{ipaddress}) + _, err := pod.NewBuilder( + APIClient, "testpod"+worker.Object.Name, tsparams.TestNamespaceName, NetConfig.CnfNetTestContainer). + DefineOnNode(worker.Object.Name).WithSecondaryNetwork(secNetwork). + WithTerminationGracePeriodSeconds(5). + CreateAndWaitUntilRunning(netparam.DefaultTimeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create a pod %s", "testpod-"+worker.Object.Name)) + } +} + +func removeLabelFromWorkersIfExists(workerList []*nodes.Builder, label map[string]string) { + key, value := netenv.MapFirstKeyValue(label) + for _, worker := range workerList { + if _, ok := worker.Object.Labels[key]; ok { + By(fmt.Sprintf("Removing label with key %s from worker %s", key, worker.Object.Name)) + _, err := worker.RemoveLabel(key, value).Update() + Expect(err).ToNot(HaveOccurred(), "Failed to delete test label") + } + } +} diff --git a/vendor/github.com/openshift-kni/eco-goinfra/pkg/pod/pod.go b/vendor/github.com/openshift-kni/eco-goinfra/pkg/pod/pod.go index e2eb33eb0..a3227d12f 100644 --- a/vendor/github.com/openshift-kni/eco-goinfra/pkg/pod/pod.go +++ b/vendor/github.com/openshift-kni/eco-goinfra/pkg/pod/pod.go @@ -1070,6 +1070,26 @@ func (builder *Builder) WithOptions(options ...AdditionalOptions) *Builder { return builder } +// WithTerminationGracePeriodSeconds configures TerminationGracePeriodSeconds on the pod. +func (builder *Builder) WithTerminationGracePeriodSeconds(terminationGracePeriodSeconds int64) *Builder { + if valid, _ := builder.validate(); !valid { + return builder + } + + glog.V(100).Infof("Applying terminationGracePeriodSeconds flag to the configuration of pod: %s in namespace: %s", + builder.Definition.Name, builder.Definition.Namespace) + + builder.isMutationAllowed("terminationGracePeriodSeconds") + + if builder.errorMsg != "" { + return builder + } + + builder.Definition.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + + return builder +} + // GetLog connects to a pod and fetches log. func (builder *Builder) GetLog(logStartTime time.Duration, containerName string) (string, error) { if valid, err := builder.validate(); !valid { diff --git a/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfig.go b/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfig.go new file mode 100644 index 000000000..a4d5b1b7c --- /dev/null +++ b/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfig.go @@ -0,0 +1,311 @@ +package sriov + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/golang/glog" + "github.com/openshift-kni/eco-goinfra/pkg/msg" + + srIovV1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/openshift-kni/eco-goinfra/pkg/clients" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + intstrutil "k8s.io/apimachinery/pkg/util/intstr" + goclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PoolConfigBuilder provides struct for SriovNetworkPoolConfig object containing connection to the cluster +// and the SriovNetworkPoolConfig definitions. +type PoolConfigBuilder struct { + // SriovNetworkPoolConfig definition. Used to create SriovNetworkPoolConfig object. + Definition *srIovV1.SriovNetworkPoolConfig + // Created sriovNetworkPoolConfig object. + Object *srIovV1.SriovNetworkPoolConfig + // Used in functions that define or mutate SriovNetworkPoolConfig definition. + // errorMsg is processed before the SriovNetworkPoolConfig object is created. + errorMsg string + // apiClient opens api connection to the cluster. + apiClient *clients.Settings +} + +// NewPoolConfigBuilder creates a new instance of PoolConfigBuilder. +func NewPoolConfigBuilder(apiClient *clients.Settings, name, nsname string) *PoolConfigBuilder { + glog.V(100).Infof( + "Initializing new SriovNetworkPoolConfig structure with the name %s in the namespace %s", name, nsname) + + builder := PoolConfigBuilder{ + apiClient: apiClient, + Definition: &srIovV1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: nsname, + }, + }} + + if name == "" { + builder.errorMsg = "SriovNetworkPoolConfig 'name' cannot be empty" + + return &builder + } + + if nsname == "" { + builder.errorMsg = "SriovNetworkPoolConfig 'nsname' cannot be empty" + + return &builder + } + + return &builder +} + +// Create generates an SriovNetworkPoolConfig in the cluster and stores the created object in struct. +func (builder *PoolConfigBuilder) Create() (*PoolConfigBuilder, error) { + if valid, err := builder.validate(); !valid { + return builder, err + } + + glog.V(100).Infof( + "Creating the SriovNetworkPoolConfig %s in namespace %s", builder.Definition.Name, + builder.Definition.Namespace) + + if !builder.Exists() { + err := builder.apiClient.Create(context.TODO(), builder.Definition) + + if err != nil { + return nil, err + } + } + + builder.Object = builder.Definition + + return builder, nil +} + +// Delete removes an SriovNetworkPoolConfig object. +func (builder *PoolConfigBuilder) Delete() error { + if valid, err := builder.validate(); !valid { + return err + } + + glog.V(100).Infof("Deleting the SriovNetworkPoolConfig object %s in namespace %s", + builder.Definition.Name, builder.Definition.Namespace) + + if !builder.Exists() { + return nil + } + + err := builder.apiClient.Delete(context.TODO(), builder.Object) + + if err != nil { + return err + } + + builder.Object = nil + + return nil +} + +// Exists checks whether the given SriovNetworkPoolConfig object exists in the cluster. +func (builder *PoolConfigBuilder) Exists() bool { + if valid, _ := builder.validate(); !valid { + return false + } + + glog.V(100).Infof("Checking if SriovNetworkPoolConfig %s exists in namespace %s", + builder.Definition.Name, builder.Definition.Namespace) + + var err error + builder.Object, err = builder.Get() + + return err == nil || !k8serrors.IsNotFound(err) +} + +// Get returns SriovNetworkPoolConfig object if found. +func (builder *PoolConfigBuilder) Get() (*srIovV1.SriovNetworkPoolConfig, error) { + if valid, err := builder.validate(); !valid { + return nil, err + } + + glog.V(100).Infof( + "Collecting SriovNetworkPoolConfig object %s in namespace %s", + builder.Definition.Name, builder.Definition.Namespace) + + poolConfig := &srIovV1.SriovNetworkPoolConfig{} + err := builder.apiClient.Get(context.TODO(), goclient.ObjectKey{ + Name: builder.Definition.Name, + Namespace: builder.Definition.Namespace, + }, poolConfig) + + if err != nil { + glog.V(100).Infof("Failed to get SriovNetworkPoolConfig %s in namespace %s", builder.Definition.Name, + builder.Definition.Namespace) + + return nil, err + } + + return poolConfig, nil +} + +// Update renovates the existing SriovNetworkPoolConfig object with the new definition in builder. +func (builder *PoolConfigBuilder) Update() (*PoolConfigBuilder, error) { + if valid, err := builder.validate(); !valid { + return builder, err + } + + glog.V(100).Infof("Updating the SriovNetworkPoolConfig object %s in namespace %s", builder.Definition.Name, + builder.Definition.Namespace) + + err := builder.apiClient.Update(context.TODO(), builder.Definition) + + if err != nil { + glog.V(100).Infof("Failed to update SriovNetworkPoolConfig %s in namespace %s", builder.Definition.Name, + builder.Definition.Namespace) + + return nil, err + } + + builder.Object = builder.Definition + + return builder, nil +} + +// WithNodeSelector sets nodeSelector in the SriovNetworkPoolConfig definition. +func (builder *PoolConfigBuilder) WithNodeSelector(nodeSelector map[string]string) *PoolConfigBuilder { + if valid, _ := builder.validate(); !valid { + return builder + } + + glog.V(100).Infof( + "Creating SriovNetworkPoolConfig %s in namespace %s with Node selector: %v", builder.Definition.Name, + builder.Definition.Namespace, nodeSelector) + + if len(nodeSelector) == 0 { + builder.errorMsg = "SriovNetworkPoolConfig 'nodeSelector' cannot be empty map" + + return builder + } + + builder.Definition.Spec.NodeSelector = &metav1.LabelSelector{MatchLabels: nodeSelector} + + return builder +} + +// WithMaxUnavailable sets MaxUnavailable in the SriovNetworkPoolConfig definition. +func (builder *PoolConfigBuilder) WithMaxUnavailable(maxUnavailable intstrutil.IntOrString) *PoolConfigBuilder { + if valid, _ := builder.validate(); !valid { + return builder + } + + glog.V(100).Infof( + "Creating SriovNetworkPoolConfig %s in namespace %s with MaxUnavailable: %v", builder.Definition.Name, + builder.Definition.Namespace, maxUnavailable) + + if maxUnavailable.Type == intstrutil.String { + if strings.HasSuffix(maxUnavailable.StrVal, "%") { + i := strings.TrimSuffix(maxUnavailable.StrVal, "%") + + value, err := strconv.Atoi(i) + if err != nil { + builder.errorMsg = fmt.Sprintf("invalid value %q: %v", maxUnavailable.StrVal, err) + + return builder + } + + if value > 100 || value < 1 { + builder.errorMsg = fmt.Sprintf("invalid value: percentage needs to be between 1 and 100: %v", maxUnavailable) + + return builder + } + } else { + builder.errorMsg = fmt.Sprintf("invalid type: strings needs to be a percentage: %v", maxUnavailable) + + return builder + } + } else { + if maxUnavailable.IntValue() < 0 { + builder.errorMsg = fmt.Sprintf("negative number is not allowed: %v", maxUnavailable) + + return builder + } + } + + builder.Definition.Spec.MaxUnavailable = &maxUnavailable + + return builder +} + +// PullPoolConfig pulls existing SriovNetworkPoolConfig from cluster. +func PullPoolConfig(apiClient *clients.Settings, name, nsname string) (*PoolConfigBuilder, error) { + glog.V(100).Infof("Pulling existing SriovNetworkPoolConfig name %s under namespace %s from cluster", name, nsname) + + if apiClient == nil { + glog.V(100).Infof("The apiClient is empty") + + return nil, fmt.Errorf("SriovNetworkPoolConfig 'apiClient' cannot be empty") + } + + builder := PoolConfigBuilder{ + apiClient: apiClient, + Definition: &srIovV1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: nsname, + }, + }, + } + + if name == "" { + glog.V(100).Infof("The name of the SriovNetworkPoolConfig is empty") + + return nil, errors.New("SriovNetworkPoolConfig 'name' cannot be empty") + } + + if nsname == "" { + glog.V(100).Infof("The namespace of the SriovNetworkPoolConfig is empty") + + return nil, errors.New("SriovNetworkPoolConfig 'namespace' cannot be empty") + } + + if !builder.Exists() { + return nil, fmt.Errorf("SriovNetworkPoolConfig object %s does not exist in namespace %s", name, nsname) + } + + builder.Definition = builder.Object + + return &builder, nil +} + +// validate will check that the builder and builder definition are properly initialized before +// accessing any member fields. +func (builder *PoolConfigBuilder) validate() (bool, error) { + resourceCRD := "SriovNetworkPoolConfig" + + if builder == nil { + glog.V(100).Infof("The %s builder is uninitialized", resourceCRD) + + return false, fmt.Errorf("error: received nil %s builder", resourceCRD) + } + + if builder.Definition == nil { + glog.V(100).Infof("The %s is undefined", resourceCRD) + + builder.errorMsg = msg.UndefinedCrdObjectErrString(resourceCRD) + } + + if builder.apiClient == nil { + glog.V(100).Infof("The %s builder apiclient is nil", resourceCRD) + + builder.errorMsg = fmt.Sprintf("%s builder cannot have nil apiClient", resourceCRD) + } + + if builder.errorMsg != "" { + glog.V(100).Infof("The %s builder has error message: %s", resourceCRD, builder.errorMsg) + + return false, fmt.Errorf(builder.errorMsg) + } + + return true, nil +} diff --git a/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfiglist.go b/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfiglist.go new file mode 100644 index 000000000..a1cf0d679 --- /dev/null +++ b/vendor/github.com/openshift-kni/eco-goinfra/pkg/sriov/poolconfiglist.go @@ -0,0 +1,78 @@ +package sriov + +import ( + "context" + "fmt" + + "github.com/golang/glog" + srIovV1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/openshift-kni/eco-goinfra/pkg/clients" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ListPoolConfigs returns a sriovNetworkPoolConfig list in a given namespace. +func ListPoolConfigs(apiClient *clients.Settings, namespace string) ([]*PoolConfigBuilder, error) { + sriovNetworkPoolConfigList := &srIovV1.SriovNetworkPoolConfigList{} + + if namespace == "" { + glog.V(100).Infof("sriovNetworkPoolConfigs 'namespace' parameter can not be empty") + + return nil, fmt.Errorf("failed to list sriovNetworkPoolConfigs, 'namespace' parameter is empty") + } + + err := apiClient.List(context.TODO(), sriovNetworkPoolConfigList, &client.ListOptions{Namespace: namespace}) + + if err != nil { + glog.V(100).Infof("Failed to list SriovNetworkPoolConfigs in namespace: %s due to %s", + namespace, err.Error()) + + return nil, err + } + + var poolConfigBuilderObjects []*PoolConfigBuilder + + for _, sriovNetworkPoolConfigObj := range sriovNetworkPoolConfigList.Items { + sriovNetworkPoolConfig := sriovNetworkPoolConfigObj + sriovNetworkPoolConfBuilder := &PoolConfigBuilder{ + apiClient: apiClient, + Definition: &sriovNetworkPoolConfig, + Object: &sriovNetworkPoolConfig, + } + + poolConfigBuilderObjects = append(poolConfigBuilderObjects, sriovNetworkPoolConfBuilder) + } + + return poolConfigBuilderObjects, nil +} + +// CleanAllPoolConfigs removes all sriovNetworkPoolConfigs. +func CleanAllPoolConfigs( + apiClient *clients.Settings, operatornsname string) error { + glog.V(100).Infof("Cleaning up SriovNetworkPoolConfigs in the %s namespace", operatornsname) + + if operatornsname == "" { + glog.V(100).Infof("'operatornsname' parameter can not be empty") + + return fmt.Errorf("failed to clean up SriovNetworkPoolConfigs, 'operatornsname' parameter is empty") + } + + poolConfigs, err := ListPoolConfigs(apiClient, operatornsname) + + if err != nil { + glog.V(100).Infof("Failed to list SriovNetworkPoolConfigs in namespace: %s", operatornsname) + + return err + } + + for _, poolConfig := range poolConfigs { + err = poolConfig.Delete() + + if err != nil { + glog.V(100).Infof("Failed to delete SriovNetworkPoolConfigs: %s", poolConfig.Object.Name) + + return err + } + } + + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f6977084d..6d48769b2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -529,7 +529,7 @@ github.com/onsi/gomega/types ## explicit; go 1.20 github.com/openshift-kni/cluster-group-upgrades-operator/pkg/api/clustergroupupgrades github.com/openshift-kni/cluster-group-upgrades-operator/pkg/api/clustergroupupgrades/v1alpha1 -# github.com/openshift-kni/eco-goinfra v0.0.0-20240806183830-e3d1d55fa491 +# github.com/openshift-kni/eco-goinfra v0.0.0-20240806191402-9800847af598 ## explicit; go 1.20 github.com/openshift-kni/eco-goinfra/pkg/assisted github.com/openshift-kni/eco-goinfra/pkg/bmh