From d34c61f9402723b7e213f9f905010e3b11431dc5 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 31 Oct 2023 13:44:59 -0400 Subject: [PATCH] feat: CNS RequestIPAddress branching for MT/V2 (#2300) * fix overlay IPAM not reporting version * revert file and var naming, add correct path to makefile * proposal design for multitenant IPAM flow * change podipinfo + linter issue * pointer issues for printf * update IPAM branching * remove comments * pod client placeholder * address lint issue for httpservicefake * getting pod info in validator * linter issue * update network container contract * renaming * mtpnc changes * rebase * revert file and var naming, add correct path to makefile * add default route * add unit tests * update unit tests for ipam * go get to fix linter * go mod tidy * update routes * update routes * remove stale comments + remove redundant method * add contexts + change address type * addressed review * embedded client to mock + enum for address type * fix error * change addressType to NICType * change isDefaultRoute to SkipDefaultRoutes * address comments * refractor: make changes according to cni/cns contract * refractor: make adding route its own func + move swift v2 ipam branching to after normal ipam flow * refractor: change vars naming * refractor: more var naming * test: add test for podv6cidr * refractor: make the returning podIpInfo init cleaner in swiftv2.go * refractor + tests: add contexts to ipconfigs req validators + set route tests * refractor: change labels for swift v2 pods * fix: fix swift v2 UT * refractor: add v4/v6 distinction for service cidr * rebase * revert file and var naming, add correct path to makefile * rebase * revert file and var naming, add correct path to makefile * change podipinfo + linter issue * update IPAM branching * pod client placeholder * getting pod info in validator * linter issue * rebase * revert file and var naming, add correct path to makefile * refractor: fix conflicts * refractor: revert podwatcher code changes * docs: change comment * refractor: change CIDR to CDIRs * refractor: parse CIDRs as semicolons separated string from env in SetRoutes * docs: add minor comment * refractor: change separator for parsing CIDRs * feat: add rbac roles * fix: gofumpt * fix: update clusterrole * fix: add namespace to clusterrolebinding * fix: UT * fix: add labels toswift v2 clusterrole * fix: release default ipconfig early if getting swiftv2 ipconfig failed * test: add more UT * fix: parsing MTPNC as CIDR instead * fix: toggle skipDefaultRoutes for infraNic to true * fix: add route for node cidr in ipv4 podipconfig * feat: add node cidrs route * fix: linter * address comments * fix: minor logs formatting * feat: move cns yaml for swiftv2 scenario to a diff file + more logging for swiftv2middleware * fix: log debugf to printf * fix: add testmain to avoid nil pointer error for loggers * Update azure-cns.yaml Signed-off-by: Quang Nguyen * fix: move parseCIDRs to a common package, use net/netip instead of net * fix: exhaustive all switch case for nic type * fix: exhaustive all switch case for nic type * refractor: change fmt.Errorf to errors.Wrapf * fix: add mtpnc status check in validator + use netip package * fix: minor * revert: azure-cns.yaml --------- Signed-off-by: Quang Nguyen Signed-off-by: Quang Nguyen --- cni/plugin.go | 6 +- cns/NetworkContainerContract.go | 25 ++- cns/api.go | 10 ++ cns/azure-cns.yaml | 2 +- cns/configuration/env.go | 42 ++++- cns/configuration/env_test.go | 20 +++ cns/fakes/cnsfake.go | 2 + cns/middlewares/mock/mockClient.go | 76 +++++++++ cns/middlewares/mock/mockSWIFTv2.go | 230 +++++++++++++++++++++++++ cns/middlewares/swiftV2.go | 214 ++++++++++++++++++++++++ cns/middlewares/swiftV2_test.go | 250 ++++++++++++++++++++++++++++ cns/middlewares/utils/utils.go | 26 +++ cns/middlewares/utils/utils_test.go | 56 +++++++ cns/restserver/internalapi_test.go | 10 +- cns/restserver/ipam.go | 72 ++++++-- cns/restserver/ipam_test.go | 128 +++++++++++++- cns/restserver/restserver.go | 27 ++- cns/restserver/util.go | 31 ++-- cns/service/main.go | 6 +- go.mod | 5 +- go.sum | 1 - 21 files changed, 1184 insertions(+), 55 deletions(-) create mode 100644 cns/middlewares/mock/mockClient.go create mode 100644 cns/middlewares/mock/mockSWIFTv2.go create mode 100644 cns/middlewares/swiftV2.go create mode 100644 cns/middlewares/swiftV2_test.go create mode 100644 cns/middlewares/utils/utils.go create mode 100644 cns/middlewares/utils/utils_test.go diff --git a/cni/plugin.go b/cni/plugin.go index 82a17f178a..1b6b85abeb 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -24,8 +24,10 @@ import ( "go.uber.org/zap" ) -var logger = log.CNILogger.With(zap.String("component", "cni-plugin")) -var storeLogger = log.CNILogger.With(zap.String("component", "cni-store")) +var ( + logger = log.CNILogger.With(zap.String("component", "cni-plugin")) + storeLogger = log.CNILogger.With(zap.String("component", "cni-store")) +) var errEmptyContent = errors.New("read content is zero bytes") diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index d1c9329fd3..b84f176f63 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -187,6 +187,8 @@ type PodInfo interface { Equals(PodInfo) bool // String implements string for logging PodInfos String() string + // SecondaryInterfacesExist returns true if there exist a secondary interface for this pod + SecondaryInterfacesExist() bool } type KubernetesPodInfo struct { @@ -199,9 +201,10 @@ var _ PodInfo = (*podInfo)(nil) // podInfo implements PodInfo for multiple schemas of Key type podInfo struct { KubernetesPodInfo - PodInfraContainerID string - PodInterfaceID string - Version podInfoScheme + PodInfraContainerID string + PodInterfaceID string + Version podInfoScheme + SecondaryInterfaceSet bool } func (p podInfo) String() string { @@ -255,6 +258,10 @@ func (p *podInfo) OrchestratorContext() (json.RawMessage, error) { return jsonContext, nil } +func (p *podInfo) SecondaryInterfacesExist() bool { + return p.SecondaryInterfaceSet +} + // NewPodInfo returns an implementation of PodInfo that returns the passed // configuration for their namesake functions. func NewPodInfo(infraContainerID, interfaceID, name, namespace string) PodInfo { @@ -292,6 +299,7 @@ func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) { } p.(*podInfo).PodInfraContainerID = req.InfraContainerID p.(*podInfo).PodInterfaceID = req.PodInterfaceID + p.(*podInfo).SecondaryInterfaceSet = req.SecondaryInterfacesExist return p, nil } @@ -453,11 +461,12 @@ type IPConfigRequest struct { // Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice type IPConfigsRequest struct { - DesiredIPAddresses []string `json:"desiredIPAddresses"` - PodInterfaceID string `json:"podInterfaceID"` - InfraContainerID string `json:"infraContainerID"` - OrchestratorContext json.RawMessage `json:"orchestratorContext"` - Ifname string `json:"ifname"` // Used by delegated IPAM + DesiredIPAddresses []string `json:"desiredIPAddresses"` + PodInterfaceID string `json:"podInterfaceID"` + InfraContainerID string `json:"infraContainerID"` + OrchestratorContext json.RawMessage `json:"orchestratorContext"` + Ifname string `json:"ifname"` // Used by delegated IPAM + SecondaryInterfacesExist bool `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func } // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD diff --git a/cns/api.go b/cns/api.go index 705907b238..19f8a63da8 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,8 +48,18 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) + AttachSWIFTv2Middleware(middleware SWIFTv2Middleware) } +// Middleware interface for testing later on +type SWIFTv2Middleware interface { + ValidateIPConfigsRequest(context.Context, *IPConfigsRequest) (types.ResponseCode, string) + GetIPConfig(context.Context, PodInfo) (PodIpInfo, error) + SetRoutes(*PodIpInfo) error +} + +type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) + // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC type IPConfigurationStatus struct { diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 541c2a6d84..260bb775c1 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -189,4 +189,4 @@ data: "ManageEndpointState": false, "ProgramSNATIPTables" : false } -# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. \ No newline at end of file diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 5713235df0..6b16668cb2 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -11,13 +11,27 @@ const ( EnvNodeName = "NODENAME" // EnvNodeIP is the IP of the node running this CNS binary EnvNodeIP = "NODE_IP" - // LabelSwiftV2 is the Node label for Swift V2 - LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" + // LabelNodeSwiftV2 is the Node label for Swift V2 + LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" + // LabelPodSwiftV2 is the Pod label for Swift V2 + LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" + EnvPodCIDRs = "POD_CIDRs" + EnvServiceCIDRs = "SERVICE_CIDRs" + EnvNodeCIDRs = "NODE_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) +// ErrPodCIDRsUnset indicates the the $EnvPodCIDRs variable is unset in the environment. +var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDRs) + +// ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. +var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) + +// ErrNodeCIDRsUnset indicates the the $EnvNodeCIDRs variable is unset in the environment. +var ErrNodeCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDRs) + // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { nodeName := os.Getenv(EnvNodeName) @@ -31,3 +45,27 @@ func NodeName() (string, error) { func NodeIP() string { return os.Getenv(EnvNodeIP) } + +func PodCIDRs() (string, error) { + podCIDRs := os.Getenv(EnvPodCIDRs) + if podCIDRs == "" { + return "", ErrPodCIDRsUnset + } + return podCIDRs, nil +} + +func ServiceCIDRs() (string, error) { + serviceCIDRs := os.Getenv(EnvServiceCIDRs) + if serviceCIDRs == "" { + return "", ErrServiceCIDRsUnset + } + return serviceCIDRs, nil +} + +func NodeCIDRs() (string, error) { + nodeCIDRs := os.Getenv(EnvNodeCIDRs) + if nodeCIDRs == "" { + return "", ErrNodeCIDRsUnset + } + return nodeCIDRs, nil +} diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index df06b6d610..cc411ec583 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -17,3 +17,23 @@ func TestNodeName(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "test", name) } + +func TestPodCIDRs(t *testing.T) { + _, err := PodCIDRs() + require.Error(t, err) + require.ErrorIs(t, err, ErrPodCIDRsUnset) + os.Setenv(EnvPodCIDRs, "test") + cidr, err := PodCIDRs() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} + +func TestServiceCIDRs(t *testing.T) { + _, err := ServiceCIDRs() + require.Error(t, err) + require.ErrorIs(t, err, ErrServiceCIDRsUnset) + os.Setenv(EnvServiceCIDRs, "test") + cidr, err := ServiceCIDRs() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} diff --git a/cns/fakes/cnsfake.go b/cns/fakes/cnsfake.go index 07618b01e4..aed304f976 100644 --- a/cns/fakes/cnsfake.go +++ b/cns/fakes/cnsfake.go @@ -276,3 +276,5 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error { } func (fake *HTTPServiceFake) Stop() {} + +func (fake *HTTPServiceFake) AttachSWIFTv2Middleware(cns.SWIFTv2Middleware) {} diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go new file mode 100644 index 0000000000..0bb8be743b --- /dev/null +++ b/cns/middlewares/mock/mockClient.go @@ -0,0 +1,76 @@ +package mock + +import ( + "context" + + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + ErrPodNotFound = errors.New("pod not found") + ErrMTPNCNotFound = errors.New("mtpnc not found") +) + +// Client implements the client.Client interface for testing. We only care about Get, the rest is nil ops. +type Client struct { + client.Client + mtPodCache map[string]*v1.Pod + mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +// NewClient returns a new MockClient. +func NewClient() *Client { + const podNetwork = "azure" + + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod3 := v1.Pod{} + testPod3.Labels = make(map[string]string) + testPod3.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod4 := v1.Pod{} + testPod4.Labels = make(map[string]string) + testPod4.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + + testMTPNC1.Status.PrimaryIP = "192.168.0.1/32" + testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + testMTPNC1.Status.GatewayIP = "10.0.0.1" + testMTPNC1.Status.NCID = "testncid" + + testMTPNC4 := v1alpha1.MultitenantPodNetworkConfig{} + + return &Client{ + mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1, "testpod3namespace/testpod3": &testPod3, "testpod4namespace/testpod4": &testPod4}, + mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ + "testpod1namespace/testpod1": &testMTPNC1, + "testpod4namespace/testpod4": &testMTPNC4, + }, + } +} + +// Get implements client.Client.Get. +func (c *Client) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error { + switch o := obj.(type) { + case *v1.Pod: + if pod, ok := c.mtPodCache[key.String()]; ok { + *o = *pod + } else { + return ErrPodNotFound + } + case *v1alpha1.MultitenantPodNetworkConfig: + if mtpnc, ok := c.mtpncCache[key.String()]; ok { + *o = *mtpnc + } else { + return ErrMTPNCNotFound + } + } + return nil +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go new file mode 100644 index 0000000000..11d1266f76 --- /dev/null +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -0,0 +1,230 @@ +package mock + +import ( + "context" + "fmt" + "net/netip" + "os" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/middlewares/utils" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + k8types "k8s.io/apimachinery/pkg/types" +) + +var ( + errMTPNCNotReady = errors.New("mtpnc is not ready") + errFailedToGetMTPNC = errors.New("failed to get mtpnc") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" +) + +type SWIFTv2Middleware struct { + mtPodState map[string]*v1.Pod + mtpncState map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +func NewMockSWIFTv2Middleware() *SWIFTv2Middleware { + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelPodSwiftV2] = "true" + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + + return &SWIFTv2Middleware{ + mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtpncState: map[string]*v1alpha1.MultitenantPodNetworkConfig{"testpod1namespace/testpod1": &testMTPNC1}, + } +} + +func (m *SWIFTv2Middleware) SetMTPNCReady() { + m.mtpncState["testpod1namespace/testpod1"].Status.PrimaryIP = "192.168.0.1" + m.mtpncState["testpod1namespace/testpod1"].Status.MacAddress = "00:00:00:00:00:00" + m.mtpncState["testpod1namespace/testpod1"].Status.GatewayIP = "10.0.0.1" + m.mtpncState["testpod1namespace/testpod1"].Status.NCID = "testncid" +} + +func (m *SWIFTv2Middleware) SetEnvVar() { + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") + os.Setenv(configuration.EnvNodeCIDRs, "10.0.3.10/24") +} + +func (m *SWIFTv2Middleware) UnsetEnvVar() error { + if err := os.Unsetenv(configuration.EnvPodCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvPodCIDRs, err) + } + if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err) + } + if err := os.Unsetenv(configuration.EnvNodeCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDRs, err) + } + return nil +} + +// validateMultitenantIPConfigsRequest validates if pod is multitenant +// nolint +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return types.UnexpectedError, errBuf.Error() + } + podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod, ok := m.mtPodState[podNamespacedName.String()] + if !ok { + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return types.UnexpectedError, errBuf.Error() + } + // check the pod labels for Swift V2, enrich the request with the multitenant flag. + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { + req.SecondaryInterfacesExist = true + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] + if !ok { + return types.UnexpectedError, errFailedToGetMTPNC.Error() + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } + } + return types.Success, "" +} + +// GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) +// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD +func (m *SWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] + if !ok { + return cns.PodIpInfo{}, errFailedToGetMTPNC + } + + podIPInfo := cns.PodIpInfo{} + podIPInfo.PodIPConfig = cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + } + podIPInfo.MacAddress = mtpnc.Status.MacAddress + podIPInfo.NICType = cns.DelegatedVMNIC + podIPInfo.SkipDefaultRoutes = false + + return podIPInfo, nil +} + +func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + podIPInfo.Routes = []cns.Route{} + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + } + podIPInfo.Routes = []cns.Route{route} + case cns.InfraNIC: + // Get and parse nodeCIDRs from env + nodeCIDRs, err := configuration.NodeCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get nodeCIDR from env") + } + nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse nodeCIDRs") + } + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse podCIDRs") + } + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse serviceCIDRs") + } + // Check if the podIPInfo is IPv4 or IPv6 + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + if ip.Is4() { + // routes for IPv4 podCIDR traffic + for _, podCIDRv4 := range podCIDRsV4 { + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) + } + // route for IPv4 serviceCIDR traffic + for _, serviceCIDRv4 := range serviceCIDRsV4 { + serviceCIDRv4Route := cns.Route{ + IPAddress: serviceCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) + } + // route for IPv4 nodeCIDR traffic + for _, nodeCIDRv4 := range nodeCIDRsv4 { + nodeCIDRv4Route := cns.Route{ + IPAddress: nodeCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) + } + } else { + // routes for IPv6 podCIDR traffic + for _, podCIDRv6 := range podCIDRv6 { + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) + } + // route for IPv6 serviceCIDR traffic + for _, serviceCIDRv6 := range serviceCIDRsV6 { + serviceCIDRv6Route := cns.Route{ + IPAddress: serviceCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) + } + // route for IPv6 nodeCIDR traffic + for _, nodeCIDRv6 := range nodeCIDRsv6 { + nodeCIDRv6Route := cns.Route{ + IPAddress: nodeCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + } + } + podIPInfo.SkipDefaultRoutes = true + case cns.BackendNIC: + default: + return errInvalidSWIFTv2NICType + } + return nil +} diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go new file mode 100644 index 0000000000..7326ccc6bd --- /dev/null +++ b/cns/middlewares/swiftV2.go @@ -0,0 +1,214 @@ +package middlewares + +import ( + "context" + "fmt" + "net/netip" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares/utils" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + errMTPNCNotReady = errors.New("mtpnc is not ready") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") + errInvalidMTPNCPrefixLength = errors.New("invalid prefix length for MTPNC primaryIP, must be 32") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" +) + +type SWIFTv2Middleware struct { + Cli client.Client +} + +// ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. +// nolint +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return types.UnexpectedError, errBuf.Error() + } + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return types.UnexpectedError, errBuf.Error() + } + + // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { + req.SecondaryInterfacesExist = true + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + } + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } + } + logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) + return types.Success, "" +} + +// GetIPConfig returns the pod's SWIFT V2 IP configuration. +func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache") + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return cns.PodIpInfo{}, errMTPNCNotReady + } + logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) + + // Parse MTPNC primaryIP to get the IP address and prefix length + p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) + if err != nil { + return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse mtpnc primaryIP %s", mtpnc.Status.PrimaryIP) + } + // Get the IP address and prefix length + ip := p.Addr() + prefixSize := p.Bits() + if prefixSize != prefixLength { + return cns.PodIpInfo{}, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) + } + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: ip.String(), + PrefixLength: uint8(prefixSize), + }, + MacAddress: mtpnc.Status.MacAddress, + NICType: cns.DelegatedVMNIC, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC + } + + return podIPInfo, nil +} + +// SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. +func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) + podIPInfo.Routes = []cns.Route{} + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + } + podIPInfo.Routes = []cns.Route{route} + case cns.InfraNIC: + // Get and parse nodeCIDRs from env + nodeCIDRs, err := configuration.NodeCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get nodeCIDR from env") + } + nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse nodeCIDRs") + } + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse podCIDRs") + } + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse serviceCIDRs") + } + // Check if the podIPInfo is IPv4 or IPv6 + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + if ip.Is4() { + // routes for IPv4 podCIDR traffic + for _, podCIDRv4 := range podCIDRsV4 { + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) + } + // route for IPv4 serviceCIDR traffic + for _, serviceCIDRv4 := range serviceCIDRsV4 { + serviceCIDRv4Route := cns.Route{ + IPAddress: serviceCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) + } + // route for IPv4 nodeCIDR traffic + for _, nodeCIDRv4 := range nodeCIDRsv4 { + nodeCIDRv4Route := cns.Route{ + IPAddress: nodeCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) + } + } else { + // routes for IPv6 podCIDR traffic + for _, podCIDRv6 := range podCIDRv6 { + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) + } + // route for IPv6 serviceCIDR traffic + for _, serviceCIDRv6 := range serviceCIDRsV6 { + serviceCIDRv6Route := cns.Route{ + IPAddress: serviceCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) + } + // route for IPv6 nodeCIDR traffic + for _, nodeCIDRv6 := range nodeCIDRsv6 { + nodeCIDRv6Route := cns.Route{ + IPAddress: nodeCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + } + } + podIPInfo.SkipDefaultRoutes = true + case cns.BackendNIC: + default: + return errInvalidSWIFTv2NICType + } + return nil +} diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go new file mode 100644 index 0000000000..74a2daf246 --- /dev/null +++ b/cns/middlewares/swiftV2_test.go @@ -0,0 +1,250 @@ +package middlewares + +import ( + "context" + "os" + "testing" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares/mock" + "github.com/Azure/azure-container-networking/cns/types" + "gotest.tools/v3/assert" +) + +var ( + testPod1GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod1Info = cns.NewPodInfo("898fb8-eth0", testPod1GUID, "testpod1", "testpod1namespace") + + testPod2GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod2Info = cns.NewPodInfo("b21e1e-eth0", testPod2GUID, "testpod2", "testpod2namespace") + + testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" + testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") + + testPod4GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod4Info = cns.NewPodInfo("b21e1e-eth0", testPod4GUID, "testpod4", "testpod4namespace") +) + +func setEnvVar() { + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") + os.Setenv(configuration.EnvNodeCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") +} + +func unsetEnvVar() { + os.Unsetenv(configuration.EnvPodCIDRs) + os.Unsetenv(configuration.EnvServiceCIDRs) + os.Unsetenv(configuration.EnvNodeCIDRs) +} + +func TestMain(m *testing.M) { + logger.InitLogger("testlogs", 0, 0, "./") + m.Run() +} + +func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + + happyReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + happyReq.OrchestratorContext = b + happyReq.SecondaryInterfacesExist = false + + respCode, err := middleware.ValidateIPConfigsRequest(context.TODO(), happyReq) + assert.Equal(t, err, "") + assert.Equal(t, respCode, types.Success) + assert.Equal(t, happyReq.SecondaryInterfacesExist, true) +} + +func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + + // Fail to unmarshal pod info test + failReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + failReq.OrchestratorContext = []byte("invalid") + respCode, _ := middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // Pod doesn't exist in cache test + failReq = &cns.IPConfigsRequest{ + PodInterfaceID: testPod2Info.InterfaceID(), + InfraContainerID: testPod2Info.InfraContainerID(), + } + b, _ := testPod2Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // Failed to get MTPNC + b, _ = testPod3Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // MTPNC not ready + b, _ = testPod4Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) +} + +func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { + setEnvVar() + defer unsetEnvVar() + + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + + ipInfo, err := middleware.GetIPConfig(context.TODO(), testPod1Info) + assert.Equal(t, err, nil) + assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) +} + +func TestGetSWIFTv2IPConfigFailure(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + + // Pod's MTPNC doesn't exist in cache test + _, err := middleware.GetIPConfig(context.TODO(), testPod2Info) + assert.ErrorContains(t, err, mock.ErrMTPNCNotFound.Error()) + + // Pod's MTPNC is not ready test + _, err = middleware.GetIPConfig(context.TODO(), testPod4Info) + assert.Error(t, err, errMTPNCNotReady.Error()) +} + +func TestSetRoutesSuccess(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + setEnvVar() + defer unsetEnvVar() + podIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + }, + } + desiredPodIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "10.0.1.10/24", + GatewayIPAddress: overlayGatewayv4, + }, + { + IPAddress: "10.0.0.0/16", + GatewayIPAddress: overlayGatewayv4, + }, + { + IPAddress: "10.240.0.1/16", + GatewayIPAddress: overlayGatewayv4, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "16A0:0010:AB00:001E::2/32", + GatewayIPAddress: overlayGatewayV6, + }, + { + IPAddress: "16A0:0010:AB00:0000::/32", + GatewayIPAddress: overlayGatewayV6, + }, + { + IPAddress: "16A0:0020:AB00:0000::/32", + GatewayIPAddress: overlayGatewayV6, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + Routes: []cns.Route{ + { + IPAddress: "0.0.0.0/0", + }, + }, + }, + } + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := middleware.SetRoutes(ipInfo) + assert.Equal(t, err, nil) + if ipInfo.NICType == cns.InfraNIC { + assert.Equal(t, ipInfo.SkipDefaultRoutes, true) + } else { + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) + } + + } + for i := range podIPInfo { + assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) + } +} + +func TestSetRoutesFailure(t *testing.T) { + // Failure due to env var not set + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} + podIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + }, + } + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := middleware.SetRoutes(ipInfo) + if err == nil { + t.Errorf("SetRoutes should fail due to env var not set") + } + } +} diff --git a/cns/middlewares/utils/utils.go b/cns/middlewares/utils/utils.go new file mode 100644 index 0000000000..2eca8551ea --- /dev/null +++ b/cns/middlewares/utils/utils.go @@ -0,0 +1,26 @@ +package utils + +import ( + "fmt" + "net/netip" + "strings" +) + +// ParseCIDRs parses the comma separated list of CIDRs and returns the IPv4 and IPv6 CIDRs. +func ParseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { + v4IPs = []string{} + v6IPs = []string{} + for _, cidr := range strings.Split(cidrs, ",") { + p, err := netip.ParsePrefix(cidr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse CIDR %s : %w", cidr, err) + } + ip := p.Addr() + if ip.Is4() { + v4IPs = append(v4IPs, cidr) + } else { + v6IPs = append(v6IPs, cidr) + } + } + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/utils/utils_test.go b/cns/middlewares/utils/utils_test.go new file mode 100644 index 0000000000..aaa85c024f --- /dev/null +++ b/cns/middlewares/utils/utils_test.go @@ -0,0 +1,56 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseCIDRs(t *testing.T) { + // Test valid IPv4 CIDR + cidr := "192.168.0.0/16" + expectedV4 := []string{"192.168.0.0/16"} + expectedV6 := []string{} + resultV4, resultV6, err := ParseCIDRs(cidr) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test valid IPv6 CIDR + cidr = "2001:db8::/32" + expectedV4 = []string{} + expectedV6 = []string{"2001:db8::/32"} + resultV4, resultV6, err = ParseCIDRs(cidr) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test multiple valid CIDRs + cidrs := "192.168.0.0/16,10.0.0.0/8,2001:db8::/32" + expectedV4 = []string{"192.168.0.0/16", "10.0.0.0/8"} + expectedV6 = []string{"2001:db8::/32"} + resultV4, resultV6, err = ParseCIDRs(cidrs) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test invalid CIDR + cidr = "192.168.0.0/33" + _, _, err = ParseCIDRs(cidr) + if err == nil { + t.Errorf("Expected error but got nil") + } + + // Test invalid CIDRs + cidrs = "192.168.0.0/33,10.0.0.0/8,2001:db8::/33" + _, _, err = ParseCIDRs(cidrs) + if err == nil { + t.Errorf("Expected error but got nil") + } +} diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go index 854fb3bc81..0d26e0f2f0 100644 --- a/cns/restserver/internalapi_test.go +++ b/cns/restserver/internalapi_test.go @@ -29,8 +29,10 @@ import ( ) const ( - primaryIp = "10.0.0.5" - gatewayIp = "10.0.0.1" + primaryIP = "10.0.0.5" + SWIFTv2IP = "192.168.0.1" + SWIFTv2MAC = "00:00:00:00:00:00" + gatewayIP = "10.0.0.1" subnetPrfixLength = 24 dockerContainerType = cns.Docker releasePercent = 50 @@ -916,9 +918,9 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest) func generateNetworkContainerRequest(secondaryIps map[string]cns.SecondaryIPConfig, ncID, ncVersion string) *cns.CreateNetworkContainerRequest { var ipConfig cns.IPConfiguration ipConfig.DNSServers = dnsservers - ipConfig.GatewayIPAddress = gatewayIp + ipConfig.GatewayIPAddress = gatewayIP var ipSubnet cns.IPSubnet - ipSubnet.IPAddress = primaryIp + ipSubnet.IPAddress = primaryIP ipSubnet.PrefixLength = subnetPrfixLength ipConfig.IPSubnet = ipSubnet diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 93984a8578..09c24578bf 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/http" @@ -23,9 +24,10 @@ var ( ErrNoNCs = errors.New("No NCs found in the CNS internal state") ) -// requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response -func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) +// requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs +func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -43,7 +45,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %s", err, ipconfigsRequest), + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), }, PodIPInfo: podIPInfo, }, err @@ -71,6 +73,50 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn } } + // Check if request is for pod with secondary interface(s) + if podInfo.SecondaryInterfacesExist() { + // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here + SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) + if err != nil { + defer func() { + logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config : %v", err) + } + }() + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", ipconfigsRequest) + } + podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) + // Setting up routes for SWIFTv2 scenario + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := service.SWIFTv2Middleware.SetRoutes(ipInfo) + if err != nil { + defer func() { //nolint:gocritic + logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config : %v", err) + } + }() + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.UnexpectedError, + Message: fmt.Sprintf("failed to set SWIFTv2 routes : %v", err), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set SWIFTv2 routes : %v", ipconfigsRequest) + } + } + } + return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.Success, @@ -117,7 +163,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r } } - ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(ipconfigsRequest) //nolint:contextcheck // appease linter + ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) //nolint:contextcheck // appease linter if errResp != nil { // As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above reserveResp := &cns.IPConfigResponse{ @@ -164,7 +210,7 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r return } - ipConfigsResp, err := service.requestIPConfigHandlerHelper(ipconfigsRequest) // nolint:contextcheck // appease linter + ipConfigsResp, err := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter if err != nil { w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) err = service.Listener.Encode(w, &ipConfigsResp) @@ -240,8 +286,8 @@ func (service *HTTPRestService) updateEndpointState(ipconfigsRequest cns.IPConfi } // releaseIPConfigHandlerHelper validates the request and removes the endpoint associated with the pod -func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) +func (service *HTTPRestService) releaseIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.Response{ ReturnCode: returnCode, @@ -255,7 +301,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn ReturnCode: types.UnexpectedError, Message: err.Error(), } - return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error + return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error } } @@ -263,7 +309,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn return &cns.Response{ ReturnCode: types.UnexpectedError, Message: err.Error(), - }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %s", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error + }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %+v", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error } return &cns.Response{ @@ -313,7 +359,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r Ifname: ipconfigRequest.Ifname, } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) @@ -335,14 +381,14 @@ func (service *HTTPRestService) releaseIPConfigsHandler(w http.ResponseWriter, r ReturnCode: types.UnexpectedError, Message: err.Error(), } - logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) + logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err) return } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index b552b15774..3518199a24 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/netip" @@ -13,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" + "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/store" @@ -65,6 +67,7 @@ func getTestService() *HTTPRestService { var config common.ServiceConfig httpsvc, _ := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, store.NewMockStore(""), nil, nil) svc = httpsvc + svc.ipConfigsRequestValidators = append(svc.ipConfigsRequestValidators, svc.validateDefaultIPConfigsRequest) httpsvc.IPAMPoolMonitor = &fakes.MonitorFake{} setOrchestratorTypeInternal(cns.KubernetesCRD) @@ -96,10 +99,10 @@ func requestIPAddressAndGetState(t *testing.T, req cns.IPConfigsRequest) ([]cns. } for i := range podIPInfo { - assert.Equal(t, primaryIp, podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress) + assert.Equal(t, primaryIP, podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress) assert.Equal(t, subnetPrfixLength, int(podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength)) assert.Equal(t, dnsservers, podIPInfo[i].NetworkContainerPrimaryIPConfig.DNSServers) - assert.Equal(t, gatewayIp, podIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress) + assert.Equal(t, gatewayIP, podIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress) assert.Equal(t, subnetPrfixLength, int(podIPInfo[i].PodIPConfig.PrefixLength)) assert.Equal(t, fakes.HostPrimaryIP, podIPInfo[i].HostPrimaryIPInfo.PrimaryIP) assert.Equal(t, fakes.HostSubnet, podIPInfo[i].HostPrimaryIPInfo.Subnet) @@ -1529,3 +1532,124 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { t.Fatalf("Expected fail requesting IPs due to only having one in the ipconfig map, IPs in the pool will not be assigned") } } + +func TestIPAMGetSWIFTv2IPSuccess(t *testing.T) { + svc := getTestService() + middleware := mock.NewMockSWIFTv2Middleware() + svc.AttachSWIFTv2Middleware(middleware) + + middleware.SetEnvVar() + defer middleware.UnsetEnvVar() //nolint:errcheck // ignore error + middleware.SetMTPNCReady() + + ncStates := []ncState{ + { + ncID: testNCID, + ips: []string{ + testIP1, + }, + }, + { + ncID: testNCIDv6, + ips: []string{ + testIP1v6, + }, + }, + } + + // Add Available Pod IP to state + for i := range ncStates { + ipconfigs := make(map[string]cns.IPConfigurationStatus, 0) + state := NewPodState(ncStates[i].ips[0], ipIDs[i][0], ncStates[i].ncID, types.Available, 0) + ipconfigs[state.ID] = state + err := UpdatePodIPConfigState(t, svc, ipconfigs, ncStates[i].ncID) + if err != nil { + t.Fatalf("Expected to not fail adding IPs to state: %+v", err) + } + } + + req := cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + req.OrchestratorContext = b + req.DesiredIPAddresses = make([]string, 2) + req.DesiredIPAddresses[0] = testIP1 + req.DesiredIPAddresses[1] = testIP1v6 + + resp, err := svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err != nil { + t.Fatalf("Expected to not fail requesting IPs: %+v", err) + } + podIPInfo := resp.PodIPInfo + + if len(podIPInfo) != 3 { + t.Fatalf("Expected to get 3 pod IP info (IPv4, IPv6, Multitenant IP), actual %d", len(podIPInfo)) + } + + // Asserting that SWIFT v2 IP is returned + assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) + assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MacAddress) + assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) + assert.False(t, podIPInfo[2].SkipDefaultRoutes) +} + +func TestIPAMGetSWIFTv2IPFailure(t *testing.T) { + svc := getTestService() + middleware := mock.NewMockSWIFTv2Middleware() + svc.AttachSWIFTv2Middleware(middleware) + ncStates := []ncState{ + { + ncID: testNCID, + ips: []string{ + testIP1, + }, + }, + { + ncID: testNCIDv6, + ips: []string{ + testIP1v6, + }, + }, + } + // Add Available Pod IP to state + for i := range ncStates { + ipconfigs := make(map[string]cns.IPConfigurationStatus, 0) + state := NewPodState(ncStates[i].ips[0], ipIDs[i][0], ncStates[i].ncID, types.Available, 0) + ipconfigs[state.ID] = state + err := UpdatePodIPConfigState(t, svc, ipconfigs, ncStates[i].ncID) + if err != nil { + t.Fatalf("Expected to not fail adding IPs to state: %+v", err) + } + } + req := cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + req.OrchestratorContext = b + req.DesiredIPAddresses = make([]string, 2) + req.DesiredIPAddresses[0] = testIP1 + req.DesiredIPAddresses[1] = testIP1v6 + _, err := svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err == nil { + t.Fatalf("Expected failing requesting IPs due to MTPNC not ready") + } + available := svc.GetAvailableIPConfigs() + if len(available) != 2 { + t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") + } + + middleware.SetMTPNCReady() + + _, err = svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err == nil { + t.Fatalf("Expected failing requesting IPs due to not able to set routes") + } + + available = svc.GetAvailableIPConfigs() + if len(available) != 2 { + t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") + } +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 70676c28e3..08fff69664 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -30,10 +30,8 @@ import ( // All helper/utility functions - util.go // Constants - const.go -var ( - // Named Lock for accessing different states in httpRestServiceState - namedLock = acn.InitNamedLock() -) +// Named Lock for accessing different states in httpRestServiceState +var namedLock = acn.InitNamedLock() type interfaceGetter interface { GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) @@ -69,11 +67,13 @@ type HTTPRestService struct { state *httpRestServiceState podsPendingIPAssignment *bounded.TimedSet sync.RWMutex - dncPartitionKey string - EndpointState map[string]*EndpointInfo // key : container id - EndpointStateStore store.KeyValueStore - cniConflistGenerator CNIConflistGenerator - generateCNIConflistOnce sync.Once + dncPartitionKey string + EndpointState map[string]*EndpointInfo // key : container id + EndpointStateStore store.KeyValueStore + cniConflistGenerator CNIConflistGenerator + generateCNIConflistOnce sync.Once + ipConfigsRequestValidators []cns.IPConfigsRequestValidator + SWIFTv2Middleware cns.SWIFTv2Middleware } type CNIConflistGenerator interface { @@ -228,6 +228,9 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } + // Adding the default ipconfigs request validator + service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} + // Add handlers. listener := service.Listener // default handlers @@ -349,3 +352,9 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { } }) } + +func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { + service.SWIFTv2Middleware = middleware + // adding the SWIFT v2 ipconfigs request validator + service.ipConfigsRequestValidators = append(service.ipConfigsRequestValidators, middleware.ValidateIPConfigsRequest) +} diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 094fe49872..b3084b9ad2 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -767,17 +767,13 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } } -func (service *HTTPRestService) validateIPConfigsRequest( - ipConfigsRequest cns.IPConfigsRequest, -) (cns.PodInfo, types.ResponseCode, string) { - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { - return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" - } - - if ipConfigsRequest.OrchestratorContext == nil { - return nil, - types.EmptyOrchestratorContext, - fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) +func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) { + // looping through all the ipconfigs request validators, if any validator fails, return the error + for _, validator := range service.ipConfigsRequestValidators { + respCode, message := validator(ctx, &ipConfigsRequest) + if respCode != types.Success { + return nil, respCode, message + } } // retrieve podinfo from orchestrator context @@ -788,6 +784,18 @@ func (service *HTTPRestService) validateIPConfigsRequest( return podInfo, types.Success, "" } +// validateDefaultIPConfigsRequest validates the request for default IP configs request +func (service *HTTPRestService) validateDefaultIPConfigsRequest(_ context.Context, ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { + return types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" + } + + if ipConfigsRequest.OrchestratorContext == nil { + return types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) + } + return types.Success, "" +} + // getPrimaryHostInterface returns the cached InterfaceInfo, if available, otherwise // queries the IMDS to get the primary interface info and caches it in the server state // before returning the result. @@ -829,6 +837,7 @@ func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus podIPInfo.HostPrimaryIPInfo.PrimaryIP = primaryHostInterface.PrimaryIP podIPInfo.HostPrimaryIPInfo.Subnet = primaryHostInterface.Subnet podIPInfo.HostPrimaryIPInfo.Gateway = primaryHostInterface.Gateway + podIPInfo.NICType = cns.InfraNIC return nil } diff --git a/cns/service/main.go b/cns/service/main.go index 6f53df9a4a..875aad6747 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -37,6 +37,7 @@ import ( nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" @@ -1154,7 +1155,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } // check the Node labels for Swift V2 - if _, ok := node.Labels[configuration.LabelSwiftV2]; ok { + if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { cnsconfig.EnableSwiftV2 = true cnsconfig.WatchPods = true // TODO(rbtr): create the NodeInfo for Swift V2 @@ -1322,6 +1323,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") } + // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service + swiftV2Middleware := middlewares.SWIFTv2Middleware{Cli: manager.GetClient()} + httpRestService.AttachSWIFTv2Middleware(&swiftV2Middleware) } // adding some routes to the root service mux diff --git a/go.mod b/go.mod index ff94c8add3..9a2ce367b0 100644 --- a/go.mod +++ b/go.mod @@ -132,7 +132,10 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) -require sigs.k8s.io/yaml v1.3.0 +require ( + gotest.tools/v3 v3.0.3 + sigs.k8s.io/yaml v1.3.0 +) require github.com/emicklei/go-restful/v3 v3.11.0 // indirect diff --git a/go.sum b/go.sum index 6d1a09c595..d744e44258 100644 --- a/go.sum +++ b/go.sum @@ -1270,7 +1270,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=