cnf network: add frrk8 metallb test cases

tests/cnf/core/network/metallb/internal/frr/frr.go

@@ -139,6 +139,53 @@ func DefineBGPConfig(localBGPASN, remoteBGPASN int, neighborsIPAddresses []strin
 	return bgpConfig
 }
 
+// DefineBGPConfigWithStaticRouteAndNetwork defines BGP config file with static route and network.
+func DefineBGPConfigWithStaticRouteAndNetwork(localBGPASN, remoteBGPASN int, hubPodIPs,
+	advertisedIPv4Routes, advertisedIPv6Routes, neighborsIPAddresses []string,
+	multiHop, bfd bool) string {
+	bgpConfig := tsparams.FRRBaseConfig +
+		fmt.Sprintf("ip route %s/32 %s\n", neighborsIPAddresses[1], hubPodIPs[0]) +
+		fmt.Sprintf("ip route %s/32 %s\n!\n", neighborsIPAddresses[0], hubPodIPs[1]) +
+		fmt.Sprintf("router bgp %d\n", localBGPASN) +
+		tsparams.FRRDefaultBGPPreConfig
+
+	for _, ipAddress := range neighborsIPAddresses {
+		bgpConfig += fmt.Sprintf("  neighbor %s remote-as %d\n  neighbor %s password %s\n",
+			ipAddress, remoteBGPASN, ipAddress, tsparams.BGPPassword)
+
+		if bfd {
+			bgpConfig += fmt.Sprintf("  neighbor %s bfd\n", ipAddress)
+		}
+
+		if multiHop {
+			bgpConfig += fmt.Sprintf("  neighbor %s ebgp-multihop 2\n", ipAddress)
+		}
+	}
+
+	bgpConfig += "!\naddress-family ipv4 unicast\n"
+	for _, ipAddress := range neighborsIPAddresses {
+		bgpConfig += fmt.Sprintf("  neighbor %s activate\n", ipAddress)
+	}
+
+	bgpConfig += fmt.Sprintf("  network %s\n", advertisedIPv4Routes[0])
+	bgpConfig += fmt.Sprintf("  network %s\n", advertisedIPv4Routes[1])
+	bgpConfig += "exit-address-family\n"
+
+	// Add network commands only once for IPv6
+	bgpConfig += "!\naddress-family ipv6 unicast\n"
+	for _, ipAddress := range neighborsIPAddresses {
+		bgpConfig += fmt.Sprintf("  neighbor %s activate\n", ipAddress)
+	}
+
+	bgpConfig += fmt.Sprintf("  network %s\n", advertisedIPv6Routes[0])
+	bgpConfig += fmt.Sprintf("  network %s\n", advertisedIPv6Routes[1])
+	bgpConfig += "exit-address-family\n"
+
+	bgpConfig += "!\nline vty\n!\nend\n"
+
+	return bgpConfig
+}
+
 // BGPNeighborshipHasState verifies that BGP session on a pod has given state.
 func BGPNeighborshipHasState(frrPod *pod.Builder, neighborIPAddress string, state string) (bool, error) {
 	var result map[string]bgpDescription
@@ -238,6 +285,34 @@ func GetBGPCommunityStatus(frrPod *pod.Builder, ipProtocolVersion string) (*bgpS
 	return getBgpStatus(frrPod, fmt.Sprintf("show bgp %s community %s json", ipProtocolVersion, "65535:65282"))
 }
 
+// FetchBGPConnectTimeValue fetches and returns the ConnectRetryTimer value for the specified BGP peer.
+func FetchBGPConnectTimeValue(frrk8sPods []*pod.Builder, bgpPeerIP string) (int, error) {
+	for _, frrk8sPod := range frrk8sPods {
+		// Run the "show bgp neighbor <bgpPeerIP> json" command on each pod
+		output, err := frrk8sPod.ExecCommand(append(netparam.VtySh,
+			fmt.Sprintf("show bgp neighbor %s json", bgpPeerIP)), "frr")
+		if err != nil {
+			return 0, fmt.Errorf("error collecting BGP neighbor info from pod %s: %w",
+				frrk8sPod.Definition.Name, err)
+		}
+
+		// Parsing JSON
+		var bgpData map[string]BGPConnectionInfo
+		err = json.Unmarshal(output.Bytes(), &bgpData)
+
+		if err != nil {
+			return 0, fmt.Errorf("error parsing BGP neighbor JSON for pod %s: %w", frrk8sPod.Definition.Name, err)
+		}
+
+		// Extracting ConnectRetryTimer from the parsed JSON
+		for _, bgpInfo := range bgpData {
+			return bgpInfo.ConnectRetryTimer, nil
+		}
+	}
+
+	return 0, fmt.Errorf("no BGP neighbor data found for peer %s", bgpPeerIP)
+}
+
 func getBgpStatus(frrPod *pod.Builder, cmd string, containerName ...string) (*bgpStatus, error) {
 	cName := "frr"
 
@@ -279,32 +354,107 @@ func runningConfig(frrPod *pod.Builder) (string, error) {
 	return bgpStateOut.String(), nil
 }
 
-// FetchBGPConnectTimeValue fetches and returns the ConnectRetryTimer value for the specified BGP peer.
-func FetchBGPConnectTimeValue(frrk8sPods []*pod.Builder, bgpPeerIP string) (int, error) {
+// GetBGPAdvertisedRoutes retrieves the routes advertised from the external frr pod to the frr nodes.
+func GetBGPAdvertisedRoutes(frrPod *pod.Builder, nodeIPs []string) (map[string]string, error) {
+	allRoutes := make(map[string]string)
+
+	// Loop through each nodeIP and execute the command
+	for _, nodeIP := range nodeIPs {
+		// Execute the BGP command for each nodeIP
+		routes, err := frrPod.ExecCommand(append(netparam.VtySh,
+			fmt.Sprintf("sh ip bgp neighbors %s advertised-routes json", nodeIP)))
+		if err != nil {
+			return nil, fmt.Errorf("error collecting BGP advertised routes from pod %s for nodeIP %s: %w",
+				frrPod.Definition.Name, nodeIP, err)
+		}
+
+		// Parse the BGP advertised routes for each nodeIP
+		bgpAdvertised, err := parseBGPAdvertisedRoutes(routes.String())
+		if err != nil {
+			return nil, fmt.Errorf("error parsing BGP advertised routes for nodeIP %s: %w", nodeIP, err)
+		}
+
+		// Store the result in the map for the current nodeIP
+		allRoutes[nodeIP] = bgpAdvertised
+	}
+
+	// Return the map of routes
+	return allRoutes, nil
+}
+
+// VerifyBGPReceivedRoutesOnFrrNodes verifies routes were received via BGP on Frr nodes.
+func VerifyBGPReceivedRoutesOnFrrNodes(frrk8sPods []*pod.Builder) (string, error) {
+	var result strings.Builder
+
 	for _, frrk8sPod := range frrk8sPods {
-		// Run the "show bgp neighbor <bgpPeerIP> json" command on each pod
-		output, err := frrk8sPod.ExecCommand(append(netparam.VtySh,
-			fmt.Sprintf("show bgp neighbor %s json", bgpPeerIP)), "frr")
+		// Run the "sh ip route bgp json" command on each pod
+		output, err := frrk8sPod.ExecCommand(append(netparam.VtySh, "sh ip route bgp json"), "frr")
 		if err != nil {
-			return 0, fmt.Errorf("error collecting BGP neighbor info from pod %s: %w",
+			return "", fmt.Errorf("error collecting BGP received routes from pod %s: %w",
 				frrk8sPod.Definition.Name, err)
 		}
 
-		// Parsing JSON
-		var bgpData map[string]BGPConnectionInfo
-		err = json.Unmarshal(output.Bytes(), &bgpData)
+		// Parse the JSON output to get the BGP routes
+		bgpRoutes, err := parseBGPReceivedRoutes(output.String())
 
 		if err != nil {
-			return 0, fmt.Errorf("error parsing BGP neighbor JSON for pod %s: %w", frrk8sPod.Definition.Name, err)
+			return "", fmt.Errorf("error parsing BGP JSON from pod %s: %w", frrk8sPod.Definition.Name, err)
 		}
 
-		// Extracting ConnectRetryTimer from the parsed JSON
-		for _, bgpInfo := range bgpData {
-			return bgpInfo.ConnectRetryTimer, nil
+		// Write the pod name to the result
+		result.WriteString(fmt.Sprintf("Pod: %s\n", frrk8sPod.Definition.Name))
+
+		// Extract and write the prefixes (keys of the Routes map) and corresponding route info
+		for prefix, routeInfos := range bgpRoutes.Routes {
+			result.WriteString(fmt.Sprintf("  Prefix: %s\n", prefix))
+
+			for _, routeInfo := range routeInfos {
+				result.WriteString(fmt.Sprintf("    Route Info: Prefix: %s", routeInfo.Prefix))
+			}
 		}
+
+		result.WriteString("\n") // Add an empty line between pods
 	}
 
-	return 0, fmt.Errorf("no BGP neighbor data found for peer %s", bgpPeerIP)
+	return result.String(), nil
+}
+
+func parseBGPReceivedRoutes(jsonData string) (*BgpReceivedRoutes, error) {
+	var bgpRoutes map[string][]RouteInfo // This matches the structure of your JSON
+
+	// Parse the JSON data into the struct
+	err := json.Unmarshal([]byte(jsonData), &bgpRoutes)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing BGP received routes: %w", err)
+	}
+
+	// Create a new BgpReceivedRoutes struct and populate it with the parsed data
+	parsedRoutes := &BgpReceivedRoutes{
+		Routes: bgpRoutes, // The map directly holds prefixes as keys
+	}
+
+	// Print the parsed routes for debugging
+	fmt.Printf("Parsed Routes: %+v\n", bgpRoutes)
+
+	return parsedRoutes, nil
+}
+
+func parseBGPAdvertisedRoutes(jsonData string) (string, error) {
+	var bgpRoutes BgpAdvertisedRoutes
+
+	// Parse the JSON data into the struct
+	err := json.Unmarshal([]byte(jsonData), &bgpRoutes)
+	if err != nil {
+		return "", fmt.Errorf("error parsing BGP advertised routes: %w", err)
+	}
+
+	// Format only the network values as a string
+	var result strings.Builder
+	for _, route := range bgpRoutes.AdvertisedRoutes {
+		result.WriteString(fmt.Sprintf("%s\n", route.Network))
+	}
+
+	return result.String(), nil
 }
 
 // ResetBGPConnection restarts the TCP connection.

tests/cnf/core/network/metallb/internal/tsparams/consts.go

@@ -9,8 +9,13 @@ const (
 	LabelBGPTestCases = "bgp"
 	// LabelLayer2TestCases represents layer2 label that can be used for test cases selection.
 	LabelLayer2TestCases = "layer2"
+	// LabelFRRTestCases represents frrk8 label that can be used for test cases selection.
+	LabelFRRTestCases = "frrk8s"
 	// BGPPassword var is used to set password for BGP session between FRR speakers.
 	BGPPassword = "bgp-test"
+	// MlbAddressListError an error message when the ECO_CNF_CORE_NET_MLB_ADDR_LIST is incorrect.
+	MlbAddressListError = "An unexpected error occurred while " +
+		"determining the IP addresses from the ECO_CNF_CORE_NET_MLB_ADDR_LIST environment variable."
 	// DaemonsFile represents FRR default daemon configuration template.
 	DaemonsFile = `
 	# This file tells the frr package which daemons to start.

tests/cnf/core/network/metallb/internal/tsparams/mlbvars.go

@@ -40,10 +40,14 @@ var (
 	MetalLbIo = "metallb"
 	// MetalLbDsName default metalLb speaker daemonset names.
 	MetalLbDsName = "speaker"
+	// FrrDsName default metalLb frr daemonset name.
+	FrrDsName = "frr-k8s"
 	// FRRK8sDefaultLabel represents the default metalLb FRRK8S pod label.
 	FRRK8sDefaultLabel = "component=frr-k8s"
 	// ExternalMacVlanNADName represents default external NetworkAttachmentDefinition name.
 	ExternalMacVlanNADName = "external"
+	// HubMacVlanNADName represents default external NetworkAttachmentDefinition name.
+	HubMacVlanNADName = "nad-hub"
 	// SleepCMD represents shel sleep command.
 	SleepCMD = []string{"/bin/bash", "-c", "sleep INF"}
 	// FRRContainerName represents default FRR's container name.

tests/cnf/core/network/metallb/tests/bfd-test.go

@@ -35,8 +35,7 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa
 		var err error
 		By("Getting MetalLb load balancer ip addresses")
 		ipv4metalLbIPList, ipv6metalLbIPList, err = metallbenv.GetMetalLbIPByIPStack()
-		Expect(err).ToNot(HaveOccurred(), "An unexpected error occurred while "+
-			"determining the IP addresses from the ECO_CNF_CORE_NET_MLB_ADDR_LIST environment variable.")
+		Expect(err).ToNot(HaveOccurred(), tsparams.MlbAddressListError)
 
 		if len(ipv4metalLbIPList) < 2 {
 			Skip("MetalLb BFD tests require 2 ip addresses. Please check ECO_CNF_CORE_NET_MLB_ADDR_LIST env var")
@@ -60,7 +59,7 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa
 
 		err = metallbenv.IsEnvVarMetalLbIPinNodeExtNetRange(ipv4NodeAddrList, ipv4metalLbIPList, nil)
 		Expect(err).ToNot(HaveOccurred(), "Failed to validate metalLb exported ip address")
-		createExternalNad()
+		createExternalNad(tsparams.ExternalMacVlanNADName)
 	})
 
 	Context("single hop", Label("singlehop"), func() {
@@ -212,7 +211,7 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa
 
 		DescribeTable("should provide fast link failure detection", reportxml.ID("47186"),
 			func(bgpProtocol, ipStack string, externalTrafficPolicy corev1.ServiceExternalTrafficPolicyType) {
-				createExternalNad()
+				createExternalNad(tsparams.ExternalMacVlanNADName)
 
 				By("Verifying that speaker route map is not empty")
 				Expect(speakerRoutesMap).ToNot(BeNil(), "Speaker route map is empty")

tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go

@@ -80,7 +80,8 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
 			metallb.GetBGPPeerGVR(),
 			metallb.GetBGPAdvertisementGVR(),
 			metallb.GetIPAddressPoolGVR(),
-			metallb.GetMetalLbIoGVR())
+			metallb.GetMetalLbIoGVR(),
+			metallb.GetFrrConfigurationGVR())
 		Expect(err).ToNot(HaveOccurred(), "Failed to remove object's from operator namespace")
 
 		By("Cleaning test namespace")
@@ -181,7 +182,7 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
 
 func createAndDeployFRRPod() *pod.Builder {
 	By("Creating External NAD")
-	createExternalNad()
+	createExternalNad(tsparams.ExternalMacVlanNADName)
 
 	By("Creating static ip annotation")
 

tests/cnf/core/network/metallb/tests/bgp-tests.go

@@ -31,8 +31,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
 		var err error
 		By("Getting MetalLb load balancer ip addresses")
 		ipv4metalLbIPList, ipv6metalLbIPList, err = metallbenv.GetMetalLbIPByIPStack()
-		Expect(err).ToNot(HaveOccurred(), "An unexpected error occurred while "+
-			"determining the IP addresses from the ECO_CNF_CORE_NET_MLB_ADDR_LIST environment variable.")
+		Expect(err).ToNot(HaveOccurred(), tsparams.MlbAddressListError)
 
 		By("Getting external nodes ip addresses")
 		cnfWorkerNodeList, err = nodes.List(APIClient,
@@ -64,7 +63,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
 
 	BeforeEach(func() {
 		By("Creating External NAD")
-		createExternalNad()
+		createExternalNad(tsparams.ExternalMacVlanNADName)
 
 		By("Listing metalLb speakers pod")
 		var err error

tests/cnf/core/network/metallb/tests/common.go

@@ -91,12 +91,22 @@ func createConfigMap(
 	return masterConfigMap
 }
 
-func createExternalNad() {
+func createHubConfigMap(name string) *configmap.Builder {
+	frrBFDConfig := frr.DefineBGPConfig(
+		tsparams.LocalBGPASN, tsparams.LocalBGPASN, []string{"10.10.0.10"}, false, false)
+	configMapData := frr.DefineBaseConfig(tsparams.DaemonsFile, frrBFDConfig, "")
+	hubConfigMap, err := configmap.NewBuilder(APIClient, name, tsparams.TestNamespaceName).WithData(configMapData).Create()
+	Expect(err).ToNot(HaveOccurred(), "Failed to create hub config map")
+
+	return hubConfigMap
+}
+
+func createExternalNad(name string) {
 	By("Creating external BR-EX NetworkAttachmentDefinition")
 
 	macVlanPlugin, err := define.MasterNadPlugin(coreparams.OvnExternalBridge, "bridge", nad.IPAMStatic())
 	Expect(err).ToNot(HaveOccurred(), "Failed to define master nad plugin")
-	externalNad, err = nad.NewBuilder(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName).
+	externalNad, err = nad.NewBuilder(APIClient, name, tsparams.TestNamespaceName).
 		WithMasterPlugin(macVlanPlugin).Create()
 	Expect(err).ToNot(HaveOccurred(), "Failed to create external NetworkAttachmentDefinition")
 	Expect(externalNad.Exists()).To(BeTrue(), "Failed to detect external NetworkAttachmentDefinition")
@@ -195,19 +205,47 @@ func createFrrPod(
 
 	By("Creating FRR container")
 
-	if configmapName != "" {
-		frrContainer := pod.NewContainerBuilder(
-			tsparams.FRRSecondContainerName, NetConfig.CnfNetTestContainer, tsparams.SleepCMD).
-			WithSecurityCapabilities([]string{"NET_ADMIN", "NET_RAW", "SYS_ADMIN"}, true)
+	frrContainer := pod.NewContainerBuilder(
+		tsparams.FRRSecondContainerName, NetConfig.CnfNetTestContainer, tsparams.SleepCMD).
+		WithSecurityCapabilities([]string{"NET_ADMIN", "NET_RAW", "SYS_ADMIN"}, true)
+
+	frrCtr, err := frrContainer.GetContainerCfg()
+	Expect(err).ToNot(HaveOccurred(), "Failed to get container configuration")
+	frrPod.WithAdditionalContainer(frrCtr).WithLocalVolume(configmapName, "/etc/frr")
+
+	By("Creating FRR pod in the test namespace")
+
+	frrPod, err = frrPod.WithPrivilegedFlag().CreateAndWaitUntilRunning(time.Minute)
+	Expect(err).ToNot(HaveOccurred(), "Failed to create FRR test pod")
+
+	return frrPod
+}
 
-		frrCtr, err := frrContainer.GetContainerCfg()
-		Expect(err).ToNot(HaveOccurred(), "Failed to get container configuration")
-		frrPod.WithAdditionalContainer(frrCtr).WithLocalVolume(configmapName, "/etc/frr")
+func createFrrHubPod(name, nodeName, configmapName string, defaultCMD []string,
+	secondaryNetConfig []*types.NetworkSelectionElement, podName ...string) *pod.Builder {
+	if len(podName) > 0 {
+		name = podName[0]
 	}
 
+	frrPod := pod.NewBuilder(APIClient, name, tsparams.TestNamespaceName, NetConfig.FrrImage).
+		DefineOnNode(nodeName).
+		WithTolerationToMaster().
+		WithSecondaryNetwork(secondaryNetConfig).
+		RedefineDefaultCMD(defaultCMD)
+
+	By("Creating FRR container")
+
+	frrContainer := pod.NewContainerBuilder(
+		tsparams.FRRSecondContainerName, NetConfig.CnfNetTestContainer, tsparams.SleepCMD).
+		WithSecurityCapabilities([]string{"NET_ADMIN", "NET_RAW", "SYS_ADMIN"}, true)
+
+	frrCtr, err := frrContainer.GetContainerCfg()
+	Expect(err).ToNot(HaveOccurred(), "Failed to get container configuration")
+	frrPod.WithAdditionalContainer(frrCtr).WithLocalVolume(configmapName, "/etc/frr")
+
 	By("Creating FRR pod in the test namespace")
 
-	frrPod, err := frrPod.WithPrivilegedFlag().CreateAndWaitUntilRunning(time.Minute)
+	frrPod, err = frrPod.WithPrivilegedFlag().CreateAndWaitUntilRunning(5 * time.Minute)
 	Expect(err).ToNot(HaveOccurred(), "Failed to create FRR test pod")
 
 	return frrPod