Skip to content

Commit

Permalink
cnf network: add bgp connect time test cases (#219)
Browse files Browse the repository at this point in the history
Co-authored-by: Gregory Kopels <[email protected]>
  • Loading branch information
gkopels and Gregory Kopels authored Sep 29, 2024
1 parent 5f2eed8 commit a913734
Show file tree
Hide file tree
Showing 5 changed files with 311 additions and 6 deletions.
85 changes: 85 additions & 0 deletions tests/cnf/core/network/metallb/internal/frr/frr.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,54 @@ type (
Bestpath bool `json:"bestpath,omitempty"`
} `json:"routes"`
}

advertisedRoute struct {
AddrPrefix string `json:"addrPrefix"`
PrefixLen int `json:"prefixLen"`
Network string `json:"network"`
NextHop string `json:"nextHop"`
Metric int `json:"metric"`
LocPrf int `json:"locPrf"`
Weight int `json:"weight"`
Path string `json:"path"`
BGPOriginCode string `json:"bgpOriginCode"`
}
// BgpAdvertisedRoutes creates struct from json output.
BgpAdvertisedRoutes struct {
BGPTableVersion int `json:"bgpTableVersion"`
BGPLocalRouterID string `json:"bgpLocalRouterId"`
DefaultLocPrf int `json:"defaultLocPrf"`
LocalAS int `json:"localAS"`
AdvertisedRoutes map[string]advertisedRoute `json:"advertisedRoutes"`
}
// BgpReceivedRoutes struct includes Routes map.
BgpReceivedRoutes struct {
Routes map[string][]RouteInfo `json:"routes"`
}
// RouteInfo struct includes route info.
RouteInfo struct {
Prefix string `json:"prefix"`
PrefixLen int `json:"prefixLen"`
Protocol string `json:"protocol"`
Metric int `json:"metric"`
Uptime string `json:"uptime"`
Nexthops []Nexthop `json:"nexthops"`
}
// Nexthop struct includes nexthop route info.
Nexthop struct {
Flags int `json:"flags"`
Fib bool `json:"fib"`
IP string `json:"ip"`
Afi string `json:"afi"`
InterfaceIndex int `json:"interfaceIndex"`
InterfaceName string `json:"interfaceName"`
Active bool `json:"active"`
Weight int `json:"weight"`
}
// BGPConnectionInfo struct includes the connectRetryTimer.
BGPConnectionInfo struct {
ConnectRetryTimer int `json:"connectRetryTimer"`
}
)

// DefineBaseConfig defines minimal required FRR configuration.
Expand Down Expand Up @@ -230,3 +278,40 @@ func runningConfig(frrPod *pod.Builder) (string, error) {

return bgpStateOut.String(), nil
}

// FetchBGPConnectTimeValue fetches and returns the ConnectRetryTimer value for the specified BGP peer.
func FetchBGPConnectTimeValue(frrk8sPods []*pod.Builder, bgpPeerIP string) (int, error) {
for _, frrk8sPod := range frrk8sPods {
// Run the "show bgp neighbor <bgpPeerIP> json" command on each pod
output, err := frrk8sPod.ExecCommand(append(netparam.VtySh,
fmt.Sprintf("show bgp neighbor %s json", bgpPeerIP)), "frr")
if err != nil {
return 0, fmt.Errorf("error collecting BGP neighbor info from pod %s: %w",
frrk8sPod.Definition.Name, err)
}

// Parsing JSON
var bgpData map[string]BGPConnectionInfo
err = json.Unmarshal(output.Bytes(), &bgpData)

if err != nil {
return 0, fmt.Errorf("error parsing BGP neighbor JSON for pod %s: %w", frrk8sPod.Definition.Name, err)
}

// Extracting ConnectRetryTimer from the parsed JSON
for _, bgpInfo := range bgpData {
return bgpInfo.ConnectRetryTimer, nil
}
}

return 0, fmt.Errorf("no BGP neighbor data found for peer %s", bgpPeerIP)
}

// ResetBGPConnection restarts the TCP connection.
func ResetBGPConnection(frrPod *pod.Builder) error {
glog.V(90).Infof("Resetting BGP session to all neighbors: %s", frrPod.Definition.Name)

_, err := frrPod.ExecCommand(append(netparam.VtySh, "clear ip bgp *"))

return err
}
5 changes: 3 additions & 2 deletions tests/cnf/core/network/metallb/tests/bfd-test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa
bfdProfile := createBFDProfileAndVerifyIfItsReady(frrk8sPods)

createBGPPeerAndVerifyIfItsReady(
ipv4metalLbIPList[0], bfdProfile.Definition.Name, tsparams.RemoteBGPASN, false, frrk8sPods)
ipv4metalLbIPList[0], bfdProfile.Definition.Name, tsparams.RemoteBGPASN, false, 0,
frrk8sPods)

By("Creating MetalLb configMap")
bfdConfigMap := createConfigMap(tsparams.RemoteBGPASN, ipv4NodeAddrList, false, true)
Expand Down Expand Up @@ -239,7 +240,7 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa
eBgpMultiHop = true
}
createBGPPeerAndVerifyIfItsReady(
masterClientPodIP, bfdProfile.Definition.Name, neighbourASN, eBgpMultiHop, frrk8sPods)
masterClientPodIP, bfdProfile.Definition.Name, neighbourASN, eBgpMultiHop, 0, frrk8sPods)

prefixLen := int32(32)
if ipStack == netparam.IPV6Family {
Expand Down
210 changes: 210 additions & 0 deletions tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
package tests

import (
"fmt"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/openshift-kni/eco-goinfra/pkg/configmap"
"github.com/openshift-kni/eco-goinfra/pkg/metallb"
"github.com/openshift-kni/eco-goinfra/pkg/nad"
"github.com/openshift-kni/eco-goinfra/pkg/namespace"
"github.com/openshift-kni/eco-goinfra/pkg/nodes"
"github.com/openshift-kni/eco-goinfra/pkg/pod"
"github.com/openshift-kni/eco-goinfra/pkg/reportxml"
"github.com/openshift-kni/eco-goinfra/pkg/service"
. "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools"
"github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/frr"
"github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/metallbenv"
"github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/tsparams"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)

var _ = Describe("FRR", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFailure, func() {
var frrk8sPods []*pod.Builder

BeforeAll(func() {
var (
err error
)

By("Getting MetalLb load balancer ip addresses")
ipv4metalLbIPList, ipv6metalLbIPList, err = metallbenv.GetMetalLbIPByIPStack()
Expect(err).ToNot(HaveOccurred(), "An error occurred while "+
"determining the IP addresses from ECO_CNF_CORE_NET_MLB_ADDR_LIST environment variable.")

By("Getting external nodes ip addresses")
cnfWorkerNodeList, err = nodes.List(APIClient,
metav1.ListOptions{LabelSelector: labels.Set(NetConfig.WorkerLabelMap).String()})
Expect(err).ToNot(HaveOccurred(), "Failed to discover worker nodes")

By("Selecting worker node for BGP tests")
workerLabelMap, workerNodeList = setWorkerNodeListAndLabelForBfdTests(cnfWorkerNodeList, metalLbTestsLabel)
ipv4NodeAddrList, err = nodes.ListExternalIPv4Networks(
APIClient, metav1.ListOptions{LabelSelector: labels.Set(workerLabelMap).String()})
Expect(err).ToNot(HaveOccurred(), "Failed to collect external nodes ip addresses")

err = metallbenv.IsEnvVarMetalLbIPinNodeExtNetRange(ipv4NodeAddrList, ipv4metalLbIPList, nil)
Expect(err).ToNot(HaveOccurred(), "Failed to validate metalLb exported ip address")

By("Listing master nodes")
masterNodeList, err = nodes.List(APIClient,
metav1.ListOptions{LabelSelector: labels.Set(NetConfig.ControlPlaneLabelMap).String()})
Expect(err).ToNot(HaveOccurred(), "Fail to list master nodes")
Expect(len(masterNodeList)).To(BeNumerically(">", 0),
"Failed to detect master nodes")
})

BeforeEach(func() {
By("Creating a new instance of MetalLB Speakers on workers")
err := metallbenv.CreateNewMetalLbDaemonSetAndWaitUntilItsRunning(tsparams.DefaultTimeout, workerLabelMap)
Expect(err).ToNot(HaveOccurred(), "Failed to recreate metalLb daemonset")

By("Collecting information before test")
frrk8sPods, err = pod.List(APIClient, NetConfig.MlbOperatorNamespace, metav1.ListOptions{
LabelSelector: tsparams.FRRK8sDefaultLabel,
})
Expect(err).ToNot(HaveOccurred(), "Failed to list frr pods")
})

AfterEach(func() {
By("Cleaning MetalLb operator namespace")
metalLbNs, err := namespace.Pull(APIClient, NetConfig.MlbOperatorNamespace)
Expect(err).ToNot(HaveOccurred(), "Failed to pull metalLb operator namespace")
err = metalLbNs.CleanObjects(
tsparams.DefaultTimeout,
metallb.GetBGPPeerGVR(),
metallb.GetBFDProfileGVR(),
metallb.GetBGPPeerGVR(),
metallb.GetBGPAdvertisementGVR(),
metallb.GetIPAddressPoolGVR(),
metallb.GetMetalLbIoGVR())
Expect(err).ToNot(HaveOccurred(), "Failed to remove object's from operator namespace")

By("Cleaning test namespace")
err = namespace.NewBuilder(APIClient, tsparams.TestNamespaceName).CleanObjects(
tsparams.DefaultTimeout,
pod.GetGVR(),
service.GetServiceGVR(),
configmap.GetGVR(),
nad.GetGVR())
Expect(err).ToNot(HaveOccurred(), "Failed to clean test namespace")
})

It("Verify configuration of a FRR node router peer with the connectTime less than the default of 120 seconds",
reportxml.ID("74414"), func() {

By("Creating BGP Peers with 10 second retry connect timer")
createBGPPeerAndVerifyIfItsReady(ipv4metalLbIPList[0], "", 64500, false,
10, frrk8sPods)

By("Validate BGP Peers with 10 second retry connect timer")
Eventually(func() int {
// Get the connect time configuration
connectTimeValue, err := frr.FetchBGPConnectTimeValue(frrk8sPods, ipv4metalLbIPList[0])
Expect(err).ToNot(HaveOccurred(), "Failed to verify BGP connect time")

// Return the integer value of ConnectRetryTimer for assertion
return connectTimeValue
}, 60*time.Second, 5*time.Second).Should(Equal(10),
"Failed to fetch BGP connect time")
})

It("Verify the retry timers reconnects to a neighbor with a timer connect less then 10s after a BGP tcp reset",
reportxml.ID("74416"), func() {

By("Create an external FRR Pod")
frrPod := createAndDeployFRRPod()

By("Creating BGP Peers with 10 second retry connect timer")
createBGPPeerAndVerifyIfItsReady(ipv4metalLbIPList[0], "", 64500, false,
10, frrk8sPods)

By("Validate BGP Peers with 10 second retry connect timer")
Eventually(func() int {
// Get the connect time configuration
connectTimeValue, err := frr.FetchBGPConnectTimeValue(frrk8sPods, ipv4metalLbIPList[0])
Expect(err).ToNot(HaveOccurred(), "Failed to verify BGP connect time")

// Return the integer value of ConnectRetryTimer for assertion
return connectTimeValue
}, 60*time.Second, 5*time.Second).Should(Equal(10),
"Failed to fetch BGP connect time")

By("Reset the BGP session ")
err := frr.ResetBGPConnection(frrPod)
Expect(err).ToNot(HaveOccurred(), "Failed to reset BGP connection")

By("Verify that BGP session is re-established and up in less then 10 seconds")
verifyMaxReConnectTime(frrPod, removePrefixFromIPList(ipv4NodeAddrList), time.Second*10)
})

It("Update the timer to less then the default on an existing BGP connection",
reportxml.ID("74417"), func() {

By("Creating BGP Peers")
createBGPPeerAndVerifyIfItsReady(ipv4metalLbIPList[0], "",
64500, false, 0, frrk8sPods)

By("Validate BGP Peers with the default retry connect timer")
Eventually(func() int {
// Get the connect time configuration
connectTimeValue, err := frr.FetchBGPConnectTimeValue(frrk8sPods, ipv4metalLbIPList[0])
Expect(err).ToNot(HaveOccurred(), "Failed to verify BGP connect time")

// Return the integer value of ConnectRetryTimer for assertion
return connectTimeValue
}, 60*time.Second, 5*time.Second).Should(Equal(120),
"Failed to fetch BGP connect time")

By("Update the BGP Peers connect timer to 10 seconds")
bgpPeer, err := metallb.PullBGPPeer(APIClient, "testpeer", NetConfig.MlbOperatorNamespace)
Expect(err).ToNot(HaveOccurred(), "Failed to find bgp peer")

_, err = bgpPeer.WithConnectTime(metav1.Duration{Duration: 10 * time.Second}).Update(true)
Expect(err).ToNot(HaveOccurred(), "Failed to update the bgp peer with a 10s connect timer")

By("Validate BGP Peers with the default retry connect timer")
Eventually(func() int {
// Get the connect time configuration
connectTimeValue, err := frr.FetchBGPConnectTimeValue(frrk8sPods, ipv4metalLbIPList[0])
Expect(err).ToNot(HaveOccurred(), "Failed to verify BGP connect time")

// Return the integer value of ConnectRetryTimer for assertion
return connectTimeValue
}, 60*time.Second, 5*time.Second).Should(Equal(10),
"Failed to fetch BGP connect time")
})
})

func createAndDeployFRRPod() *pod.Builder {
By("Creating External NAD")
createExternalNad()

By("Creating static ip annotation")

staticIPAnnotation := pod.StaticIPAnnotation(
externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")})

By("Creating MetalLb configMap")

masterConfigMap := createConfigMap(tsparams.LocalBGPASN, ipv4NodeAddrList, false, false)

By("Creating FRR Pod")

frrPod := createFrrPod(
masterNodeList[0].Object.Name, masterConfigMap.Definition.Name, []string{}, staticIPAnnotation)

return frrPod
}

func verifyMaxReConnectTime(frrPod *pod.Builder, peerAddrList []string, maxConnectTime time.Duration) {
for _, peerAddress := range removePrefixFromIPList(peerAddrList) {
Eventually(frr.BGPNeighborshipHasState,
maxConnectTime, time.Second).
WithArguments(frrPod, peerAddress, "Established").Should(
BeTrue(), "Failed to receive BGP status UP")
}
}
8 changes: 5 additions & 3 deletions tests/cnf/core/network/metallb/tests/bgp-tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
Expect(len(frrk8sPods)).To(BeNumerically(">", 0),
"Failed the number of frr speaker pods is 0")
createBGPPeerAndVerifyIfItsReady(
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, frrk8sPods)
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, 0, frrk8sPods)
})

AfterEach(func() {
Expand Down Expand Up @@ -110,7 +110,8 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
}

createBGPPeerAndVerifyIfItsReady(
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, frrk8sPods)
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, 0,
frrk8sPods)

By("Setting test iteration parameters")
_, subMask, mlbAddressList, nodeAddrList, addressPool, _, err :=
Expand Down Expand Up @@ -172,7 +173,8 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa
masterNodeList[0].Object.Name, masterConfigMap.Definition.Name, []string{}, staticIPAnnotation)

createBGPPeerAndVerifyIfItsReady(
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, frrk8sPods)
ipv4metalLbIPList[0], "", tsparams.LocalBGPASN, false, 0,
frrk8sPods)

By("Checking that BGP session is established and up")
verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList))
Expand Down
9 changes: 8 additions & 1 deletion tests/cnf/core/network/metallb/tests/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/tsparams"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// test cases variables that are accessible across entire package.
Expand Down Expand Up @@ -102,7 +103,8 @@ func createExternalNad() {
}

func createBGPPeerAndVerifyIfItsReady(
peerIP, bfdProfileName string, remoteAsn uint32, eBgpMultiHop bool, frrk8sPods []*pod.Builder) {
peerIP, bfdProfileName string, remoteAsn uint32, eBgpMultiHop bool, connectTime int,
frrk8sPods []*pod.Builder) {
By("Creating BGP Peer")

bgpPeer := metallb.NewBPGPeerBuilder(APIClient, "testpeer", NetConfig.MlbOperatorNamespace,
Expand All @@ -112,6 +114,11 @@ func createBGPPeerAndVerifyIfItsReady(
bgpPeer.WithBFDProfile(bfdProfileName)
}

if connectTime != 0 {
// Convert connectTime int to time.Duration in seconds
bgpPeer.WithConnectTime(metav1.Duration{Duration: time.Duration(connectTime) * time.Second})
}

_, err := bgpPeer.Create()
Expect(err).ToNot(HaveOccurred(), "Failed to create BGP peer")

Expand Down

0 comments on commit a913734

Please sign in to comment.