Skip to content

Commit

Permalink
WIP: e2e test for import vrfs
Browse files Browse the repository at this point in the history
Signed-off-by: Federico Paolinelli <[email protected]>
  • Loading branch information
fedepaol committed Jun 26, 2024
1 parent f0635db commit 783b7e9
Show file tree
Hide file tree
Showing 4 changed files with 308 additions and 9 deletions.
30 changes: 30 additions & 0 deletions e2etests/pkg/address/address.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package address

import (
"net"

"go.universe.tf/e2etest/pkg/ipfamily"
)

func FilterForFamily(ips []string, family ipfamily.Family) []string {
if family == ipfamily.DualStack {
return ips
}
var res []string
for _, ip := range ips {
parsedIP, _, _ := net.ParseCIDR(ip)
if parsedIP == nil {
panic("invalid ip") // it's a test after all, should never happen
}
isV4 := (parsedIP.To4() != nil)
if family == ipfamily.IPv4 && isV4 {
res = append(res, ip)
continue
}
if family == ipfamily.IPv6 && !isV4 {
res = append(res, ip)
continue
}
}
return res
}
7 changes: 3 additions & 4 deletions e2etests/pkg/routes/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,19 @@ import (

// PodHasPrefixFromContainer tells if the given frr-k8s pod has recevied a route for
// the given prefix from the given container.
func PodHasPrefixFromContainer(pod *v1.Pod, frr frrcontainer.FRR, prefix string) bool {
func PodHasPrefixFromContainer(pod *v1.Pod, frr frrcontainer.FRR, vrf, prefix string) bool {
_, cidr, _ := net.ParseCIDR(prefix)
ipFamily := ipfamily.ForCIDR(cidr)
nextHop := frr.Ipv4
if ipFamily == ipfamily.IPv6 {
nextHop = frr.Ipv6
}
vrf := frr.RouterConfig.VRF
return hasPrefix(pod, ipFamily, cidr, nextHop, vrf)
}

// CheckNeighborHasPrefix tells if the given frr container has a route toward the given prefix
// via the set of node passed to this function.
func CheckNeighborHasPrefix(neighbor frrcontainer.FRR, prefix string, nodes []v1.Node) error {
func CheckNeighborHasPrefix(neighbor frrcontainer.FRR, vrf, prefix string, nodes []v1.Node) error {
routesV4, routesV6, err := frr.Routes(neighbor)
if err != nil {
return err
Expand All @@ -47,7 +46,7 @@ func CheckNeighborHasPrefix(neighbor frrcontainer.FRR, prefix string, nodes []v1
}

cidrFamily := ipfamily.ForCIDR(cidr)
err = frr.RoutesMatchNodes(nodes, route, cidrFamily, neighbor.RouterConfig.VRF)
err = frr.RoutesMatchNodes(nodes, route, cidrFamily, vrf)
if err != nil {
return nil
}
Expand Down
257 changes: 257 additions & 0 deletions e2etests/tests/import_vrf.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
// SPDX-License-Identifier:Apache-2.0

package tests

import (
"github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.universe.tf/e2etest/pkg/frr/container"
frrcontainer "go.universe.tf/e2etest/pkg/frr/container"

frrk8sv1beta1 "github.com/metallb/frr-k8s/api/v1beta1"
"github.com/metallb/frrk8stests/pkg/address"
"github.com/metallb/frrk8stests/pkg/config"
"github.com/metallb/frrk8stests/pkg/dump"
"github.com/metallb/frrk8stests/pkg/infra"
"github.com/metallb/frrk8stests/pkg/k8s"
"github.com/metallb/frrk8stests/pkg/k8sclient"
frrconfig "go.universe.tf/e2etest/pkg/frr/config"
"go.universe.tf/e2etest/pkg/ipfamily"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
)

var _ = ginkgo.Describe("Advertising and Receiving routes", func() {
var cs clientset.Interface

defer ginkgo.GinkgoRecover()
updater, err := config.NewUpdater()
Expect(err).NotTo(HaveOccurred())
reporter := dump.NewK8sReporter(k8s.FRRK8sNamespace)

ginkgo.AfterEach(func() {
if ginkgo.CurrentSpecReport().Failed() {
testName := ginkgo.CurrentSpecReport().LeafNodeText
dump.K8sInfo(testName, reporter)
dump.BGPInfo(testName, infra.FRRContainers, cs)
}
})

ginkgo.BeforeEach(func() {
ginkgo.By("Clearing any previous configuration")

for _, c := range infra.FRRContainers {
err := c.UpdateBGPConfigFile(frrconfig.Empty)
Expect(err).NotTo(HaveOccurred())
}
err := updater.Clean()
Expect(err).NotTo(HaveOccurred())

cs = k8sclient.New()
})

initNeighbors := func(useVrf bool, ipFamily ipfamily.Family) ([]*frrcontainer.FRR, config.PeersConfig, []frrk8sv1beta1.Neighbor) {
frrs := config.ContainersForVRF(infra.FRRContainers, "")
if useVrf {
frrs = config.ContainersForVRF(infra.FRRContainers, infra.VRFName)
}
peersConfig := config.PeersForContainers(frrs, ipFamily)
neighbors := config.NeighborsFromPeers(peersConfig.PeersV4, peersConfig.PeersV6)
return frrs, peersConfig, neighbors
}

pairWithNodes := func(frrs []*frrcontainer.FRR, family ipfamily.Family, toAdvertise []string) {
for _, c := range frrs {
err := container.PairWithNodes(cs, c, family, func(frr *container.FRR) {
frr.NeighborConfig.ToAdvertiseV4 = address.FilterForFamily(toAdvertise, ipfamily.IPv4)
frr.NeighborConfig.ToAdvertiseV6 = address.FilterForFamily(toAdvertise, ipfamily.IPv6)
})
Expect(err).NotTo(HaveOccurred())
}
}

updateAndCheckPeered := func(config frrk8sv1beta1.FRRConfiguration, peersDefault, peersVRF config.PeersConfig, frrsDefault, frrsVRF []*frrcontainer.FRR, family ipfamily.Family) {
err := updater.Update(peersDefault.Secrets, config)
Expect(err).NotTo(HaveOccurred())

err = updater.Update(peersVRF.Secrets, config)
Expect(err).NotTo(HaveOccurred())

nodes, err := k8s.Nodes(cs)
Expect(err).NotTo(HaveOccurred())

for _, c := range frrsDefault {
ValidateFRRPeeredWithNodes(nodes, c, family)
}
for _, c := range frrsVRF {
ValidateFRRPeeredWithNodes(nodes, c, family)
}
}

baseConfig := frrk8sv1beta1.FRRConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: k8s.FRRK8sNamespace,
},
Spec: frrk8sv1beta1.FRRConfigurationSpec{
BGP: frrk8sv1beta1.BGPConfig{
Routers: []frrk8sv1beta1.Router{
{
ASN: infra.FRRK8sASN,
},
{
ASN: infra.FRRK8sASNVRF,
VRF: infra.VRFName,
},
},
},
},
}

ginkgo.Context("Receiving IPs", func() {
type params struct {
ipFamily ipfamily.Family
toAdvertise []string // advertised from outside
toAdvertiseVRF []string
}

ginkgo.DescribeTable("When the default VRF imports a vrf", func(p params) {

frrsDefault, peersDefault, neighborsDefault := initNeighbors(false, p.ipFamily)
frrsVRF, peersVRF, neighborsVRF := initNeighbors(true, p.ipFamily)

ginkgo.By("pairing with nodes")
pairWithNodes(frrsDefault, p.ipFamily, p.toAdvertise)
pairWithNodes(frrsVRF, p.ipFamily, p.toAdvertiseVRF)

config := baseConfig.DeepCopy()
config.Spec.BGP.Routers[0].Neighbors = neighborsDefault
config.Spec.BGP.Routers[0].Imports = []frrk8sv1beta1.Import{{VRF: infra.VRFName}}
config.Spec.BGP.Routers[1].Neighbors = neighborsVRF

for i := range config.Spec.BGP.Routers[0].Neighbors {
config.Spec.BGP.Routers[0].Neighbors[i].ToReceive.Allowed.Mode = frrk8sv1beta1.AllowAll
}
for i := range config.Spec.BGP.Routers[1].Neighbors {
config.Spec.BGP.Routers[0].Neighbors[i].ToReceive.Allowed.Mode = frrk8sv1beta1.AllowAll
}
updateAndCheckPeered(*config, peersDefault, peersVRF, frrsDefault, frrsVRF, p.ipFamily)

ginkgo.By("validating")
pods, err := k8s.FRRK8sPods(cs)
Expect(err).NotTo(HaveOccurred())

for _, frr := range frrsDefault {
ValidateNodesHaveRoutes(pods, *frr, p.toAdvertise...)
}
for _, frr := range frrsVRF {
// The routes advertised from the peers through red must appear in the default VRF too
ValidateNodesHaveRoutesVRF(pods, *frr, "default", p.toAdvertiseVRF...)
ValidateNodesHaveRoutes(pods, *frr, p.toAdvertiseVRF...)
}
},
ginkgo.Entry("IPV4 - default imports red", params{
ipFamily: ipfamily.IPv4,
toAdvertise: []string{"192.168.2.0/24"},
toAdvertiseVRF: []string{"192.169.2.0/24"},
}),
)

ginkgo.DescribeTable("When the red VRF imports the default VRF", func(p params) {

frrsDefault, peersDefault, neighborsDefault := initNeighbors(false, p.ipFamily)
frrsVRF, peersVRF, neighborsVRF := initNeighbors(true, p.ipFamily)

ginkgo.By("pairing with nodes")
pairWithNodes(frrsDefault, p.ipFamily, p.toAdvertise)
pairWithNodes(frrsVRF, p.ipFamily, p.toAdvertiseVRF)

config := baseConfig.DeepCopy()
config.Spec.BGP.Routers[0].Neighbors = neighborsDefault
config.Spec.BGP.Routers[1].Neighbors = neighborsVRF
config.Spec.BGP.Routers[1].Imports = []frrk8sv1beta1.Import{{VRF: "default"}}
for i := range config.Spec.BGP.Routers[0].Neighbors {
config.Spec.BGP.Routers[0].Neighbors[i].ToReceive.Allowed.Mode = frrk8sv1beta1.AllowAll
}
for i := range config.Spec.BGP.Routers[1].Neighbors {
config.Spec.BGP.Routers[0].Neighbors[i].ToReceive.Allowed.Mode = frrk8sv1beta1.AllowAll
}

updateAndCheckPeered(*config, peersDefault, peersVRF, frrsDefault, frrsVRF, p.ipFamily)

ginkgo.By("validating")
pods, err := k8s.FRRK8sPods(cs)
Expect(err).NotTo(HaveOccurred())

for _, frr := range frrsDefault {
ValidateNodesHaveRoutesVRF(pods, *frr, infra.VRFName, p.toAdvertise...)
ValidateNodesHaveRoutes(pods, *frr, p.toAdvertise...)
}
for _, frr := range frrsVRF {
ValidateNodesHaveRoutes(pods, *frr, p.toAdvertiseVRF...)
}
},
ginkgo.Entry("IPV4 - default imports red", params{
ipFamily: ipfamily.IPv4,
toAdvertise: []string{"192.168.2.0/24"},
toAdvertiseVRF: []string{"192.169.2.0/24"},
}),
)

})

ginkgo.Context("Advertising IPs", func() {
type params struct {
ipFamily ipfamily.Family
toAdvertise []string // advertised from outside
toAdvertiseVRF []string
}

ginkgo.DescribeTable("When the default VRF imports a vrf", func(p params) {
frrsDefault, peersDefault, neighborsDefault := initNeighbors(false, p.ipFamily)
frrsVRF, peersVRF, neighborsVRF := initNeighbors(true, p.ipFamily)

ginkgo.By("pairing with nodes")
pairWithNodes(frrsDefault, p.ipFamily, p.toAdvertise)
pairWithNodes(frrsVRF, p.ipFamily, p.toAdvertiseVRF)

config := baseConfig.DeepCopy()
config.Spec.BGP.Routers[0].Neighbors = neighborsDefault
config.Spec.BGP.Routers[0].Prefixes = p.toAdvertise
config.Spec.BGP.Routers[0].Imports = []frrk8sv1beta1.Import{{VRF: infra.VRFName}}
config.Spec.BGP.Routers[1].Neighbors = neighborsVRF
config.Spec.BGP.Routers[1].Prefixes = p.toAdvertiseVRF

for i := range config.Spec.BGP.Routers[0].Neighbors {
config.Spec.BGP.Routers[0].Neighbors[i].ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowRestricted
config.Spec.BGP.Routers[0].Neighbors[i].ToAdvertise.Allowed.Prefixes = append(p.toAdvertise, p.toAdvertiseVRF...)
}
for i := range config.Spec.BGP.Routers[1].Neighbors {
config.Spec.BGP.Routers[1].Neighbors[i].ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowRestricted
config.Spec.BGP.Routers[1].Neighbors[i].ToAdvertise.Allowed.Prefixes = p.toAdvertiseVRF
}

updateAndCheckPeered(*config, peersDefault, peersVRF, frrsDefault, frrsVRF, p.ipFamily)

ginkgo.By("validating")

nodes, err := k8s.Nodes(cs)
Expect(err).NotTo(HaveOccurred())

for _, frr := range frrsDefault {
ValidatePrefixesForNeighbor(*frr, nodes, p.toAdvertise...)
ValidatePrefixesForNeighborVRF(*frr, nodes, infra.VRFName, p.toAdvertiseVRF...)
}
for _, frr := range frrsVRF {
ValidatePrefixesForNeighbor(*frr, nodes, p.toAdvertiseVRF...)
}
},
ginkgo.Entry("IPV4 - default imports red", params{
ipFamily: ipfamily.IPv4,
toAdvertise: []string{"192.168.2.0/24"},
toAdvertiseVRF: []string{"192.169.2.0/24"},
}),
)
})

})
23 changes: 18 additions & 5 deletions e2etests/tests/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,15 @@ func ValidateFRRPeeredWithNodes(nodes []corev1.Node, c *frrcontainer.FRR, ipFami
}

func ValidatePrefixesForNeighbor(neigh frrcontainer.FRR, nodes []v1.Node, prefixes ...string) {
ValidatePrefixesForNeighborVRF(neigh, nodes, neigh.RouterConfig.VRF, prefixes...)
}

// Validates the given neighbor has the prefixes towards the given VRF
func ValidatePrefixesForNeighborVRF(neigh frrcontainer.FRR, nodes []v1.Node, vrfName string, prefixes ...string) {
ginkgo.By(fmt.Sprintf("checking prefixes %v for %s", prefixes, neigh.Name))
Eventually(func() error {
for _, prefix := range prefixes {
err := routes.CheckNeighborHasPrefix(neigh, prefix, nodes)
err := routes.CheckNeighborHasPrefix(neigh, vrfName, prefix, nodes)
if err != nil {
return fmt.Errorf("Neigh %s does not have prefix %s: %w", neigh.Name, prefix, err)
}
Expand All @@ -57,10 +62,14 @@ func ValidatePrefixesForNeighbor(neigh frrcontainer.FRR, nodes []v1.Node, prefix
}

func ValidateNeighborNoPrefixes(neigh frrcontainer.FRR, nodes []v1.Node, prefixes ...string) {
ValidateNeighborNoPrefixesVRF(neigh, nodes, neigh.RouterConfig.VRF, prefixes...)
}

func ValidateNeighborNoPrefixesVRF(neigh frrcontainer.FRR, nodes []v1.Node, vrfName string, prefixes ...string) {
ginkgo.By(fmt.Sprintf("checking prefixes %v not announced to %s", prefixes, neigh.Name))
Eventually(func() error {
for _, prefix := range prefixes {
err := routes.CheckNeighborHasPrefix(neigh, prefix, nodes)
err := routes.CheckNeighborHasPrefix(neigh, vrfName, prefix, nodes)
if err != nil {
return fmt.Errorf("Neigh %s does not have prefix %s: %w", neigh.Name, prefix, err)
}
Expand Down Expand Up @@ -97,12 +106,12 @@ func ValidateNeighborCommunityPrefixes(neigh frrcontainer.FRR, community string,
}, 5*time.Second, time.Second).ShouldNot(HaveOccurred())
}

func ValidateNodesHaveRoutes(pods []*v1.Pod, neigh frrcontainer.FRR, prefixes ...string) {
func ValidateNodesHaveRoutesVRF(pods []*v1.Pod, neigh frrcontainer.FRR, vrf string, prefixes ...string) {
ginkgo.By(fmt.Sprintf("Checking routes %v from %s", prefixes, neigh.Name))
Eventually(func() error {
for _, prefix := range prefixes {
for _, pod := range pods {
if !routes.PodHasPrefixFromContainer(pod, neigh, prefix) {
if !routes.PodHasPrefixFromContainer(pod, neigh, vrf, prefix) {
return fmt.Errorf("pod %s does not have prefix %s from %s", pod.Name, prefix, neigh.Name)
}
}
Expand All @@ -111,12 +120,16 @@ func ValidateNodesHaveRoutes(pods []*v1.Pod, neigh frrcontainer.FRR, prefixes ..
}, time.Minute, time.Second).ShouldNot(HaveOccurred())
}

func ValidateNodesHaveRoutes(pods []*v1.Pod, neigh frrcontainer.FRR, prefixes ...string) {
ValidateNodesHaveRoutesVRF(pods, neigh, neigh.RouterConfig.VRF, prefixes...)
}

func ValidateNodesDoNotHaveRoutes(pods []*v1.Pod, neigh frrcontainer.FRR, prefixes ...string) {
ginkgo.By(fmt.Sprintf("Checking routes %v not injected from %s", prefixes, neigh.Name))
shouldPassConsistently(func() error {
for _, prefix := range prefixes {
for _, pod := range pods {
if routes.PodHasPrefixFromContainer(pod, neigh, prefix) {
if routes.PodHasPrefixFromContainer(pod, neigh, neigh.RouterConfig.VRF, prefix) {
return fmt.Errorf("pod %s has prefix %s from %s", pod.Name, prefix, neigh.Name)
}
}
Expand Down

0 comments on commit 783b7e9

Please sign in to comment.