Testing for proper handling of non-/32 routing

Signed-off-by: Ed Warnicke <[email protected]>

go.mod

@@ -25,4 +25,5 @@ require (
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
 	google.golang.org/grpc v1.38.0
+	google.golang.org/protobuf v1.26.0
 )

internal/tests/routes/server.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package routes provides a simple cahin element for adding routes
+package routes
+
+import (
+	"context"
+
+	"google.golang.org/protobuf/types/known/emptypb"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+)
+
+type testRoutesServer struct {
+	srcRoutes []*networkservice.Route
+	dstRoutes []*networkservice.Route
+}
+
+// NewServer - add routes to response
+func NewServer(srcRoutes, dstRoutes []*networkservice.Route) networkservice.NetworkServiceServer {
+	return &testRoutesServer{
+		srcRoutes: srcRoutes,
+		dstRoutes: dstRoutes,
+	}
+}
+
+func (r *testRoutesServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) {
+	if request.GetConnection() == nil {
+		request.Connection = &networkservice.Connection{}
+	}
+	if request.GetConnection().GetContext() == nil {
+		request.GetConnection().Context = &networkservice.ConnectionContext{}
+	}
+	if request.GetConnection().GetContext().GetIpContext() == nil {
+		request.GetConnection().GetContext().IpContext = &networkservice.IPContext{}
+	}
+	request.GetConnection().GetContext().GetIpContext().SrcRoutes = append(request.GetConnection().GetContext().GetIpContext().SrcRoutes, r.srcRoutes...)
+	request.GetConnection().GetContext().GetIpContext().DstRoutes = append(request.GetConnection().GetContext().GetIpContext().DstRoutes, r.dstRoutes...)
+	return next.Server(ctx).Request(ctx, request)
+}
+
+func (r *testRoutesServer) Close(ctx context.Context, conn *networkservice.Connection) (*emptypb.Empty, error) {
+	return next.Server(ctx).Close(ctx, conn)
+}

internal/tests/suite_combinatronics_test.go

@@ -20,6 +20,7 @@ package tests
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"net"
 	"net/url"
@@ -53,30 +54,59 @@ import (
 func (f *ForwarderTestSuite) TestCombinations() {
 	_, prefix1, err := net.ParseCIDR("10.0.0.0/24")
 	f.Require().NoError(err)
-	_, prefix2, err := net.ParseCIDR("fc00::/7")
+	_, prefix2, err := net.ParseCIDR("fc00::/64")
 	f.Require().NoError(err)
+	srcRoutes := []*networkservice.Route{}
+	dstRoutes := []*networkservice.Route{}
+	for i := 1; i < 6; i++ {
+		srcRoutes = append(srcRoutes,
+			&networkservice.Route{
+				Prefix: fmt.Sprintf("10.0.%d.0/24", i),
+			},
+			&networkservice.Route{
+				Prefix: fmt.Sprintf("fc00:0:%d::/64", i),
+			},
+		)
+		dstRoutes = append(dstRoutes,
+			&networkservice.Route{
+				Prefix: fmt.Sprintf("10.0.%d.1/24", i),
+			},
+			&networkservice.Route{
+				Prefix: fmt.Sprintf("fc00:0:%d::1/64", i),
+			},
+		)
+	}
+
 	endpoints := map[string]func(ctx context.Context) verifiableEndpoint{
 		kernel.MECHANISM: func(ctx context.Context) verifiableEndpoint {
 			return newKernelVerifiableEndpoint(ctx,
 				prefix1,
 				prefix2,
+				srcRoutes,
+				dstRoutes,
 				spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime),
 			)
 		},
 		memif.MECHANISM: func(ctx context.Context) verifiableEndpoint {
 			return newMemifVerifiableEndpoint(ctx, prefix1, prefix2,
+				srcRoutes,
+				dstRoutes,
 				spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime),
 				f.vppServerConn,
 			)
 		},
 		vxlan.MECHANISM: func(ctx context.Context) verifiableEndpoint {
 			return newVxlanVerifiableEndpoint(ctx, prefix1, prefix2,
+				srcRoutes,
+				dstRoutes,
 				spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime),
 				f.vppServerConn,
 			)
 		},
 		wireguard.MECHANISM: func(ctx context.Context) verifiableEndpoint {
 			return newWireguardVerifiableEndpoint(ctx, prefix1, prefix2,
+				srcRoutes,
+				dstRoutes,
 				spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime),
 				f.vppServerConn,
 			)

internal/tests/suite_kernel_test.go

@@ -31,6 +31,9 @@ import (
 	"github.com/vishvananda/netns"
 	"google.golang.org/grpc"
 
+	"github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel"
@@ -74,6 +77,7 @@ type kernelVerifiableEndpoint struct {
 
 func newKernelVerifiableEndpoint(ctx context.Context,
 	prefix1, prefix2 *net.IPNet,
+	srcRoutes, dstRoutes []*networkservice.Route,
 	tokenGenerator token.GeneratorFunc,
 ) verifiableEndpoint {
 	rootNSHandle, err := netns.Get()
@@ -108,6 +112,7 @@ func newKernelVerifiableEndpoint(ctx context.Context,
 			endpoint.WithAdditionalFunctionality(
 				point2pointipam.NewServer(prefix1),
 				point2pointipam.NewServer(prefix2),
+				routes.NewServer(srcRoutes, dstRoutes),
 				mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
 					kernel.MECHANISM: chain.NewNetworkServiceServer(
 						kernelmechanism.NewServer(kernelmechanism.WithInterfaceName(endpointNSName)),
@@ -134,6 +139,9 @@ func (k *kernelVerifiableEndpoint) VerifyConnection(conn *networkservice.Connect
 	if err := checkKernelInterface(namingConn, conn.GetContext().GetIpContext().GetDstIPNets(), k.endpointNSHandle); err != nil {
 		return err
 	}
+	if err := checkRoutes(namingConn, k.endpointNSHandle, metadata.IsClient(k)); err != nil {
+		return err
+	}
 	for _, ip := range conn.GetContext().GetIpContext().GetSrcIPNets() {
 		if err := pingKernel(ip, k.endpointNSHandle); err != nil {
 			return err
@@ -195,6 +203,9 @@ func (k *kernelVerifiableClient) VerifyConnection(conn *networkservice.Connectio
 	if err := checkKernelInterface(conn, conn.GetContext().GetIpContext().GetSrcIPNets(), k.clientNSHandle); err != nil {
 		return err
 	}
+	if err := checkRoutes(conn, k.clientNSHandle, metadata.IsClient(k)); err != nil {
+		return err
+	}
 	for _, ip := range conn.GetContext().GetIpContext().GetDstIPNets() {
 		if err := pingKernel(ip, k.clientNSHandle); err != nil {
 			return err
@@ -277,3 +288,61 @@ func pingKernel(ipnet *net.IPNet, handle netns.NsHandle) error {
 	}
 	return nil
 }
+
+func checkRoutes(conn *networkservice.Connection, nsHandle netns.NsHandle, isClient bool) error {
+	if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil {
+		curNetNS, err := netns.Get()
+		if err != nil {
+			return errors.Wrap(err, "unable to get current netns")
+		}
+		netlinkHandle, err := netlink.NewHandleAtFrom(nsHandle, curNetNS)
+		if err != nil {
+			return errors.Wrap(err, "unable to get netlink Handle in target netNs")
+		}
+		kernelRoutes, err := netlinkHandle.RouteList(nil, netlink.FAMILY_ALL)
+		if err != nil {
+			return errors.Wrap(err, "unable to get routes")
+		}
+		nsmRoutes := conn.GetContext().GetIpContext().GetDstRoutesWithExplicitNextHop()
+		if isClient {
+			nsmRoutes = conn.GetContext().GetIpContext().GetSrcRoutesWithExplicitNextHop()
+		}
+
+		link, err := netlinkHandle.LinkByName((mechanism.GetInterfaceName()))
+		if err != nil {
+			return errors.Wrapf(err, "unable to find link %s", mechanism.GetInterfaceName())
+		}
+
+		for _, nsmRoute := range nsmRoutes {
+			routeFound := false
+			for i := range kernelRoutes {
+				if !(kernelRoutes[i].Dst.IP.Equal(nsmRoute.GetPrefixIPNet().IP.Mask(nsmRoute.GetPrefixIPNet().Mask))) {
+					continue
+				}
+				kernelOnes, kernelBit := kernelRoutes[i].Dst.Mask.Size()
+				nsmOnes, nsmBits := nsmRoute.GetPrefixIPNet().Mask.Size()
+				if kernelOnes != nsmOnes {
+					continue
+				}
+				if kernelBit != nsmBits {
+					continue
+				}
+
+				if !kernelRoutes[i].Gw.Equal(nsmRoute.GetNextHopIP()) {
+					continue
+				}
+
+				if kernelRoutes[i].LinkIndex != link.Attrs().Index {
+					continue
+				}
+
+				routeFound = true
+				break
+			}
+			if !routeFound {
+				return errors.Errorf("unable to find expected route: %s in routes %+v", nsmRoute, kernelRoutes)
+			}
+		}
+	}
+	return nil
+}

internal/tests/suite_memif_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/edwarnicke/vpphelper"
 	"github.com/pkg/errors"
 
+	"github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/sendfd"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
@@ -61,6 +62,7 @@ type memifVerifiableEndpoint struct {
 
 func newMemifVerifiableEndpoint(ctx context.Context,
 	prefix1, prefix2 *net.IPNet,
+	srcRoutes, dstRoutes []*networkservice.Route,
 	tokenGenerator token.GeneratorFunc,
 	vppConn vpphelper.Connection,
 ) verifiableEndpoint {
@@ -76,6 +78,7 @@ func newMemifVerifiableEndpoint(ctx context.Context,
 				sendfd.NewServer(),
 				point2pointipam.NewServer(prefix1),
 				point2pointipam.NewServer(prefix2),
+				routes.NewServer(srcRoutes, dstRoutes),
 				mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
 					memif.MECHANISM: chain.NewNetworkServiceServer(
 						metadata.NewServer(),

internal/tests/suite_util_test.go

@@ -32,7 +32,7 @@ import (
 )
 
 const (
-	contextTimeout = 100 * time.Second
+	contextTimeout = 10000 * time.Second
 	forwarderIP    = "10.0.2.1"
 	clientIP       = "10.0.2.2"
 	serverIP       = "10.0.2.3"

internal/tests/suite_vxlan_test.go

@@ -23,6 +23,7 @@ import (
 	"git.fd.io/govpp.git/api"
 	"google.golang.org/grpc"
 
+	"github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes"
 	"github.com/networkservicemesh/sdk-vpp/pkg/networkservice/pinhole"
 
 	"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms"
@@ -48,6 +49,7 @@ type vxlanVerifiableEndpoint struct {
 
 func newVxlanVerifiableEndpoint(ctx context.Context,
 	prefix1, prefix2 *net.IPNet,
+	srcRoutes, dstRoutes []*networkservice.Route,
 	tokenGenerator token.GeneratorFunc,
 	vppConn api.Connection) verifiableEndpoint {
 	rv := &vxlanVerifiableEndpoint{
@@ -63,6 +65,7 @@ func newVxlanVerifiableEndpoint(ctx context.Context,
 			metadata.NewServer(),
 			point2pointipam.NewServer(prefix1),
 			point2pointipam.NewServer(prefix2),
+			routes.NewServer(srcRoutes, dstRoutes),
 			connectioncontext.NewServer(vppConn),
 			pinhole.NewServer(vppConn),
 			mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{

internal/tests/suite_wireguard_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/edwarnicke/vpphelper"
 	"google.golang.org/grpc"
 
+	"github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/chains/client"
 
 	"github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize"
@@ -49,6 +50,7 @@ type wireguardVerifiableEndpoint struct {
 
 func newWireguardVerifiableEndpoint(ctx context.Context,
 	prefix1, prefix2 *net.IPNet,
+	srcRoutes, dstRoutes []*networkservice.Route,
 	tokenGenerator token.GeneratorFunc,
 	vppConn vpphelper.Connection) verifiableEndpoint {
 	rv := &vxlanVerifiableEndpoint{
@@ -64,6 +66,7 @@ func newWireguardVerifiableEndpoint(ctx context.Context,
 			metadata.NewServer(),
 			point2pointipam.NewServer(prefix1),
 			point2pointipam.NewServer(prefix2),
+			routes.NewServer(srcRoutes, dstRoutes),
 			up.NewServer(ctx, vppConn),
 			pinhole.NewServer(vppConn),
 			connectioncontext.NewServer(vppConn),