diff --git a/go.mod b/go.mod index efe276c1..9bf841f0 100644 --- a/go.mod +++ b/go.mod @@ -25,4 +25,5 @@ require ( github.com/vishvananda/netlink v1.1.0 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae google.golang.org/grpc v1.38.0 + google.golang.org/protobuf v1.26.0 ) diff --git a/internal/imports/imports_linux.go b/internal/imports/imports_linux.go index 642455c1..f2e27f7c 100644 --- a/internal/imports/imports_linux.go +++ b/internal/imports/imports_linux.go @@ -92,6 +92,7 @@ import ( _ "google.golang.org/grpc" _ "google.golang.org/grpc/credentials" _ "google.golang.org/grpc/health/grpc_health_v1" + _ "google.golang.org/protobuf/types/known/emptypb" _ "io/ioutil" _ "net" _ "net/url" diff --git a/internal/tests/routes/server.go b/internal/tests/routes/server.go new file mode 100644 index 00000000..31490e0f --- /dev/null +++ b/internal/tests/routes/server.go @@ -0,0 +1,59 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package routes provides a simple cahin element for adding routes +package routes + +import ( + "context" + + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" +) + +type testRoutesServer struct { + srcRoutes []*networkservice.Route + dstRoutes []*networkservice.Route +} + +// NewServer - add routes to response +func NewServer(srcRoutes, dstRoutes []*networkservice.Route) networkservice.NetworkServiceServer { + return &testRoutesServer{ + srcRoutes: srcRoutes, + dstRoutes: dstRoutes, + } +} + +func (r *testRoutesServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + if request.GetConnection() == nil { + request.Connection = &networkservice.Connection{} + } + if request.GetConnection().GetContext() == nil { + request.GetConnection().Context = &networkservice.ConnectionContext{} + } + if request.GetConnection().GetContext().GetIpContext() == nil { + request.GetConnection().GetContext().IpContext = &networkservice.IPContext{} + } + request.GetConnection().GetContext().GetIpContext().SrcRoutes = append(request.GetConnection().GetContext().GetIpContext().SrcRoutes, r.srcRoutes...) + request.GetConnection().GetContext().GetIpContext().DstRoutes = append(request.GetConnection().GetContext().GetIpContext().DstRoutes, r.dstRoutes...) + return next.Server(ctx).Request(ctx, request) +} + +func (r *testRoutesServer) Close(ctx context.Context, conn *networkservice.Connection) (*emptypb.Empty, error) { + return next.Server(ctx).Close(ctx, conn) +} diff --git a/internal/tests/suite_combinatronics_test.go b/internal/tests/suite_combinatronics_test.go index 8ea6241e..410ee538 100644 --- a/internal/tests/suite_combinatronics_test.go +++ b/internal/tests/suite_combinatronics_test.go @@ -20,6 +20,7 @@ package tests import ( "context" + "fmt" "io/ioutil" "net" "net/url" @@ -53,30 +54,59 @@ import ( func (f *ForwarderTestSuite) TestCombinations() { _, prefix1, err := net.ParseCIDR("10.0.0.0/24") f.Require().NoError(err) - _, prefix2, err := net.ParseCIDR("fc00::/7") + _, prefix2, err := net.ParseCIDR("fc00::/64") f.Require().NoError(err) + srcRoutes := []*networkservice.Route{} + dstRoutes := []*networkservice.Route{} + for i := 1; i < 6; i++ { + srcRoutes = append(srcRoutes, + &networkservice.Route{ + Prefix: fmt.Sprintf("10.0.%d.0/24", i), + }, + &networkservice.Route{ + Prefix: fmt.Sprintf("fc00:0:%d::/64", i), + }, + ) + dstRoutes = append(dstRoutes, + &networkservice.Route{ + Prefix: fmt.Sprintf("10.0.%d.1/24", i), + }, + &networkservice.Route{ + Prefix: fmt.Sprintf("fc00:0:%d::1/64", i), + }, + ) + } + endpoints := map[string]func(ctx context.Context) verifiableEndpoint{ kernel.MECHANISM: func(ctx context.Context) verifiableEndpoint { return newKernelVerifiableEndpoint(ctx, prefix1, prefix2, + srcRoutes, + dstRoutes, spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime), ) }, memif.MECHANISM: func(ctx context.Context) verifiableEndpoint { return newMemifVerifiableEndpoint(ctx, prefix1, prefix2, + srcRoutes, + dstRoutes, spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime), f.vppServerConn, ) }, vxlan.MECHANISM: func(ctx context.Context) verifiableEndpoint { return newVxlanVerifiableEndpoint(ctx, prefix1, prefix2, + srcRoutes, + dstRoutes, spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime), f.vppServerConn, ) }, wireguard.MECHANISM: func(ctx context.Context) verifiableEndpoint { return newWireguardVerifiableEndpoint(ctx, prefix1, prefix2, + srcRoutes, + dstRoutes, spiffejwt.TokenGeneratorFunc(f.x509source, f.config.MaxTokenLifetime), f.vppServerConn, ) diff --git a/internal/tests/suite_kernel_test.go b/internal/tests/suite_kernel_test.go index 8a9b30d3..d0d4d641 100644 --- a/internal/tests/suite_kernel_test.go +++ b/internal/tests/suite_kernel_test.go @@ -31,6 +31,9 @@ import ( "github.com/vishvananda/netns" "google.golang.org/grpc" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes" + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" @@ -74,6 +77,7 @@ type kernelVerifiableEndpoint struct { func newKernelVerifiableEndpoint(ctx context.Context, prefix1, prefix2 *net.IPNet, + srcRoutes, dstRoutes []*networkservice.Route, tokenGenerator token.GeneratorFunc, ) verifiableEndpoint { rootNSHandle, err := netns.Get() @@ -108,6 +112,7 @@ func newKernelVerifiableEndpoint(ctx context.Context, endpoint.WithAdditionalFunctionality( point2pointipam.NewServer(prefix1), point2pointipam.NewServer(prefix2), + routes.NewServer(srcRoutes, dstRoutes), mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{ kernel.MECHANISM: chain.NewNetworkServiceServer( kernelmechanism.NewServer(kernelmechanism.WithInterfaceName(endpointNSName)), @@ -134,6 +139,9 @@ func (k *kernelVerifiableEndpoint) VerifyConnection(conn *networkservice.Connect if err := checkKernelInterface(namingConn, conn.GetContext().GetIpContext().GetDstIPNets(), k.endpointNSHandle); err != nil { return err } + if err := checkRoutes(namingConn, k.endpointNSHandle, metadata.IsClient(k)); err != nil { + return err + } for _, ip := range conn.GetContext().GetIpContext().GetSrcIPNets() { if err := pingKernel(ip, k.endpointNSHandle); err != nil { return err @@ -195,6 +203,9 @@ func (k *kernelVerifiableClient) VerifyConnection(conn *networkservice.Connectio if err := checkKernelInterface(conn, conn.GetContext().GetIpContext().GetSrcIPNets(), k.clientNSHandle); err != nil { return err } + if err := checkRoutes(conn, k.clientNSHandle, metadata.IsClient(k)); err != nil { + return err + } for _, ip := range conn.GetContext().GetIpContext().GetDstIPNets() { if err := pingKernel(ip, k.clientNSHandle); err != nil { return err @@ -277,3 +288,61 @@ func pingKernel(ipnet *net.IPNet, handle netns.NsHandle) error { } return nil } + +func checkRoutes(conn *networkservice.Connection, nsHandle netns.NsHandle, isClient bool) error { + if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil { + curNetNS, err := netns.Get() + if err != nil { + return errors.Wrap(err, "unable to get current netns") + } + netlinkHandle, err := netlink.NewHandleAtFrom(nsHandle, curNetNS) + if err != nil { + return errors.Wrap(err, "unable to get netlink Handle in target netNs") + } + kernelRoutes, err := netlinkHandle.RouteList(nil, netlink.FAMILY_ALL) + if err != nil { + return errors.Wrap(err, "unable to get routes") + } + nsmRoutes := conn.GetContext().GetIpContext().GetDstRoutesWithExplicitNextHop() + if isClient { + nsmRoutes = conn.GetContext().GetIpContext().GetSrcRoutesWithExplicitNextHop() + } + + link, err := netlinkHandle.LinkByName((mechanism.GetInterfaceName())) + if err != nil { + return errors.Wrapf(err, "unable to find link %s", mechanism.GetInterfaceName()) + } + + for _, nsmRoute := range nsmRoutes { + routeFound := false + for i := range kernelRoutes { + if !(kernelRoutes[i].Dst.IP.Equal(nsmRoute.GetPrefixIPNet().IP.Mask(nsmRoute.GetPrefixIPNet().Mask))) { + continue + } + kernelOnes, kernelBit := kernelRoutes[i].Dst.Mask.Size() + nsmOnes, nsmBits := nsmRoute.GetPrefixIPNet().Mask.Size() + if kernelOnes != nsmOnes { + continue + } + if kernelBit != nsmBits { + continue + } + + if !kernelRoutes[i].Gw.Equal(nsmRoute.GetNextHopIP()) { + continue + } + + if kernelRoutes[i].LinkIndex != link.Attrs().Index { + continue + } + + routeFound = true + break + } + if !routeFound { + return errors.Errorf("unable to find expected route: %s in routes %+v", nsmRoute, kernelRoutes) + } + } + } + return nil +} diff --git a/internal/tests/suite_memif_test.go b/internal/tests/suite_memif_test.go index 112ea57e..6bf0816c 100644 --- a/internal/tests/suite_memif_test.go +++ b/internal/tests/suite_memif_test.go @@ -30,6 +30,7 @@ import ( "github.com/edwarnicke/vpphelper" "github.com/pkg/errors" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/sendfd" "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" @@ -61,6 +62,7 @@ type memifVerifiableEndpoint struct { func newMemifVerifiableEndpoint(ctx context.Context, prefix1, prefix2 *net.IPNet, + srcRoutes, dstRoutes []*networkservice.Route, tokenGenerator token.GeneratorFunc, vppConn vpphelper.Connection, ) verifiableEndpoint { @@ -76,6 +78,7 @@ func newMemifVerifiableEndpoint(ctx context.Context, sendfd.NewServer(), point2pointipam.NewServer(prefix1), point2pointipam.NewServer(prefix2), + routes.NewServer(srcRoutes, dstRoutes), mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{ memif.MECHANISM: chain.NewNetworkServiceServer( metadata.NewServer(), diff --git a/internal/tests/suite_util_test.go b/internal/tests/suite_util_test.go index a31dc85e..0a2f67aa 100644 --- a/internal/tests/suite_util_test.go +++ b/internal/tests/suite_util_test.go @@ -32,7 +32,7 @@ import ( ) const ( - contextTimeout = 100 * time.Second + contextTimeout = 10000 * time.Second forwarderIP = "10.0.2.1" clientIP = "10.0.2.2" serverIP = "10.0.2.3" diff --git a/internal/tests/suite_vxlan_test.go b/internal/tests/suite_vxlan_test.go index c5f8e37b..a0744567 100644 --- a/internal/tests/suite_vxlan_test.go +++ b/internal/tests/suite_vxlan_test.go @@ -23,6 +23,7 @@ import ( "git.fd.io/govpp.git/api" "google.golang.org/grpc" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/pinhole" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms" @@ -48,6 +49,7 @@ type vxlanVerifiableEndpoint struct { func newVxlanVerifiableEndpoint(ctx context.Context, prefix1, prefix2 *net.IPNet, + srcRoutes, dstRoutes []*networkservice.Route, tokenGenerator token.GeneratorFunc, vppConn api.Connection) verifiableEndpoint { rv := &vxlanVerifiableEndpoint{ @@ -63,6 +65,7 @@ func newVxlanVerifiableEndpoint(ctx context.Context, metadata.NewServer(), point2pointipam.NewServer(prefix1), point2pointipam.NewServer(prefix2), + routes.NewServer(srcRoutes, dstRoutes), connectioncontext.NewServer(vppConn), pinhole.NewServer(vppConn), mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{ diff --git a/internal/tests/suite_wireguard_test.go b/internal/tests/suite_wireguard_test.go index 14734356..915a458d 100644 --- a/internal/tests/suite_wireguard_test.go +++ b/internal/tests/suite_wireguard_test.go @@ -23,6 +23,7 @@ import ( "github.com/edwarnicke/vpphelper" "google.golang.org/grpc" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/tests/routes" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" @@ -49,6 +50,7 @@ type wireguardVerifiableEndpoint struct { func newWireguardVerifiableEndpoint(ctx context.Context, prefix1, prefix2 *net.IPNet, + srcRoutes, dstRoutes []*networkservice.Route, tokenGenerator token.GeneratorFunc, vppConn vpphelper.Connection) verifiableEndpoint { rv := &vxlanVerifiableEndpoint{ @@ -64,6 +66,7 @@ func newWireguardVerifiableEndpoint(ctx context.Context, metadata.NewServer(), point2pointipam.NewServer(prefix1), point2pointipam.NewServer(prefix2), + routes.NewServer(srcRoutes, dstRoutes), up.NewServer(ctx, vppConn), pinhole.NewServer(vppConn), connectioncontext.NewServer(vppConn),