From 88ee6c8fd1ce364179e2566036938b40b2597478 Mon Sep 17 00:00:00 2001 From: Tao Date: Tue, 7 Jun 2022 10:29:13 +0200 Subject: [PATCH] Support dynamic route ops with muti -hops. --- netlink.go | 94 ++++++++++++++++++--------- threadUnsafeRoutetable.go | 132 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 31 deletions(-) create mode 100644 threadUnsafeRoutetable.go diff --git a/netlink.go b/netlink.go index 18ee002..a82573f 100644 --- a/netlink.go +++ b/netlink.go @@ -17,7 +17,6 @@ package metalbond import ( "fmt" "net" - "sync" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" @@ -28,7 +27,8 @@ const METALBOND_RT_PROTO netlink.RouteProtocol = 254 type NetlinkClient struct { config NetlinkClientConfig tunDevice netlink.Link - mtx sync.Mutex + + rt threadUnsafeRouteTable } type NetlinkClientConfig struct { @@ -49,21 +49,45 @@ func NewNetlinkClient(config NetlinkClientConfig) (*NetlinkClient, error) { return &NetlinkClient{ config: config, tunDevice: link, + rt: newThreadUnsafeRouteTable(), }, nil } func (c *NetlinkClient) AddRoute(vni VNI, dest Destination, hop NextHop) error { - c.mtx.Lock() - defer c.mtx.Unlock() if c.config.IPv4Only && dest.IPVersion != IPV4 { log.Infof("Received non-IPv4 route will not be installed in kernel route table (IPv4-only mode)") return nil } + err := c.rt.AddNextHop(vni, dest, hop) + if err != nil { + return fmt.Errorf("failed to add route for netlink client internal rt") + } + + nxtHops := c.rt.GetNextHopsByDestination(vni, dest) + + err = c.updateRoute(vni, dest, nxtHops) + if err != nil { + return fmt.Errorf("failed to update route") + } + + return nil + +} + +func (c *NetlinkClient) RemoveRoute(vni VNI, dest Destination, hop NextHop) error { + + err, _ := c.rt.RemoveNextHop(vni, dest, hop) + if err != nil { + return fmt.Errorf("failed to remove nxthop for netlink client internal rt") + } + + nxtHops := c.rt.GetNextHopsByDestination(vni, dest) + table, exists := c.config.VNITableMap[vni] if !exists { - return fmt.Errorf("No route table ID known for given VNI") + return fmt.Errorf("no route table ID known for given VNI") } _, dst, err := net.ParseCIDR(dest.Prefix.String()) @@ -71,29 +95,27 @@ func (c *NetlinkClient) AddRoute(vni VNI, dest Destination, hop NextHop) error { return fmt.Errorf("cannot parse destination prefix: %v", err) } - encap := netlink.IP6tnlEncap{ - Dst: net.ParseIP(hop.TargetAddress.String()), - Src: net.ParseIP("::"), // what source ip to put here? Metalbond object, m, does not contain this info yet. - } - - route := &netlink.Route{ - LinkIndex: c.tunDevice.Attrs().Index, - Dst: dst, - Encap: &encap, - Table: table, - Protocol: METALBOND_RT_PROTO, - } // by default, the route is already installed into the kernel table without explicite specification - - if err := netlink.RouteAdd(route); err != nil { - return fmt.Errorf("cannot add route to %s (table %d) to kernel: %v", dest, table, err) + if len(nxtHops) == 0 { + + route := &netlink.Route{ + Dst: dst, + Table: table, + } // by default, the route is already installed into the kernel table without explicite specification + + if err := netlink.RouteDel(route); err != nil { + return fmt.Errorf("cannot remove route to %s (table %d) from kernel: %v", dest, table, err) + } + } else { + err := c.updateRoute(vni, dest, nxtHops) + if err != nil { + return fmt.Errorf("failed to update route") + } } return nil } -func (c *NetlinkClient) RemoveRoute(vni VNI, dest Destination, hop NextHop) error { - c.mtx.Lock() - defer c.mtx.Unlock() +func (c *NetlinkClient) updateRoute(vni VNI, dest Destination, nexthops []NextHop) error { if c.config.IPv4Only && dest.IPVersion != IPV4 { return nil @@ -109,21 +131,31 @@ func (c *NetlinkClient) RemoveRoute(vni VNI, dest Destination, hop NextHop) erro return fmt.Errorf("cannot parse destination prefix: %v", err) } - encap := netlink.IP6tnlEncap{ - Dst: net.ParseIP(hop.TargetAddress.String()), - Src: net.ParseIP("::"), // what source ip to put here? Metalbond object, m, does not contain this info yet. + var nextHopInfos []*netlink.NexthopInfo + + for _, hop := range nexthops { + + encap := netlink.IP6tnlEncap{ + Dst: net.ParseIP(hop.TargetAddress.String()), + Src: net.ParseIP("::"), + } + + nextHopInfos = append(nextHopInfos, &netlink.NexthopInfo{ + LinkIndex: c.tunDevice.Attrs().Index, + Encap: &encap, + // other fields value to be decided + }) } route := &netlink.Route{ - LinkIndex: c.tunDevice.Attrs().Index, Dst: dst, - Encap: &encap, + MultiPath: nextHopInfos, Table: table, Protocol: METALBOND_RT_PROTO, - } // by default, the route is already installed into the kernel table without explicite specification + } - if err := netlink.RouteDel(route); err != nil { - return fmt.Errorf("cannot remove route to %s (table %d) from kernel: %v", dest, table, err) + if err := netlink.RouteReplace(route); err != nil { + return fmt.Errorf("cannot update route to %s (table %d) to kernel: %v", dest, table, err) } return nil diff --git a/threadUnsafeRoutetable.go b/threadUnsafeRoutetable.go new file mode 100644 index 0000000..9dcfaef --- /dev/null +++ b/threadUnsafeRoutetable.go @@ -0,0 +1,132 @@ +// Copyright 2022 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metalbond + +import ( + "fmt" +) + +type threadUnsafeRouteTable struct { + routes map[VNI]map[Destination]map[NextHop]bool +} + +func newThreadUnsafeRouteTable() threadUnsafeRouteTable { + return threadUnsafeRouteTable{ + routes: make(map[VNI]map[Destination]map[NextHop]bool), + } +} + +func (rt *threadUnsafeRouteTable) GetVNIs() []VNI { + + vnis := []VNI{} + for k := range rt.routes { + vnis = append(vnis, k) + } + return vnis +} + +func (rt *threadUnsafeRouteTable) GetDestinationsByVNI(vni VNI) map[Destination][]NextHop { + + ret := make(map[Destination][]NextHop) + + if _, exists := rt.routes[vni]; !exists { + return ret + } + + for dest, nhm := range rt.routes[vni] { + nhs := []NextHop{} + + for nh := range nhm { + nhs = append(nhs, nh) + } + + ret[dest] = nhs + } + + return ret +} + +func (rt *threadUnsafeRouteTable) GetNextHopsByDestination(vni VNI, dest Destination) []NextHop { + + nh := []NextHop{} + + // TODO Performance: reused found map pointers + if _, exists := rt.routes[vni]; !exists { + return nh + } + + if _, exists := rt.routes[vni][dest]; !exists { + return nh + } + + for k := range rt.routes[vni][dest] { + nh = append(nh, k) + } + + return nh +} + +func (rt *threadUnsafeRouteTable) RemoveNextHop(vni VNI, dest Destination, nh NextHop) (error, int) { + + if rt.routes == nil { + rt.routes = make(map[VNI]map[Destination]map[NextHop]bool) + } + + // TODO Performance: reused found map pointers + if _, exists := rt.routes[vni]; !exists { + return fmt.Errorf("Nexthop does not exist"), 0 + } + + if _, exists := rt.routes[vni][dest]; !exists { + return fmt.Errorf("Nexthop does not exist"), 0 + } + + if _, exists := rt.routes[vni][dest][nh]; !exists { + return fmt.Errorf("Nexthop does not exist"), 0 + } + + delete(rt.routes[vni][dest], nh) + left := len(rt.routes[vni][dest]) + + if len(rt.routes[vni][dest]) == 0 { + delete(rt.routes[vni], dest) + } + + if len(rt.routes[vni]) == 0 { + delete(rt.routes, vni) + } + + return nil, left +} + +func (rt *threadUnsafeRouteTable) AddNextHop(vni VNI, dest Destination, nh NextHop) error { + + // TODO Performance: reused found map pointers + if _, exists := rt.routes[vni]; !exists { + rt.routes[vni] = make(map[Destination]map[NextHop]bool) + } + + if _, exists := rt.routes[vni][dest]; !exists { + rt.routes[vni][dest] = make(map[NextHop]bool) + } + + if _, exists := rt.routes[vni][dest][nh]; exists { + return fmt.Errorf("Nexthop already exists") + } + + rt.routes[vni][dest][nh] = true + + return nil +}