From d5b7730d5985d6e0faf1a61cb3dd26188699e9c8 Mon Sep 17 00:00:00 2001 From: Guvenc Gulce Date: Mon, 6 May 2024 11:30:17 +0200 Subject: [PATCH] Introduce HA support Change the client behaviour in such a way that in case a second metalbond server distributing the exact some routes as the "main" server is added to the server list then the client is aware of the second server when adding removing the routes from its routing table hence withdraws already received routes only if both servers withdraw the routes in question. Single server operation is same as before. (Backward compatible) Signed-off-by: Guvenc Gulce --- cmd/cmd.go | 5 ++ metalbond.go | 26 ++++----- routetable.go | 145 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 160 insertions(+), 16 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 22b2901..348ae0f 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -86,6 +86,7 @@ func main() { log.Infof("Client") log.Infof(" servers: %v", CLI.Client.Server) var err error + var serverCount = 0 if CLI.Client.Verbose { log.SetLevel(log.DebugLevel) @@ -139,6 +140,10 @@ func main() { } for _, server := range CLI.Client.Server { + serverCount++ + if serverCount == 3 { + panic(fmt.Errorf("can not connect to more than 2 servers")) + } if err := m.AddPeer(server, ""); err != nil { panic(fmt.Errorf("failed to add server: %v", err)) } diff --git a/metalbond.go b/metalbond.go index 8342ec2..6e3c0c9 100644 --- a/metalbond.go +++ b/metalbond.go @@ -310,8 +310,20 @@ func (m *MetalBond) GetSubscribedVnis() []VNI { return vnis } +func (m *MetalBond) getHAPeerIfExists(target *metalBondPeer) *metalBondPeer { + m.mtxPeers.RLock() + defer m.mtxPeers.RUnlock() + + for _, peer := range m.peers { + if peer.localAddr != target.localAddr { + return peer + } + } + return nil +} + func (m *MetalBond) addReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Destination, hop NextHop) error { - err := m.routeTable.AddNextHop(vni, dest, hop, fromPeer) + err := m.routeTable.AddNextHopHAAware(vni, dest, hop, fromPeer) if err != nil { return fmt.Errorf("Cannot add route to route table: %v", err) } @@ -326,16 +338,11 @@ func (m *MetalBond) addReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Dest m.log().Errorf("Could not distribute route to peers: %v", err) } - err = m.client.AddRoute(vni, dest, hop) - if err != nil { - m.log().Errorf("Client.AddRoute call failed: %v", err) - } - return nil } func (m *MetalBond) removeReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Destination, hop NextHop) error { - err, remaining := m.routeTable.RemoveNextHop(vni, dest, hop, fromPeer) + err, remaining := m.routeTable.RemoveNextHopHAAware(vni, dest, hop, fromPeer) if err != nil { return fmt.Errorf("Cannot remove route from route table: %v", err) } @@ -352,11 +359,6 @@ func (m *MetalBond) removeReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest D } } - err = m.client.RemoveRoute(vni, dest, hop) - if err != nil { - m.log().Errorf("Client.RemoveRoute call failed: %v", err) - } - return nil } diff --git a/routetable.go b/routetable.go index 5560f3f..bbc0adb 100644 --- a/routetable.go +++ b/routetable.go @@ -145,11 +145,66 @@ func (rt *routeTable) RemoveNextHop(vni VNI, dest Destination, nh NextHop, recei return nil, left } -func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error { +func (rt *routeTable) RemoveNextHopHAAware(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) (error, int) { + var haPeer *metalBondPeer = nil rt.rwmtx.Lock() defer rt.rwmtx.Unlock() + if rt.routes == nil { + rt.routes = make(map[VNI]map[Destination]map[NextHop]map[*metalBondPeer]bool) + } + // TODO Performance: reused found map pointers + if _, exists := rt.routes[vni]; !exists { + return fmt.Errorf("VNI does not exist"), 0 + } + + if _, exists := rt.routes[vni][dest]; !exists { + return fmt.Errorf("Destination does not exist"), 0 + } + + if _, exists := rt.routes[vni][dest][nh]; !exists { + return fmt.Errorf("nexthop does not exist"), 0 + } + + if _, exists := rt.routes[vni][dest][nh][receivedFrom]; !exists { + return fmt.Errorf("ReceivedFrom does not exist"), 0 + } + + delete(rt.routes[vni][dest][nh], receivedFrom) + left := len(rt.routes[vni][dest][nh]) + + if len(rt.routes[vni][dest][nh]) == 0 { + delete(rt.routes[vni][dest], nh) + } + + if len(rt.routes[vni][dest]) == 0 { + delete(rt.routes[vni], dest) + } + + if len(rt.routes[vni]) == 0 { + delete(rt.routes, vni) + } + + if !receivedFrom.isServer { + haPeer = receivedFrom.metalbond.getHAPeerIfExists(receivedFrom) + } + + if rt.IsRouteOkToRemoveFromTheClient(vni, dest, nh, receivedFrom, haPeer) { + err := receivedFrom.metalbond.client.RemoveRoute(vni, dest, nh) + if err != nil { + receivedFrom.metalbond.log().Errorf("Client.RemoveRoute call failed: %v", err) + return err, 0 + } + } + + return nil, left +} + +func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error { + rt.rwmtx.Lock() + defer rt.rwmtx.Unlock() + if _, exists := rt.routes[vni]; !exists { rt.routes[vni] = make(map[Destination]map[NextHop]map[*metalBondPeer]bool) } @@ -163,7 +218,7 @@ func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, received } if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists { - return fmt.Errorf("Nexthop already exists") + return fmt.Errorf("nexthop already exists") } rt.routes[vni][dest][nh][receivedFrom] = true @@ -171,11 +226,11 @@ func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, received return nil } -func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool { +func (rt *routeTable) AddNextHopHAAware(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error { + var haPeer *metalBondPeer = nil rt.rwmtx.Lock() defer rt.rwmtx.Unlock() - // TODO Performance: reused found map pointers if _, exists := rt.routes[vni]; !exists { rt.routes[vni] = make(map[Destination]map[NextHop]map[*metalBondPeer]bool) } @@ -189,6 +244,88 @@ func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, recei } if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists { + return fmt.Errorf("nexthop already exists") + } + + rt.routes[vni][dest][nh][receivedFrom] = true + + if !receivedFrom.isServer { + haPeer = receivedFrom.metalbond.getHAPeerIfExists(receivedFrom) + } + + if rt.IsRouteOkToAddToTheClient(vni, dest, nh, receivedFrom, haPeer) { + err := receivedFrom.metalbond.client.AddRoute(vni, dest, nh) + if err != nil { + receivedFrom.metalbond.log().Errorf("Client.AddRoute call failed: %v", err) + return err + } + } + + return nil +} + +func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool { + rt.rwmtx.RLock() + defer rt.rwmtx.RUnlock() + return rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom) +} + +func (rt *routeTable) NextHopExistsUnlocked(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool { + if _, exists := rt.routes[vni]; !exists { + return false + } + + if _, exists := rt.routes[vni][dest]; !exists { + return false + } + + if _, exists := rt.routes[vni][dest][nh]; !exists { + return false + } + + if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists { + return true + } + + return false +} + +// Call this function only with r/w lock of the table +func (rt *routeTable) IsRouteOkToAddToTheClient(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer, haPeer *metalBondPeer) bool { + count := 0 + + if rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom) { + count++ + } + + if haPeer != nil { + if rt.NextHopExistsUnlocked(vni, dest, nh, haPeer) { + count++ + } + } + + if count == 1 { + return true + } + + return false +} + +// Call this function only with r/w lock of the table +func (rt *routeTable) IsRouteOkToRemoveFromTheClient(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer, haPeer *metalBondPeer) bool { + count := 0 + + if rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom) { + count++ + } + + if haPeer != nil { + if rt.NextHopExistsUnlocked(vni, dest, nh, haPeer) { + count++ + } + } + + if count == 0 { return true }