Introduce HA support

Change the client behaviour in such a way that in case a second metalbond server distributing the exact some routes as the "main" server is added to the server list then the client is aware of the second server when adding removing the routes from its routing table hence withdraws already received routes only if both servers withdraw the routes in question. Single server operation is same as before. (Backward compatible) Signed-off-by: Guvenc Gulce <[email protected]>
ironcore-dev · May 6, 2024 · d47c123 · d47c123
1 parent a3c16b2
commit d47c123
Show file tree

Hide file tree

Showing 3 changed files with 157 additions and 16 deletions.
diff --git a/cmd/cmd.go b/cmd/cmd.go
@@ -86,6 +86,7 @@ func main() {
 		log.Infof("Client")
 		log.Infof("  servers: %v", CLI.Client.Server)
 		var err error
+		var serverCount = 0
 
 		if CLI.Client.Verbose {
 			log.SetLevel(log.DebugLevel)
@@ -139,6 +140,10 @@ func main() {
 		}
 
 		for _, server := range CLI.Client.Server {
+			serverCount++
+			if serverCount == 3 {
+				panic(fmt.Errorf("can not connect to more than 2 servers"))
+			}
 			if err := m.AddPeer(server, ""); err != nil {
 				panic(fmt.Errorf("failed to add server: %v", err))
 			}

diff --git a/metalbond.go b/metalbond.go
@@ -310,8 +310,17 @@ func (m *MetalBond) GetSubscribedVnis() []VNI {
 	return vnis
 }
 
+func (m *MetalBond) getHAPeerIfExists(target *metalBondPeer) *metalBondPeer {
+	for _, peer := range m.peers {
+		if peer.localAddr != target.localAddr {
+			return peer
+		}
+	}
+	return nil
+}
+
 func (m *MetalBond) addReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Destination, hop NextHop) error {
-	err := m.routeTable.AddNextHop(vni, dest, hop, fromPeer)
+	err := m.routeTable.AddNextHopHAAware(vni, dest, hop, fromPeer)
 	if err != nil {
 		return fmt.Errorf("Cannot add route to route table: %v", err)
 	}
@@ -326,16 +335,11 @@ func (m *MetalBond) addReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Dest
 		m.log().Errorf("Could not distribute route to peers: %v", err)
 	}
 
-	err = m.client.AddRoute(vni, dest, hop)
-	if err != nil {
-		m.log().Errorf("Client.AddRoute call failed: %v", err)
-	}
-
 	return nil
 }
 
 func (m *MetalBond) removeReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest Destination, hop NextHop) error {
-	err, remaining := m.routeTable.RemoveNextHop(vni, dest, hop, fromPeer)
+	err, remaining := m.routeTable.RemoveNextHopHAAware(vni, dest, hop, fromPeer)
 	if err != nil {
 		return fmt.Errorf("Cannot remove route from route table: %v", err)
 	}
@@ -352,11 +356,6 @@ func (m *MetalBond) removeReceivedRoute(fromPeer *metalBondPeer, vni VNI, dest D
 		}
 	}
 
-	err = m.client.RemoveRoute(vni, dest, hop)
-	if err != nil {
-		m.log().Errorf("Client.RemoveRoute call failed: %v", err)
-	}
-
 	return nil
 }
 

diff --git a/routetable.go b/routetable.go
@@ -145,11 +145,66 @@ func (rt *routeTable) RemoveNextHop(vni VNI, dest Destination, nh NextHop, recei
 	return nil, left
 }
 
-func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error {
+func (rt *routeTable) RemoveNextHopHAAware(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) (error, int) {
+	var haPeer *metalBondPeer = nil
 	rt.rwmtx.Lock()
 	defer rt.rwmtx.Unlock()
 
+	if rt.routes == nil {
+		rt.routes = make(map[VNI]map[Destination]map[NextHop]map[*metalBondPeer]bool)
+	}
+
 	// TODO Performance: reused found map pointers
+	if _, exists := rt.routes[vni]; !exists {
+		return fmt.Errorf("VNI does not exist"), 0
+	}
+
+	if _, exists := rt.routes[vni][dest]; !exists {
+		return fmt.Errorf("Destination does not exist"), 0
+	}
+
+	if _, exists := rt.routes[vni][dest][nh]; !exists {
+		return fmt.Errorf("nexthop does not exist"), 0
+	}
+
+	if _, exists := rt.routes[vni][dest][nh][receivedFrom]; !exists {
+		return fmt.Errorf("ReceivedFrom does not exist"), 0
+	}
+
+	delete(rt.routes[vni][dest][nh], receivedFrom)
+	left := len(rt.routes[vni][dest][nh])
+
+	if len(rt.routes[vni][dest][nh]) == 0 {
+		delete(rt.routes[vni][dest], nh)
+	}
+
+	if len(rt.routes[vni][dest]) == 0 {
+		delete(rt.routes[vni], dest)
+	}
+
+	if len(rt.routes[vni]) == 0 {
+		delete(rt.routes, vni)
+	}
+
+	if !receivedFrom.isServer {
+		haPeer = receivedFrom.metalbond.getHAPeerIfExists(receivedFrom)
+	}
+
+	if rt.IsRouteOkToRemoveFromTheClient(vni, dest, nh, receivedFrom, haPeer) {
+		err := receivedFrom.metalbond.client.RemoveRoute(vni, dest, nh)
+		if err != nil {
+			receivedFrom.metalbond.log().Errorf("Client.RemoveRoute call failed: %v", err)
+			return err, 0
+		}
+	}
+
+	return nil, left
+}
+
+func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error {
+	rt.rwmtx.Lock()
+	defer rt.rwmtx.Unlock()
+
 	if _, exists := rt.routes[vni]; !exists {
 		rt.routes[vni] = make(map[Destination]map[NextHop]map[*metalBondPeer]bool)
 	}
@@ -163,19 +218,19 @@ func (rt *routeTable) AddNextHop(vni VNI, dest Destination, nh NextHop, received
 	}
 
 	if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists {
-		return fmt.Errorf("Nexthop already exists")
+		return fmt.Errorf("nexthop already exists")
 	}
 
 	rt.routes[vni][dest][nh][receivedFrom] = true
 
 	return nil
 }
 
-func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool {
+func (rt *routeTable) AddNextHopHAAware(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) error {
+	var haPeer *metalBondPeer = nil
 	rt.rwmtx.Lock()
 	defer rt.rwmtx.Unlock()
 
-	// TODO Performance: reused found map pointers
 	if _, exists := rt.routes[vni]; !exists {
 		rt.routes[vni] = make(map[Destination]map[NextHop]map[*metalBondPeer]bool)
 	}
@@ -189,6 +244,88 @@ func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, recei
 	}
 
 	if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists {
+		return fmt.Errorf("nexthop already exists")
+	}
+
+	rt.routes[vni][dest][nh][receivedFrom] = true
+
+	if !receivedFrom.isServer {
+		haPeer = receivedFrom.metalbond.getHAPeerIfExists(receivedFrom)
+	}
+
+	if rt.IsRouteOkToAddToTheClient(vni, dest, nh, receivedFrom, haPeer) {
+		err := receivedFrom.metalbond.client.AddRoute(vni, dest, nh)
+		if err != nil {
+			receivedFrom.metalbond.log().Errorf("Client.AddRoute call failed: %v", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (rt *routeTable) NextHopExists(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool {
+	rt.rwmtx.RLock()
+	defer rt.rwmtx.RUnlock()
+	return rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom)
+}
+
+func (rt *routeTable) NextHopExistsUnlocked(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer) bool {
+	if _, exists := rt.routes[vni]; !exists {
+		return false
+	}
+
+	if _, exists := rt.routes[vni][dest]; !exists {
+		return false
+	}
+
+	if _, exists := rt.routes[vni][dest][nh]; !exists {
+		return false
+	}
+
+	if _, exists := rt.routes[vni][dest][nh][receivedFrom]; exists {
+		return true
+	}
+
+	return false
+}
+
+// Call this function only with r/w lock of the table
+func (rt *routeTable) IsRouteOkToAddToTheClient(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer, haPeer *metalBondPeer) bool {
+	count := 0
+
+	if rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom) {
+		count++
+	}
+
+	if haPeer != nil {
+		if rt.NextHopExistsUnlocked(vni, dest, nh, haPeer) {
+			count++
+		}
+	}
+
+	if count == 1 {
+		return true
+	}
+
+	return false
+}
+
+// Call this function only with r/w lock of the table
+func (rt *routeTable) IsRouteOkToRemoveFromTheClient(vni VNI, dest Destination, nh NextHop, receivedFrom *metalBondPeer, haPeer *metalBondPeer) bool {
+	count := 0
+
+	if rt.NextHopExistsUnlocked(vni, dest, nh, receivedFrom) {
+		count++
+	}
+
+	if haPeer != nil {
+		if rt.NextHopExistsUnlocked(vni, dest, nh, haPeer) {
+			count++
+		}
+	}
+
+	if count == 0 {
 		return true
 	}