Support for kube-router.io/peer.localips annotation (#1392)

* Support for kube-router.io/peer.localips annotation

* Fix checking for valid addresses in kube-router.io/peer.localips

docs/bgp.md

@@ -97,16 +97,36 @@ kubectl annotate node <kube-node> "kube-router.io/peer.asns=65000,65000"
 
 For traffic shaping purposes, you may want to prepend the AS path announced to peers.
 This can be accomplished on a per-node basis with annotations:
+
 - `kube-router.io/path-prepend.as`
 - `kube-router.io/path-prepend.repeat-n`
 
 If you wanted to prepend all routes from a particular node with the AS 65000 five times,
 you would run the following commands:
+
 ```
 kubectl annotate node <kube-node> "kube-router.io/path-prepend.as=65000"
 kubectl annotate node <kube-node> "kube-router.io/path-prepend.repeat-n=5"
 ```
 
+### BGP Peer Local IP configuration
+
+In some setups it might be desirable to set local IP address used for connectin external BGP
+peers. This can be accomplished on nodes with annotations:
+
+- `kube-router.io/peer.localips`
+
+If set, this must be a list with a local IP address for each peer, or left empty to use nodeIP.
+
+Example:
+
+```
+kubectl annotate node <kube-node> "kube-router.io/peer.localips=10.1.1.1,10.1.1.2"
+```
+
+This will instruct kube-router to use IP `10.1.1.1` for first BGP peer as a local address, and use `10.1.1.2`
+for the second.
+
 ### BGP Peer Password Authentication
 
 The examples above have assumed there is no password authentication with BGP

pkg/controllers/routing/bgp_peers.go

@@ -254,8 +254,8 @@ func (nrc *NetworkRoutingController) connectToExternalBGPPeers(server *gobgp.Bgp
 }
 
 // Does validation and returns neighbor configs
-func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []string, holdtime float64,
-	localAddress string) ([]*gobgpapi.Peer, error) {
+func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []string, localips []string,
+	holdtime float64, localAddress string) ([]*gobgpapi.Peer, error) {
 	peers := make([]*gobgpapi.Peer, 0)
 
 	// Validations
@@ -275,6 +275,12 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str
 			"Example: \"port,,port\" OR [\"port\",\"\",\"port\"]", strconv.Itoa(options.DefaultBgpPort))
 	}
 
+	if len(ips) != len(localips) && len(localips) != 0 {
+		return nil, fmt.Errorf("invalid peer router config. The number of localIPs should either be zero, or "+
+			"one per peer router. If blank items are used, it will default to nodeIP, %s. "+
+			"Example: \"10.1.1.1,,10.1.1.2\" OR [\"10.1.1.1\",\"\",\"10.1.1.2\"]", localAddress)
+	}
+
 	for i := 0; i < len(ips); i++ {
 		if !((asns[i] >= 1 && asns[i] <= 23455) ||
 			(asns[i] >= 23457 && asns[i] <= 63999) ||
@@ -285,8 +291,7 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str
 				asns[i])
 		}
 
-		// explicitly set neighbors.transport.config.local-address with nodeIP which is configured
-		// as their neighbor address at the remote peers.
+		// explicitly set neighbors.transport.config.local-address
 		// this prevents the controller from initiating connection to its peers with a different IP address
 		// when multiple L3 interfaces are active.
 		peer := &gobgpapi.Peer{
@@ -309,6 +314,10 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str
 			peer.Conf.AuthPassword = passwords[i]
 		}
 
+		if len(localips) != 0 && localips[i] != "" {
+			peer.Transport.LocalAddress = localips[i]
+		}
+
 		peers = append(peers, peer)
 	}
 

pkg/controllers/routing/network_routes_controller.go

@@ -47,6 +47,7 @@ const (
 	pathPrependRepeatNAnnotation     = "kube-router.io/path-prepend.repeat-n"
 	peerASNAnnotation                = "kube-router.io/peer.asns"
 	peerIPAnnotation                 = "kube-router.io/peer.ips"
+	peerLocalIPAnnotation            = "kube-router.io/peer.localips"
 	//nolint:gosec // this is not a hardcoded password
 	peerPasswordAnnotation             = "kube-router.io/peer.passwords"
 	peerPortAnnotation                 = "kube-router.io/peer.ports"
@@ -1130,9 +1131,38 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error {
 			}
 		}
 
+		// Get Global Peer Router LocalIP configs
+		var peerLocalIPs []string
+		nodeBGPPeerLocalIPs, ok := node.ObjectMeta.Annotations[peerLocalIPAnnotation]
+		if !ok {
+			klog.Infof("Could not find BGP peer local ip info in the node's annotations. Assuming node IP.")
+		} else {
+			peerLocalIPs = stringToSlice(nodeBGPPeerLocalIPs, ",")
+			err = func() error {
+				for _, s := range peerLocalIPs {
+					if s != "" {
+						ip := net.ParseIP(s)
+						if ip == nil {
+							return fmt.Errorf("could not parse \"%s\" as an IP", s)
+						}
+					}
+				}
+
+				return nil
+			}()
+			if err != nil {
+				err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{})
+				if err2 != nil {
+					klog.Errorf("Failed to stop bgpServer: %s", err2)
+				}
+
+				return fmt.Errorf("failed to parse node's Peer Local Addresses Annotation: %s", err)
+			}
+		}
+
 		// Create and set Global Peer Router complete configs
-		nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerPorts, peerASNs, peerPasswords, nrc.bgpHoldtime,
-			nrc.nodeIP.String())
+		nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerPorts, peerASNs, peerPasswords, peerLocalIPs,
+			nrc.bgpHoldtime, nrc.nodeIP.String())
 		if err != nil {
 			err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{})
 			if err2 != nil {
@@ -1327,7 +1357,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
 	}
 
 	nrc.globalPeerRouters, err = newGlobalPeers(kubeRouterConfig.PeerRouters, peerPorts,
-		peerASNs, peerPasswords, nrc.bgpHoldtime, nrc.nodeIP.String())
+		peerASNs, peerPasswords, nil, nrc.bgpHoldtime, nrc.nodeIP.String())
 	if err != nil {
 		return nil, fmt.Errorf("error processing Global Peer Router configs: %s", err)
 	}