support multiple NIC with name specified

Signed-off-by: Icarus9913 <[email protected]>

docs/usage/spider-ippool-zh_CN.md

@@ -70,6 +70,22 @@ ipam.spidernet.io/ippool: |-
   }
 ```
 
+在使用注解 `ipam.spidernet.io/ippools` 用于多网卡指定时，你可显式的通过指定 `interface` 字段标明网卡名，也可以通过**数组顺序排列**让第几张卡用哪些 IP 池。另外，字段 `cleangateway` 标明是否需要根据 IP 池中的 `gateway` 字段生成一条默认路由，当 `cleangateway` 为 `true` 标明无需生成默认路由。(默认为false)
+
+> 多网卡场景下，一般无法为路由表 `main` 表生成两条及以上的默认路由。搭配 `Coordinator` 插件可为你处理好该问题，因此你可以忽略 `cleangateway` 字段。或者在单独使用 Spiderpool IPAM 插件时，可借助 `cleangateway: true` 来标明不根据 IP 池 gateway 字段生成默认路由。
+
+```yaml
+ipam.spidernet.io/ippools: |-
+  [{
+      "ipv4": ["demo-v4-ippool1"],
+      "ipv6": ["demo-v6-ippool1"],
+   },{
+      "ipv4": ["demo-v4-ippool2"],
+      "ipv6": ["demo-v6-ippool2"],
+      "cleangateway": true
+  }]
+```
+
 ```yaml
 ipam.spidernet.io/ippools: |-
   [{

docs/usage/spider-ippool.md

@@ -69,6 +69,22 @@ ipam.spidernet.io/ippool: |-
   }
 ```
 
+When using the annotation `ipam.spidernet.io/ippools` for specifying multiple network interfaces, you can explicitly indicate the interface name by specifying the `interface` field. Alternatively, you can use **array ordering** to determine which IP pools are assigned to which network interfaces. Additionally, the `cleangateway` field indicates whether a default route should be generated based on the `gateway` field of the IPPool. When `cleangateway` is set to true, it means that no default route needs to be generated (default is false).
+
+> In scenarios with multiple network interfaces, it is generally not possible to generate two or more default routes in the `main` routing table. The plugin `Coordinator` already solved this problem and you can ignore `clengateway` field. If you want to use Spiderpool IPAM plugin alone, you can use `cleangateway: true` to indicate that a default route should not be generated based on the IPPool `gateway` field.
+
+```yaml
+ipam.spidernet.io/ippools: |-
+  [{
+      "ipv4": ["demo-v4-ippool1"],
+      "ipv6": ["demo-v6-ippool1"],
+   },{
+      "ipv4": ["demo-v4-ippool2"],
+      "ipv6": ["demo-v6-ippool2"],
+      "cleangateway": true
+  }]
+```
+
 ```yaml
 ipam.spidernet.io/ippools: |-
   [{

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/sasha-s/go-deadlock v0.3.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/spidernet-io/e2eframework v0.0.0-20230724150324-2eee77078275
+	github.com/spidernet-io/e2eframework v0.0.0-20231122092103-bfaf2546ded3
 	github.com/spidernet-io/spiderdoctor v0.3.0
 	github.com/tigera/operator v1.30.5
 	github.com/vishvananda/netlink v1.2.1-beta.2.0.20230621221334-77712cff8739

go.sum

@@ -546,8 +546,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=
 github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg=
-github.com/spidernet-io/e2eframework v0.0.0-20230724150324-2eee77078275 h1:F82/wYh0nch/B1iw7a6d2n+g6l6C4p0q2K7bn7YVISI=
-github.com/spidernet-io/e2eframework v0.0.0-20230724150324-2eee77078275/go.mod h1:RdWxk8i4tb80fYXBOEb8lm4Giusp6DFAGxgmt7OIGOo=
+github.com/spidernet-io/e2eframework v0.0.0-20231122092103-bfaf2546ded3 h1:E/Hqa5S13QvHOgNIy7MVfElDxBK5DMAdRQQPKwCCshY=
+github.com/spidernet-io/e2eframework v0.0.0-20231122092103-bfaf2546ded3/go.mod h1:RdWxk8i4tb80fYXBOEb8lm4Giusp6DFAGxgmt7OIGOo=
 github.com/spidernet-io/spiderdoctor v0.3.0 h1:Nq8xf8f1GkT3dRmeM2gt38ivkS5Fjpnz7LzueWGL6cA=
 github.com/spidernet-io/spiderdoctor v0.3.0/go.mod h1:4TcvXLBhlgqTYFO+LnagEKsEF2UKTukLhodPrsuqYnA=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=

pkg/ipam/allocate.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -79,7 +80,7 @@ func (i *ipam) Allocate(ctx context.Context, addArgs *models.IpamAddArgs) (*mode
 		}
 	} else {
 		logger.Debug("Try to retrieve the existing IP allocation")
-		addResp, err := i.retrieveExistingIPAllocation(ctx, string(pod.UID), *addArgs.IfName, endpoint)
+		addResp, err := i.retrieveExistingIPAllocation(ctx, string(pod.UID), *addArgs.IfName, endpoint, IsMultipleNicWithNoName(pod.Annotations))
 		if err != nil {
 			return nil, fmt.Errorf("failed to retrieve the existing IP allocation: %w", err)
 		}
@@ -113,7 +114,7 @@ func (i *ipam) retrieveStaticIPAllocation(ctx context.Context, nic string, pod *
 	}
 
 	logger.Info("Refresh the current IP allocation of the Endpoint")
-	if err := i.endpointManager.ReallocateCurrentIPAllocation(ctx, string(pod.UID), pod.Spec.NodeName, endpoint); err != nil {
+	if err := i.endpointManager.ReallocateCurrentIPAllocation(ctx, string(pod.UID), pod.Spec.NodeName, nic, endpoint, IsMultipleNicWithNoName(pod.Annotations)); err != nil {
 		return nil, fmt.Errorf("failed to refresh the current IP allocation of %s: %w", endpoint.Status.OwnerControllerType, err)
 	}
 
@@ -177,7 +178,7 @@ func (i *ipam) reallocateIPPoolIPRecords(ctx context.Context, uid string, endpoi
 	return nil
 }
 
-func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string, endpoint *spiderpoolv2beta1.SpiderEndpoint) (*models.IpamAddResponse, error) {
+func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string, endpoint *spiderpoolv2beta1.SpiderEndpoint, isMultipleNicWithNoName bool) (*models.IpamAddResponse, error) {
 	logger := logutils.FromContext(ctx)
 
 	// Create -> Delete -> Create a Pod with the same namespace and name in
@@ -193,6 +194,15 @@ func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string
 		return nil, nil
 	}
 
+	// update Endpoint NIC name in multiple NIC with no name mode by annotation "ipam.spidernet.io/ippools"
+	if isMultipleNicWithNoName {
+		var err error
+		allocation, err = i.endpointManager.UpdateAllocationNICName(ctx, endpoint, nic)
+		if nil != err {
+			return nil, fmt.Errorf("failed to update SpiderEndpoint allocation details NIC name %s, error: %v", nic, err)
+		}
+	}
+
 	ips, routes := convert.ConvertIPDetailsToIPConfigsAndAllRoutes(allocation.IPs)
 	addResp := &models.IpamAddResponse{
 		Ips:    ips,
@@ -205,6 +215,7 @@ func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string
 
 func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamAddArgs, pod *corev1.Pod, endpoint *spiderpoolv2beta1.SpiderEndpoint, podController types.PodTopController) (*models.IpamAddResponse, error) {
 	logger := logutils.FromContext(ctx)
+	isMultipleNicWithNoName := IsMultipleNicWithNoName(pod.Annotations)
 
 	logger.Debug("Parse custom routes")
 	customRoutes, err := getCustomRoutes(pod)
@@ -220,7 +231,7 @@ func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamA
 
 	var results []*types.AllocationResult
 	defer func() {
-		if err != nil {
+		if err != nil && !isMultipleNicWithNoName {
 			if len(results) != 0 {
 				i.failure.addFailureIPs(string(pod.UID), results)
 			}
@@ -241,11 +252,40 @@ func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamA
 	}
 
 	logger.Debug("Patch IP allocation results to Endpoint")
-	if err = i.endpointManager.PatchIPAllocationResults(ctx, results, endpoint, pod, podController); err != nil {
+	if err = i.endpointManager.PatchIPAllocationResults(ctx, results, endpoint, pod, podController, isMultipleNicWithNoName); err != nil {
 		return nil, fmt.Errorf("failed to patch IP allocation results to Endpoint: %v", err)
 	}
 
+	// sort the results in order by NIC sequence in multiple NIC with no name specified mode
+	if isMultipleNicWithNoName {
+		sort.Slice(results, func(i, j int) bool {
+			pre, err := strconv.Atoi(*results[i].IP.Nic)
+			if nil != err {
+				return false
+			}
+			latter, err := strconv.Atoi(*results[j].IP.Nic)
+			if nil != err {
+				return false
+			}
+			return pre < latter
+		})
+		for index := range results {
+			if *results[index].IP.Nic == strconv.Itoa(0) {
+				// replace the first NIC name from "0" to "eth0"
+				*results[index].IP.Nic = constant.ClusterDefaultInterfaceName
+
+				// replace the routes NIC name from "0" to "eth0"
+				for j := range results[index].Routes {
+					*results[index].Routes[j].IfName = constant.ClusterDefaultInterfaceName
+				}
+			}
+		}
+	}
+
 	resIPs, resRoutes := convert.ConvertResultsToIPConfigsAndAllRoutes(results)
+
+	// Actually in allocate Standard Mode, we just need the current turn NIC allocation result,
+	// but here are the all NICs results
 	addResp := &models.IpamAddResponse{
 		Ips:    resIPs,
 		Routes: resRoutes,
@@ -353,6 +393,7 @@ func (i *ipam) allocateIPsFromAllCandidates(ctx context.Context, tt ToBeAllocate
 		return results, utilerrors.NewAggregate(errs)
 	}
 
+	// the results are not in order by the NIC sequence right now
 	return results, nil
 }
 
@@ -537,7 +578,7 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 
 		podAnno := pod.GetAnnotations()
 		// the first NIC
-		if nic == constant.ClusterDefaultInterfaceName {
+		if nic == constant.ClusterDefaultInterfaceName || nic == strconv.Itoa(0) {
 			// default net-attach-def specified in the annotations
 			defaultMultusObj := podAnno[constant.MultusDefaultNetAnnot]
 			if len(defaultMultusObj) == 0 {
@@ -572,7 +613,9 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 					networkSelectionElements[idx].InterfaceRequest = fmt.Sprintf("net%d", idx+1)
 				}
 
-				if nic == networkSelectionElements[idx].InterfaceRequest {
+				// We regard the NIC name was specified by the user for the previous judgement.
+				// For the latter judgement(multiple NIC with no name specified mode), we just need to check whether the sequence is same with the net-attach-def resource
+				if (nic == networkSelectionElements[idx].InterfaceRequest) || (nic == strconv.Itoa(idx)) {
 					multusNS = networkSelectionElements[idx].Namespace
 					multusName = networkSelectionElements[idx].Name
 					isFound = true

pkg/ipam/pool_selections.go

@@ -49,7 +49,7 @@ func (i *ipam) getPoolCandidates(ctx context.Context, addArgs *models.IpamAddArg
 
 	// Select IPPool candidates through the Pod annotation "ipam.spidernet.io/ippools".
 	if anno, ok := pod.Annotations[constant.AnnoPodIPPools]; ok {
-		return getPoolFromPodAnnoPools(ctx, anno, *addArgs.IfName)
+		return getPoolFromPodAnnoPools(ctx, anno)
 	}
 
 	// Select IPPool candidates through the Pod annotation "ipam.spidernet.io/ippool".
@@ -312,7 +312,7 @@ func (i *ipam) applyThirdControllerAutoPool(ctx context.Context, subnetName stri
 	return pool, nil
 }
 
-func getPoolFromPodAnnoPools(ctx context.Context, anno, nic string) (ToBeAllocateds, error) {
+func getPoolFromPodAnnoPools(ctx context.Context, anno string) (ToBeAllocateds, error) {
 	logger := logutils.FromContext(ctx)
 	logger.Sugar().Infof("Use IPPools from Pod annotation '%s'", constant.AnnoPodIPPools)
 
@@ -322,23 +322,11 @@ func getPoolFromPodAnnoPools(ctx context.Context, anno, nic string) (ToBeAllocat
 	if err != nil {
 		return nil, fmt.Errorf("%w: %v", errPrefix, err)
 	}
-	if len(annoPodIPPools) == 0 {
-		return nil, fmt.Errorf("%w: value requires at least one item", errPrefix)
-	}
-
-	nicSet := map[string]struct{}{}
-	for _, v := range annoPodIPPools {
-		if v.NIC == "" {
-			return nil, fmt.Errorf("%w: interface must be specified", errPrefix)
-		}
-		if _, ok := nicSet[v.NIC]; ok {
-			return nil, fmt.Errorf("%w: duplicate interface %s", errPrefix, v.NIC)
-		}
-		nicSet[v.NIC] = struct{}{}
-	}
 
-	if _, ok := nicSet[nic]; !ok {
-		return nil, fmt.Errorf("%w: interfaces do not contain that requested by runtime", errPrefix)
+	// validate and mutate the IPPools annotation value
+	err = validateAndMutateMultipleNICAnnotations(annoPodIPPools)
+	if nil != err {
+		return nil, fmt.Errorf("%w: %v", errPrefix, err)
 	}
 
 	var tt ToBeAllocateds

pkg/ipam/utils.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"strconv"
 
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -161,3 +162,57 @@ func isPoolIPsDesired(pool *spiderpoolv2beta1.SpiderIPPool, desiredIPCount int)
 
 	return false
 }
+
+func IsMultipleNicWithNoName(anno map[string]string) bool {
+	annoVal, ok := anno[constant.AnnoPodIPPools]
+	if !ok {
+		return false
+	}
+
+	var annoPodIPPools types.AnnoPodIPPoolsValue
+	err := json.Unmarshal([]byte(annoVal), &annoPodIPPools)
+	if nil != err {
+		return false
+	}
+	if len(annoPodIPPools) == 0 {
+		return false
+	}
+
+	result := true
+	for _, v := range annoPodIPPools {
+		if v.NIC != "" {
+			result = false
+		}
+	}
+
+	return result
+}
+
+func validateAndMutateMultipleNICAnnotations(annoIPPoolsValue types.AnnoPodIPPoolsValue) error {
+	if len(annoIPPoolsValue) == 0 {
+		return fmt.Errorf("value requires at least one item")
+	}
+
+	nicSet := map[string]struct{}{}
+	isEmpty := annoIPPoolsValue[0].NIC == ""
+	for index := range annoIPPoolsValue {
+		// require all item NIC should be same in specified or unspecified.
+		if (annoIPPoolsValue[index].NIC == "") != isEmpty {
+			return fmt.Errorf("NIC should be either all specified or all unspecified")
+		}
+
+		// once we met the unspecified NIC in multiple NIC with no name specified mode, we give it an index name.
+		// since the latter allocation will be executed in concurrency for all NICs in the first cmdAdd,
+		// we could sort the allocation results in sequence by the NIC name.
+		if annoIPPoolsValue[index].NIC == "" {
+			annoIPPoolsValue[index].NIC = strconv.Itoa(index)
+		}
+
+		// require no NIC name duplicated, this works for multiple NIC specified by the users
+		if _, ok := nicSet[annoIPPoolsValue[index].NIC]; ok {
+			return fmt.Errorf("duplicate interface %s", annoIPPoolsValue[index].NIC)
+		}
+	}
+
+	return nil
+}

pkg/ippoolmanager/ippool_manager.go

@@ -109,7 +109,7 @@ func (im *ipPoolManager) AllocateIP(ctx context.Context, poolName, nic string, p
 		}
 
 		logger.Debug("Generate a random IP address")
-		allocatedIP, err := im.genRandomIP(ctx, nic, ipPool, pod, podController)
+		allocatedIP, err := im.genRandomIP(ctx, ipPool, pod, podController)
 		if err != nil {
 			return err
 		}
@@ -139,7 +139,7 @@ func (im *ipPoolManager) AllocateIP(ctx context.Context, poolName, nic string, p
 	return ipConfig, nil
 }
 
-func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *spiderpoolv2beta1.SpiderIPPool, pod *corev1.Pod, podController types.PodTopController) (net.IP, error) {
+func (im *ipPoolManager) genRandomIP(ctx context.Context, ipPool *spiderpoolv2beta1.SpiderIPPool, pod *corev1.Pod, podController types.PodTopController) (net.IP, error) {
 	logger := logutils.FromContext(ctx)
 
 	var tmpPod *corev1.Pod
@@ -182,7 +182,7 @@ func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *sp
 	if len(availableIPs) == 0 {
 		// traverse the usedIPs to find the previous allocated IPs if there be
 		// reference issue: https://github.com/spidernet-io/spiderpool/issues/2517
-		allocatedIPFromRecords, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, key, string(pod.UID))
+		allocatedIPFromRecords, hasFound := findAllocatedIPFromRecords(allocatedRecords, key, string(pod.UID))
 		if !hasFound {
 			return nil, constant.ErrIPUsedOut
 		}
@@ -199,7 +199,6 @@ func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *sp
 		allocatedRecords = spiderpoolv2beta1.PoolIPAllocations{}
 	}
 	allocatedRecords[resIP.String()] = spiderpoolv2beta1.PoolIPAllocation{
-		NIC:            nic,
 		NamespacedName: key,
 		PodUID:         string(pod.UID),
 	}

pkg/ippoolmanager/ippool_manager_test.go

@@ -433,7 +433,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 
 				records := spiderpoolv2beta1.PoolIPAllocations{
 					ip.String(): spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            nic,
 						NamespacedName: key,
 						PodUID:         string(podT.UID),
 					},
@@ -473,7 +472,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 				uid = string(uuid.NewUUID())
 				records = spiderpoolv2beta1.PoolIPAllocations{
 					ip: spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            "eth0",
 						NamespacedName: "default/pod",
 						PodUID:         uid,
 					},
@@ -564,7 +562,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 				uid = string(uuid.NewUUID())
 				records = spiderpoolv2beta1.PoolIPAllocations{
 					ip: spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            "eth0",
 						NamespacedName: "default/pod",
 						PodUID:         uid,
 					},

pkg/ippoolmanager/ippool_webhook_test.go

@@ -1822,7 +1822,6 @@ var _ = Describe("IPPoolWebhook", Label("ippool_webhook_test"), func() {
 					data, err := convert.MarshalIPPoolAllocatedIPs(
 						spiderpoolv2beta1.PoolIPAllocations{
 							"172.18.40.10": spiderpoolv2beta1.PoolIPAllocation{
-								NIC:            "eth0",
 								NamespacedName: "default/pod",
 								PodUID:         string(uuid.NewUUID()),
 							},

pkg/ippoolmanager/utils.go

@@ -126,10 +126,9 @@ func (b ByPoolPriority) Less(i, j int) bool {
 
 // findAllocatedIPFromRecords try to find pod NIC previous allocated IP from the IPPool.Status.AllocatedIPs
 // this function serves for the issue: https://github.com/spidernet-io/spiderpool/issues/2517
-func findAllocatedIPFromRecords(allocatedRecords spiderpoolv2beta1.PoolIPAllocations, nic, namespacedName, podUID string) (previousIP string, hasFound bool) {
+func findAllocatedIPFromRecords(allocatedRecords spiderpoolv2beta1.PoolIPAllocations, namespacedName, podUID string) (previousIP string, hasFound bool) {
 	for tmpIP, poolIPAllocation := range allocatedRecords {
-		if poolIPAllocation.NIC == nic &&
-			poolIPAllocation.NamespacedName == namespacedName &&
+		if poolIPAllocation.NamespacedName == namespacedName &&
 			poolIPAllocation.PodUID == podUID {
 			return tmpIP, true
 		}