support multiple NIC with name specified

Signed-off-by: Icarus9913 <[email protected]>

pkg/ipam/allocate.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -79,7 +80,7 @@ func (i *ipam) Allocate(ctx context.Context, addArgs *models.IpamAddArgs) (*mode
 		}
 	} else {
 		logger.Debug("Try to retrieve the existing IP allocation")
-		addResp, err := i.retrieveExistingIPAllocation(ctx, string(pod.UID), *addArgs.IfName, endpoint)
+		addResp, err := i.retrieveExistingIPAllocation(ctx, string(pod.UID), *addArgs.IfName, endpoint, IsMultipleNicWithNoName(pod.Annotations))
 		if err != nil {
 			return nil, fmt.Errorf("failed to retrieve the existing IP allocation: %w", err)
 		}
@@ -113,7 +114,7 @@ func (i *ipam) retrieveStaticIPAllocation(ctx context.Context, nic string, pod *
 	}
 
 	logger.Info("Refresh the current IP allocation of the Endpoint")
-	if err := i.endpointManager.ReallocateCurrentIPAllocation(ctx, string(pod.UID), pod.Spec.NodeName, endpoint); err != nil {
+	if err := i.endpointManager.ReallocateCurrentIPAllocation(ctx, string(pod.UID), pod.Spec.NodeName, nic, endpoint, IsMultipleNicWithNoName(pod.Annotations)); err != nil {
 		return nil, fmt.Errorf("failed to refresh the current IP allocation of %s: %w", endpoint.Status.OwnerControllerType, err)
 	}
 
@@ -177,7 +178,7 @@ func (i *ipam) reallocateIPPoolIPRecords(ctx context.Context, uid string, endpoi
 	return nil
 }
 
-func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string, endpoint *spiderpoolv2beta1.SpiderEndpoint) (*models.IpamAddResponse, error) {
+func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string, endpoint *spiderpoolv2beta1.SpiderEndpoint, isMultipleNicWithNoName bool) (*models.IpamAddResponse, error) {
 	logger := logutils.FromContext(ctx)
 
 	// Create -> Delete -> Create a Pod with the same namespace and name in
@@ -193,6 +194,15 @@ func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string
 		return nil, nil
 	}
 
+	// update Endpoint NIC name in multiple NIC with no name mode by annotation "ipam.spidernet.io/ippools"
+	if isMultipleNicWithNoName {
+		var err error
+		allocation, err = i.endpointManager.UpdateAllocationNICName(ctx, endpoint, nic)
+		if nil != err {
+			return nil, fmt.Errorf("failed to update SpiderEndpoint allocation details NIC name %s, error: %v", nic, err)
+		}
+	}
+
 	ips, routes := convert.ConvertIPDetailsToIPConfigsAndAllRoutes(allocation.IPs)
 	addResp := &models.IpamAddResponse{
 		Ips:    ips,
@@ -205,6 +215,7 @@ func (i *ipam) retrieveExistingIPAllocation(ctx context.Context, uid, nic string
 
 func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamAddArgs, pod *corev1.Pod, endpoint *spiderpoolv2beta1.SpiderEndpoint, podController types.PodTopController) (*models.IpamAddResponse, error) {
 	logger := logutils.FromContext(ctx)
+	isMultipleNicWithNoName := IsMultipleNicWithNoName(pod.Annotations)
 
 	logger.Debug("Parse custom routes")
 	customRoutes, err := getCustomRoutes(pod)
@@ -220,7 +231,7 @@ func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamA
 
 	var results []*types.AllocationResult
 	defer func() {
-		if err != nil {
+		if err != nil && !isMultipleNicWithNoName {
 			if len(results) != 0 {
 				i.failure.addFailureIPs(string(pod.UID), results)
 			}
@@ -240,12 +251,37 @@ func (i *ipam) allocateInStandardMode(ctx context.Context, addArgs *models.IpamA
 		return nil, fmt.Errorf("failed to group custom routes %+v: %v", customRoutes, err)
 	}
 
+	// sort the results in order by NIC sequence in multiple NIC with no name spacefied mode
+	if IsMultipleNicWithNoName(pod.Annotations) {
+		sort.Slice(results, func(i, j int) bool {
+			pre, err := strconv.Atoi(*results[i].IP.Nic)
+			if nil != err {
+				return false
+			}
+			latter, err := strconv.Atoi(*results[j].IP.Nic)
+			if nil != err {
+				return false
+			}
+			return pre < latter
+		})
+	}
+
 	logger.Debug("Patch IP allocation results to Endpoint")
-	if err = i.endpointManager.PatchIPAllocationResults(ctx, results, endpoint, pod, podController); err != nil {
+	if err = i.endpointManager.PatchIPAllocationResults(ctx, results, endpoint, pod, podController, isMultipleNicWithNoName); err != nil {
 		return nil, fmt.Errorf("failed to patch IP allocation results to Endpoint: %v", err)
 	}
 
 	resIPs, resRoutes := convert.ConvertResultsToIPConfigsAndAllRoutes(results)
+	if isMultipleNicWithNoName {
+		for index := range resIPs {
+			if *resIPs[index].Nic == strconv.Itoa(0) {
+				*resIPs[index].Nic = constant.ClusterDefaultInterfaceName
+			}
+		}
+	}
+
+	// Actually in allocate Standard Mode, we just need the current turn NIC allocation result,
+	// but here are the all NICs results
 	addResp := &models.IpamAddResponse{
 		Ips:    resIPs,
 		Routes: resRoutes,
@@ -353,6 +389,7 @@ func (i *ipam) allocateIPsFromAllCandidates(ctx context.Context, tt ToBeAllocate
 		return results, utilerrors.NewAggregate(errs)
 	}
 
+	// the results are not in order by the NIC sequence right now
 	return results, nil
 }
 
@@ -537,7 +574,7 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 
 		podAnno := pod.GetAnnotations()
 		// the first NIC
-		if nic == constant.ClusterDefaultInterfaceName {
+		if nic == constant.ClusterDefaultInterfaceName || nic == strconv.Itoa(0) {
 			// default net-attach-def specified in the annotations
 			defaultMultusObj := podAnno[constant.MultusDefaultNetAnnot]
 			if len(defaultMultusObj) == 0 {
@@ -572,6 +609,7 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 					networkSelectionElements[idx].InterfaceRequest = fmt.Sprintf("net%d", idx+1)
 				}
 
+				// TODO 这里nic如果字段为空，或者为01234怎么办？？？？？？？？？？？？？
 				if nic == networkSelectionElements[idx].InterfaceRequest {
 					multusNS = networkSelectionElements[idx].Namespace
 					multusName = networkSelectionElements[idx].Name

pkg/ipam/pool_selections.go

@@ -49,7 +49,7 @@ func (i *ipam) getPoolCandidates(ctx context.Context, addArgs *models.IpamAddArg
 
 	// Select IPPool candidates through the Pod annotation "ipam.spidernet.io/ippools".
 	if anno, ok := pod.Annotations[constant.AnnoPodIPPools]; ok {
-		return getPoolFromPodAnnoPools(ctx, anno, *addArgs.IfName)
+		return getPoolFromPodAnnoPools(ctx, anno)
 	}
 
 	// Select IPPool candidates through the Pod annotation "ipam.spidernet.io/ippool".
@@ -312,7 +312,7 @@ func (i *ipam) applyThirdControllerAutoPool(ctx context.Context, subnetName stri
 	return pool, nil
 }
 
-func getPoolFromPodAnnoPools(ctx context.Context, anno, nic string) (ToBeAllocateds, error) {
+func getPoolFromPodAnnoPools(ctx context.Context, anno string) (ToBeAllocateds, error) {
 	logger := logutils.FromContext(ctx)
 	logger.Sugar().Infof("Use IPPools from Pod annotation '%s'", constant.AnnoPodIPPools)
 
@@ -322,23 +322,11 @@ func getPoolFromPodAnnoPools(ctx context.Context, anno, nic string) (ToBeAllocat
 	if err != nil {
 		return nil, fmt.Errorf("%w: %v", errPrefix, err)
 	}
-	if len(annoPodIPPools) == 0 {
-		return nil, fmt.Errorf("%w: value requires at least one item", errPrefix)
-	}
-
-	nicSet := map[string]struct{}{}
-	for _, v := range annoPodIPPools {
-		if v.NIC == "" {
-			return nil, fmt.Errorf("%w: interface must be specified", errPrefix)
-		}
-		if _, ok := nicSet[v.NIC]; ok {
-			return nil, fmt.Errorf("%w: duplicate interface %s", errPrefix, v.NIC)
-		}
-		nicSet[v.NIC] = struct{}{}
-	}
 
-	if _, ok := nicSet[nic]; !ok {
-		return nil, fmt.Errorf("%w: interfaces do not contain that requested by runtime", errPrefix)
+	// validate and mutate the IPPools annotation value
+	err = validateAndMutateMultipleNICAnnotations(annoPodIPPools)
+	if nil != err {
+		return nil, fmt.Errorf("%w: %v", errPrefix, err)
 	}
 
 	var tt ToBeAllocateds

pkg/ipam/utils.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"strconv"
 
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -161,3 +162,55 @@ func isPoolIPsDesired(pool *spiderpoolv2beta1.SpiderIPPool, desiredIPCount int)
 
 	return false
 }
+
+func IsMultipleNicWithNoName(anno map[string]string) bool {
+	annoVal, ok := anno[constant.AnnoPodIPPools]
+	if !ok {
+		return false
+	}
+
+	var annoPodIPPools types.AnnoPodIPPoolsValue
+	err := json.Unmarshal([]byte(annoVal), &annoPodIPPools)
+	if nil != err {
+		return false
+	}
+	if len(annoPodIPPools) == 0 {
+		return false
+	}
+
+	result := true
+	for _, v := range annoPodIPPools {
+		if v.NIC != "" {
+			result = false
+		}
+	}
+
+	return result
+}
+
+func validateAndMutateMultipleNICAnnotations(annoIPPoolsValue types.AnnoPodIPPoolsValue) error {
+	if len(annoIPPoolsValue) == 0 {
+		return fmt.Errorf("value requires at least one item")
+	}
+
+	nicSet := map[string]struct{}{}
+	isEmpty := annoIPPoolsValue[0].NIC == ""
+	for index := range annoIPPoolsValue {
+		// require all item NIC should be same in specified or unspecified.
+		if (annoIPPoolsValue[index].NIC == "") != isEmpty {
+			return fmt.Errorf("NIC should be either all specified or all unspecified")
+		}
+
+		// once we met the unspecified NIC in multiple NIC with no name specified mode, we give it an index name
+		if annoIPPoolsValue[index].NIC == "" {
+			annoIPPoolsValue[index].NIC = strconv.Itoa(index)
+		}
+
+		// require no NIC name duplicated, this works for multiple NIC specified by the users
+		if _, ok := nicSet[annoIPPoolsValue[index].NIC]; ok {
+			return fmt.Errorf("duplicate interface %s", annoIPPoolsValue[index].NIC)
+		}
+	}
+
+	return nil
+}

pkg/ippoolmanager/ippool_manager.go

@@ -109,7 +109,7 @@ func (im *ipPoolManager) AllocateIP(ctx context.Context, poolName, nic string, p
 		}
 
 		logger.Debug("Generate a random IP address")
-		allocatedIP, err := im.genRandomIP(ctx, nic, ipPool, pod, podController)
+		allocatedIP, err := im.genRandomIP(ctx, ipPool, pod, podController)
 		if err != nil {
 			return err
 		}
@@ -139,7 +139,7 @@ func (im *ipPoolManager) AllocateIP(ctx context.Context, poolName, nic string, p
 	return ipConfig, nil
 }
 
-func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *spiderpoolv2beta1.SpiderIPPool, pod *corev1.Pod, podController types.PodTopController) (net.IP, error) {
+func (im *ipPoolManager) genRandomIP(ctx context.Context, ipPool *spiderpoolv2beta1.SpiderIPPool, pod *corev1.Pod, podController types.PodTopController) (net.IP, error) {
 	logger := logutils.FromContext(ctx)
 
 	var tmpPod *corev1.Pod
@@ -182,7 +182,7 @@ func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *sp
 	if len(availableIPs) == 0 {
 		// traverse the usedIPs to find the previous allocated IPs if there be
 		// reference issue: https://github.com/spidernet-io/spiderpool/issues/2517
-		allocatedIPFromRecords, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, key, string(pod.UID))
+		allocatedIPFromRecords, hasFound := findAllocatedIPFromRecords(allocatedRecords, key, string(pod.UID))
 		if !hasFound {
 			return nil, constant.ErrIPUsedOut
 		}
@@ -199,7 +199,6 @@ func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *sp
 		allocatedRecords = spiderpoolv2beta1.PoolIPAllocations{}
 	}
 	allocatedRecords[resIP.String()] = spiderpoolv2beta1.PoolIPAllocation{
-		NIC:            nic,
 		NamespacedName: key,
 		PodUID:         string(pod.UID),
 	}

pkg/ippoolmanager/ippool_manager_test.go

@@ -245,6 +245,7 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 		})
 
 		Describe("AllocateIP", func() {
+			// TODO: NIC here???????????????
 			var nic string
 			var podT *corev1.Pod
 
@@ -433,7 +434,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 
 				records := spiderpoolv2beta1.PoolIPAllocations{
 					ip.String(): spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            nic,
 						NamespacedName: key,
 						PodUID:         string(podT.UID),
 					},
@@ -473,7 +473,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 				uid = string(uuid.NewUUID())
 				records = spiderpoolv2beta1.PoolIPAllocations{
 					ip: spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            "eth0",
 						NamespacedName: "default/pod",
 						PodUID:         uid,
 					},
@@ -564,7 +563,6 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 				uid = string(uuid.NewUUID())
 				records = spiderpoolv2beta1.PoolIPAllocations{
 					ip: spiderpoolv2beta1.PoolIPAllocation{
-						NIC:            "eth0",
 						NamespacedName: "default/pod",
 						PodUID:         uid,
 					},

pkg/ippoolmanager/ippool_webhook_test.go

@@ -1822,7 +1822,6 @@ var _ = Describe("IPPoolWebhook", Label("ippool_webhook_test"), func() {
 					data, err := convert.MarshalIPPoolAllocatedIPs(
 						spiderpoolv2beta1.PoolIPAllocations{
 							"172.18.40.10": spiderpoolv2beta1.PoolIPAllocation{
-								NIC:            "eth0",
 								NamespacedName: "default/pod",
 								PodUID:         string(uuid.NewUUID()),
 							},

pkg/ippoolmanager/utils.go

@@ -126,10 +126,9 @@ func (b ByPoolPriority) Less(i, j int) bool {
 
 // findAllocatedIPFromRecords try to find pod NIC previous allocated IP from the IPPool.Status.AllocatedIPs
 // this function serves for the issue: https://github.com/spidernet-io/spiderpool/issues/2517
-func findAllocatedIPFromRecords(allocatedRecords spiderpoolv2beta1.PoolIPAllocations, nic, namespacedName, podUID string) (previousIP string, hasFound bool) {
+func findAllocatedIPFromRecords(allocatedRecords spiderpoolv2beta1.PoolIPAllocations, namespacedName, podUID string) (previousIP string, hasFound bool) {
 	for tmpIP, poolIPAllocation := range allocatedRecords {
-		if poolIPAllocation.NIC == nic &&
-			poolIPAllocation.NamespacedName == namespacedName &&
+		if poolIPAllocation.NamespacedName == namespacedName &&
 			poolIPAllocation.PodUID == podUID {
 			return tmpIP, true
 		}

pkg/ippoolmanager/utils_test.go

@@ -294,44 +294,37 @@ var _ = Describe("IPPoolManager-utils", Label("ippool_manager_utils"), func() {
 
 	Context("Test findAllocatedIPFromRecords", func() {
 		var allocatedRecords spiderpoolv2beta1.PoolIPAllocations
-		var ip, nic, namespacedName, podUID string
+		var ip, namespacedName, podUID string
 
 		BeforeEach(func() {
 			ip = "172.18.40.40"
-			nic = "eth0"
 			namespacedName = "default/testPod"
 			podUID = string(uuid.NewUUID())
 			allocatedRecords = spiderpoolv2beta1.PoolIPAllocations{
 				ip: spiderpoolv2beta1.PoolIPAllocation{
-					NIC:            nic,
 					NamespacedName: namespacedName,
 					PodUID:         podUID,
 				},
 			}
 		})
 
 		It("find previous allocated IP in the records", func() {
-			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, namespacedName, podUID)
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, namespacedName, podUID)
 			Expect(hasFound).To(BeTrue())
 		})
 
 		It("no previous allocated IP in the records", func() {
-			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "eth1", "kube-system/testPod1", string(uuid.NewUUID()))
-			Expect(hasFound).To(BeFalse())
-		})
-
-		It("no previous allocated IP in records due to different nic", func() {
-			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "eth1", namespacedName, podUID)
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "kube-system/testPod1", string(uuid.NewUUID()))
 			Expect(hasFound).To(BeFalse())
 		})
 
 		It("no previous allocated IP in records due to different NamespacedName", func() {
-			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, "kube-system/testPod1", podUID)
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "kube-system/testPod1", podUID)
 			Expect(hasFound).To(BeFalse())
 		})
 
 		It("no previous allocated IP in records due to different podUID", func() {
-			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, namespacedName, string(uuid.NewUUID()))
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, namespacedName, string(uuid.NewUUID()))
 			Expect(hasFound).To(BeFalse())
 		})
 	})

pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spiderippool_types.go

@@ -87,7 +87,6 @@ type IPPoolStatus struct {
 type PoolIPAllocations map[string]PoolIPAllocation
 
 type PoolIPAllocation struct {
-	NIC            string `json:"interface"`
 	NamespacedName string `json:"pod"`
 	PodUID         string `json:"podUid"`
 }