Merge pull request #1152 from DanielXiao/ipaddressallocation

Replace IPPool with IPAddressAllocation API to Claim Pod CIDR in the VPC network

go.mod

@@ -1,8 +1,8 @@
 module k8s.io/cloud-provider-vsphere
 
-go 1.22.0
+go 1.22.5
 
-toolchain go1.22.3
+toolchain go1.22.6
 
 replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v1.8.6
 
@@ -15,13 +15,14 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
-	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20231227030711-34390278f3cd
-	github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240103020117-417d1d01f14b
+	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816024231-ca8ea47d918d
+	github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240816024231-ca8ea47d918d
 	github.com/vmware-tanzu/vm-operator/api v1.8.6
 	github.com/vmware/govmomi v0.42.0
 	github.com/vmware/vsphere-automation-sdk-go/lib v0.7.0
 	github.com/vmware/vsphere-automation-sdk-go/runtime v0.7.0
 	github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.12.0
+	golang.org/x/net v0.26.0
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.31.0
@@ -102,7 +103,6 @@ require (
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect

go.sum

@@ -170,10 +170,16 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20231227030711-34390278f3cd h1:rl/Yg5m2WrecTBHvf1ZSg/lMBOguIWOeaCf1uo5ARTQ=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20231227030711-34390278f3cd/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240103020117-417d1d01f14b h1:QRC2/l0VwKDVpe5bQFYR4oX1Fad6Ch5FlDaQmkgLW1g=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240103020117-417d1d01f14b/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240722023446-b3932c52a8d9 h1:OKX8/g78hSESdqSmj/x9txl+mQf6Z5729mzo+Fn7kuk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240722023446-b3932c52a8d9/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240815061344-52b9aad78304 h1:EwBVU0fh4fL/dwX+XmKBSzIvkc5eWS5C4C/pczBTRPs=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240815061344-52b9aad78304/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816024231-ca8ea47d918d h1:xxx1jmQDDdNTG3eSFlTaehbVQMrLnXQfzpty+DwAWj0=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816024231-ca8ea47d918d/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240722023446-b3932c52a8d9 h1:3fFxEDDAuEPthEokGzdH+mHTIIuD9epUX2cLRkLksFo=
+github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240722023446-b3932c52a8d9/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
+github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240816024231-ca8ea47d918d h1:8XztAkqq7nKQTBzusKuXLdDCFQ/HAFt7JU4kNlBCYuY=
+github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240816024231-ca8ea47d918d/go.mod h1:Hsn4QLNjQA6wQYGI6IlQJrTqM+29KWCVdyFgS7a2Sck=
 github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU=
 github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A=
 github.com/vmware/govmomi v0.42.0 h1:MbvAlVfjNBE1mHMaQ7yOSop1KLB0/93x6VAGuCtjqtI=

pkg/cloudprovider/vsphereparavirtual/config.go

@@ -25,6 +25,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
+
+	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/nsxipmanager"
 )
 
 const (
@@ -47,10 +49,6 @@ const (
 	SupervisorServiceAccountNameEnv string = "SUPERVISOR_CLUSTER_SERVICEACCOUNT_SECRET_NAME"
 	// SupervisorAPIServerFQDN reads supervisor service API server's fully qualified domain name from env
 	SupervisorAPIServerFQDN string = "supervisor.default.svc"
-	// PublicIPPoolType allows Pod IP address routable outside of Tier 0 router.
-	PublicIPPoolType = "Public"
-	// PrivateIPPoolType allows Pod IP address routable within VPC router.
-	PrivateIPPoolType = "Private"
 )
 
 // SupervisorEndpoint is the supervisor cluster endpoint
@@ -146,7 +144,7 @@ func checkPodIPPoolType(vpcModeEnabled bool, podIPPoolType string) error {
 			return errors.New("--pod-ip-pool-type is required in the NSX-T VPC network")
 		}
 
-		if podIPPoolType != PublicIPPoolType && podIPPoolType != PrivateIPPoolType {
+		if podIPPoolType != nsxipmanager.PublicIPPoolType && podIPPoolType != nsxipmanager.PrivateIPPoolType {
 			return errors.New("--pod-ip-pool-type can be either Public or Private in NSX-T VPC network, " + podIPPoolType + " is not supported")
 
 		}

pkg/cloudprovider/vsphereparavirtual/controllers/routablepod/core.go

@@ -17,14 +17,19 @@ import (
 	"context"
 	"fmt"
 
+	nsxclients "github.com/vmware-tanzu/nsx-operator/pkg/client/clientset/versioned"
+	nsxinformers "github.com/vmware-tanzu/nsx-operator/pkg/client/informers/externalversions"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 
+	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/controllers/routablepod/ipaddressallocation"
 	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/controllers/routablepod/ippool"
 	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/controllers/routablepod/node"
-	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/ippoolmanager"
+	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/ippoolmanager/helper"
+	ippmv1alpha1 "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/ippoolmanager/v1alpha1"
+	"k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphereparavirtual/nsxipmanager"
 	k8s "k8s.io/cloud-provider-vsphere/pkg/common/kubernetes"
 )
 
@@ -42,18 +47,59 @@ func StartControllers(scCfg *rest.Config, client kubernetes.Interface,
 
 	klog.V(2).Info("Routable pod controllers start with VPC mode enabled: ", vpcModeEnabled)
 
-	ippManager, err := ippoolmanager.GetIPPoolManager(vpcModeEnabled, scCfg, clusterNS, podIPPoolType)
-	if err != nil {
-		return fmt.Errorf("fail to get ippool manager or start ippool controller: %w", err)
-	}
+	ctx := informerManager.GetContext()
+	var nsxIPManager nsxipmanager.NSXIPManager
+	if vpcModeEnabled {
+		nsxClient, nsxInformerFactory, err := getNSXClientAndInformer(scCfg, clusterNS)
+		if err != nil {
+			return fmt.Errorf("fail to get NSX client or informer factory: %w", err)
+		}
 
-	ippoolController := ippool.NewController(client, ippManager)
-	go ippoolController.Run(context.Background().Done())
+		startIPAddressAllocationController(ctx, client, informerManager, nsxInformerFactory)
 
-	ippManager.StartIPPoolInformers()
+		nsxIPManager = nsxipmanager.NewNSXVPCIPManager(nsxClient, nsxInformerFactory, clusterNS, podIPPoolType, ownerRef)
+	} else {
+		ippManager, err := ippmv1alpha1.NewIPPoolManager(scCfg, clusterNS)
+		if err != nil {
+			return fmt.Errorf("fail to get ippool manager or start ippool controller: %w", err)
+		}
 
-	nodeController := node.NewController(client, ippManager, informerManager, clusterName, clusterNS, ownerRef)
+		startIPPoolController(ctx, client, ippManager)
+
+		nsxIPManager = nsxipmanager.NewNSXT1IPManager(ippManager, clusterName, clusterNS, ownerRef)
+	}
+
+	nodeController := node.NewController(client, nsxIPManager, informerManager, clusterName, clusterNS, ownerRef)
 	go nodeController.Run(context.Background().Done())
 
 	return nil
 }
+
+func startIPAddressAllocationController(ctx context.Context, client kubernetes.Interface, informerManager *k8s.InformerManager, nsxInformerFactory nsxinformers.SharedInformerFactory) {
+	ipAddressAllocationController := ipaddressallocation.NewController(
+		ctx,
+		client,
+		informerManager.GetNodeLister(),
+		informerManager.IsNodeInformerSynced(),
+		nsxInformerFactory.Crd().V1alpha1().IPAddressAllocations())
+	go ipAddressAllocationController.Run(ctx, 1)
+	nsxInformerFactory.Start(ctx.Done())
+}
+
+func startIPPoolController(ctx context.Context, client kubernetes.Interface, ippManager *ippmv1alpha1.IPPoolManager) {
+	ippoolController := ippool.NewController(client, ippManager)
+	go ippoolController.Run(ctx.Done())
+	ippManager.StartIPPoolInformers(ctx.Done())
+}
+
+func getNSXClientAndInformer(svCfg *rest.Config, svNamespace string) (nsxclients.Interface, nsxinformers.SharedInformerFactory, error) {
+	client, err := nsxclients.NewForConfig(svCfg)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error building nsx-operator clientset: %w", err)
+	}
+
+	informerFactory := nsxinformers.NewSharedInformerFactoryWithOptions(client,
+		helper.DefaultResyncTime, nsxinformers.WithNamespace(svNamespace))
+
+	return client, informerFactory, nil
+}