Add support for clustersetIP

1. Add ClustersetIP flags to deploy-broker and join.
2. Add ClustersetIP CIDRs to config CRs.
3. Show clustersetupCIDR in `show networks` output.
4. Add use-clusterset-ip flag to `service export`

Refer: submariner-io/enhancements#230

Signed-off-by: Vishal Thapar <[email protected]>

cmd/subctl/deploybroker.go

@@ -36,6 +36,7 @@ import (
 	"github.com/submariner-io/subctl/pkg/broker"
 	"github.com/submariner-io/subctl/pkg/cluster"
 	"github.com/submariner-io/subctl/pkg/deploy"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/clustersetip"
 	"github.com/submariner-io/submariner-operator/pkg/discovery/globalnet"
 	"k8s.io/utils/set"
 )
@@ -89,6 +90,10 @@ func addDeployBrokerFlags(flags *pflag.FlagSet) {
 
 	flags.StringVar(&deployflags.BrokerURL, "broker-url", "",
 		"broker API endpoint URL (stored in the broker information file, defaults to the context URL)")
+	flags.BoolVar(&deployflags.BrokerSpec.ClustersetIPEnabled, "enable-clusterset-ip", false,
+		"set default support for use of clusterset IP for exported services in connecting clusters (default disabled)")
+	flags.StringVar(&deployflags.BrokerSpec.ClustersetIPCIDRRange, "clusterset-ip-cidr-range",
+		clustersetip.DefaultCIDR, "Clusterset IP CIDR supernet range for allocating Clusterset IP CIDRs to each cluster")
 }
 
 func deployBrokerInContext(clusterInfo *cluster.Info, namespace string, status reporter.Interface) error {

cmd/subctl/export.go

@@ -32,6 +32,7 @@ import (
 )
 
 var (
+	useClustersetIP          string
 	exportRestConfigProducer = restconfig.NewProducer().WithNamespace()
 
 	exportCmd = &cobra.Command{
@@ -50,7 +51,7 @@ var (
 
 			exit.OnError(exportRestConfigProducer.RunOnSelectedContext(
 				func(clusterInfo *cluster.Info, namespace string, status reporter.Interface) error {
-					return service.Export(clusterInfo.ClientProducer, namespace, args[0], status)
+					return service.Export(clusterInfo.ClientProducer, namespace, args[0], useClustersetIP, status)
 				}, cli.NewReporter()))
 		},
 	}
@@ -60,7 +61,7 @@ func init() {
 	err := mcsv1a1.Install(scheme.Scheme)
 	exit.OnErrorWithMessage(err, "Failed to add to scheme")
 
-	exportRestConfigProducer.SetupFlags(exportServiceCmd.Flags())
+	addExportServiceFlags(exportServiceCmd)
 	exportCmd.AddCommand(exportServiceCmd)
 	rootCmd.AddCommand(exportCmd)
 }
@@ -72,3 +73,10 @@ func validateArguments(args []string) error {
 
 	return nil
 }
+
+func addExportServiceFlags(cmd *cobra.Command) {
+	const clustersetIPFlag = "use-clusterset-ip"
+	exportRestConfigProducer.SetupFlags(exportServiceCmd.Flags())
+	cmd.PersistentFlags().StringVar(&useClustersetIP, clustersetIPFlag, "", "use clusterset IP for this service")
+	cmd.PersistentFlags().Lookup(clustersetIPFlag).NoOptDefVal = "true"
+}

cmd/subctl/join.go

@@ -147,6 +147,10 @@ func addJoinFlags(cmd *cobra.Command) {
 		"check the broker certificate (disable this to allow \"insecure\" connections)")
 	cmd.Flags().StringVar(&joinFlags.BrokerURL, "broker-url", "",
 		"URL of the broker API endpoint (overrides the URL stored in the broker information file)")
+	cmd.Flags().BoolVar(&joinFlags.EnableClustersetIP, "enable-clusterset-ip", false,
+		"set default support for use of clusterset IP for exported services in the cluster (default disabled)")
+	cmd.Flags().StringVar(&joinFlags.ClustersetIPCIDR, "clusterset-ip-cidr", "",
+		"Clusterset IP CIDR to be allocated to the cluster")
 }
 
 func joinInContext(brokerInfo *broker.Info, clusterInfo *cluster.Info, status reporter.Interface) error {

go.mod

@@ -18,16 +18,16 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/submariner-io/admiral v0.19.0-m3
 	github.com/submariner-io/cloud-prepare v0.19.0-m3
-	github.com/submariner-io/lighthouse v0.19.0-m3
+	github.com/submariner-io/lighthouse v0.19.0-m3.0.20240923085330-70fcb8443189
 	github.com/submariner-io/shipyard v0.19.0-m3
 	github.com/submariner-io/submariner v0.19.0-m3.0.20240917155703-5a6c358065a2
-	github.com/submariner-io/submariner-operator v0.19.0-m3
+	github.com/submariner-io/submariner-operator v0.19.0-m3.0.20240923150922-268a9960c6b7
 	github.com/uw-labs/lichen v0.1.7
 	golang.org/x/net v0.29.0
 	golang.org/x/oauth2 v0.23.0
 	google.golang.org/api v0.195.0
 	k8s.io/api v0.31.1
-	k8s.io/apiextensions-apiserver v0.31.0
+	k8s.io/apiextensions-apiserver v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.1
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8

go.sum

@@ -518,14 +518,14 @@ github.com/submariner-io/admiral v0.19.0-m3 h1:LTkYxCvB8S1210P2FZtCb6dzjaPpIgBrR
 github.com/submariner-io/admiral v0.19.0-m3/go.mod h1:xRpP1rDOblEdPHr0qrC+plcTNfShYJAOH2fexqOmI1A=
 github.com/submariner-io/cloud-prepare v0.19.0-m3 h1:f2PR4fFSJnwI5Ta9gTSmH0+Y2ZgES+hmBvAF29j45vM=
 github.com/submariner-io/cloud-prepare v0.19.0-m3/go.mod h1:LEyZLtFxBytG73MPS2kvmF/ASoNmVPIKrEjroWJFSOQ=
-github.com/submariner-io/lighthouse v0.19.0-m3 h1:CDv7V6lM/ixurJKvM9H9D2ckVXD9bJpY4F2IHPHcp/8=
-github.com/submariner-io/lighthouse v0.19.0-m3/go.mod h1:SA5PyBm+pM2Dx2MgWFNz/eJPN3Wde4BrnNWysWQzBRQ=
+github.com/submariner-io/lighthouse v0.19.0-m3.0.20240923085330-70fcb8443189 h1:ZymF4Ymmb0eaZXSICnBC3amzWgqHwKwZR87CAC/KLIw=
+github.com/submariner-io/lighthouse v0.19.0-m3.0.20240923085330-70fcb8443189/go.mod h1:wJNaL1Nk0Cli01KWAIqhy4Ww2a5OUDA0dT7qMn4OlkQ=
 github.com/submariner-io/shipyard v0.19.0-m3 h1:NliwAktRPF4OsLj1TDgpaOJD/bmmZW/FH9+mJmWgxbk=
 github.com/submariner-io/shipyard v0.19.0-m3/go.mod h1:BY1ceSnPz1/hN5F9uljcSzy5n5qgAOENsIvZpJ+XPOU=
 github.com/submariner-io/submariner v0.19.0-m3.0.20240917155703-5a6c358065a2 h1:SffTAy7zUR6fOr41EIGccVZHRNuExA78b7lLr84qizg=
 github.com/submariner-io/submariner v0.19.0-m3.0.20240917155703-5a6c358065a2/go.mod h1:hKbs5L9QPDslJ6n4k3fsPRbr7JbpT5AVr58YgWQQCKQ=
-github.com/submariner-io/submariner-operator v0.19.0-m3 h1:Op0KzkYvXK6cF3cG7KgeaRxKCfj5FJHeUq+s3JetOvI=
-github.com/submariner-io/submariner-operator v0.19.0-m3/go.mod h1:XP9FHppFlUyUkoanMOzxJUpDNXZR8hr99cUNqfb2Rug=
+github.com/submariner-io/submariner-operator v0.19.0-m3.0.20240923150922-268a9960c6b7 h1:grcgOt7T/fStYdOOK/E+cmAm/dm59KNxOm5kc0fpYrQ=
+github.com/submariner-io/submariner-operator v0.19.0-m3.0.20240923150922-268a9960c6b7/go.mod h1:OAbOn8vkkVtrGKTQ92aT7HNtAxCH5wHhPjb2t/8qKVM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -794,8 +794,8 @@ k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU=
 k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI=
 k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY=
 k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio=
-k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=
-k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
+k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40=
+k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ=
 k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
 k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=

internal/show/network.go

@@ -38,10 +38,11 @@ func Network(clusterInfo *cluster.Info, _ string, status reporter.Interface) err
 	if clusterInfo.Submariner != nil {
 		msg = "    Discovered network details via Submariner:"
 		clusterNetwork = &network.ClusterNetwork{
-			PodCIDRs:      []string{clusterInfo.Submariner.Status.ClusterCIDR},
-			ServiceCIDRs:  []string{clusterInfo.Submariner.Status.ServiceCIDR},
-			NetworkPlugin: clusterInfo.Submariner.Status.NetworkPlugin,
-			GlobalCIDR:    clusterInfo.Submariner.Status.GlobalCIDR,
+			PodCIDRs:         []string{clusterInfo.Submariner.Status.ClusterCIDR},
+			ServiceCIDRs:     []string{clusterInfo.Submariner.Status.ServiceCIDR},
+			NetworkPlugin:    clusterInfo.Submariner.Status.NetworkPlugin,
+			GlobalCIDR:       clusterInfo.Submariner.Status.GlobalCIDR,
+			ClustersetIPCIDR: clusterInfo.Submariner.Status.ClustersetIPCIDR,
 		}
 	} else {
 		msg = "    Discovered network details"

pkg/deploy/broker.go

@@ -31,7 +31,9 @@ import (
 	"github.com/submariner-io/subctl/pkg/image"
 	"github.com/submariner-io/subctl/pkg/operator"
 	operatorv1alpha1 "github.com/submariner-io/submariner-operator/api/v1alpha1"
+	"github.com/submariner-io/submariner-operator/pkg/cidr"
 	"github.com/submariner-io/submariner-operator/pkg/crd"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/clustersetip"
 	"github.com/submariner-io/submariner-operator/pkg/discovery/globalnet"
 	"golang.org/x/net/http/httpproxy"
 	"k8s.io/utils/set"
@@ -66,6 +68,10 @@ func Broker(options *BrokerOptions, clientProducer client.Producer, status repor
 		return status.Error(err, "invalid GlobalCIDR configuration")
 	}
 
+	if err := cidr.IsValid(options.BrokerSpec.ClustersetIPCIDRRange); err != nil {
+		return status.Error(err, "invalid ClustersetIP configuration")
+	}
+
 	err := Deploy(ctx, options, status, clientProducer)
 	if err != nil {
 		return err
@@ -82,6 +88,15 @@ func Broker(options *BrokerOptions, clientProducer client.Producer, status repor
 		return status.Error(err, "error creating globalCIDR configmap on Broker")
 	}
 
+	if err = clustersetip.ValidateExistingClustersetIPNetworks(ctx, clientProducer.ForGeneral(), options.BrokerNamespace); err != nil {
+		return status.Error(err, "error validating existing clustersetIPCIDR configmap")
+	}
+
+	if err = clustersetip.CreateConfigMap(ctx, clientProducer.ForGeneral(), options.BrokerSpec.ClustersetIPEnabled,
+		options.BrokerSpec.ClustersetIPCIDRRange, 0, options.BrokerNamespace); err != nil {
+		return status.Error(err, "error creating clustersetIPCIDR configmap on Broker")
+	}
+
 	return nil
 }
 
@@ -136,11 +151,11 @@ func checkGlobalnetConfig(options *BrokerOptions) error {
 		return nil
 	}
 
-	options.BrokerSpec.DefaultGlobalnetClusterSize, err = globalnet.GetValidClusterSize(options.BrokerSpec.GlobalnetCIDRRange,
+	options.BrokerSpec.DefaultGlobalnetClusterSize, err = cidr.GetValidAllocationSize(options.BrokerSpec.GlobalnetCIDRRange,
 		options.BrokerSpec.DefaultGlobalnetClusterSize)
 	if err != nil {
 		return err
 	}
 
-	return globalnet.IsValidCIDR(options.BrokerSpec.GlobalnetCIDRRange)
+	return cidr.IsValid(options.BrokerSpec.GlobalnetCIDRRange)
 }

pkg/deploy/servicediscovery.go

@@ -29,24 +29,28 @@ import (
 	"github.com/submariner-io/subctl/pkg/image"
 	"github.com/submariner-io/subctl/pkg/servicediscoverycr"
 	operatorv1alpha1 "github.com/submariner-io/submariner-operator/api/v1alpha1"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/clustersetip"
 	v1 "k8s.io/api/core/v1"
 	controllerClient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type ServiceDiscoveryOptions struct {
 	SubmarinerDebug        bool
 	BrokerK8sInsecure      bool
+	ClustersetIPEnabled    bool
 	ClusterID              string
 	CoreDNSCustomConfigMap string
 	Repository             string
 	ImageVersion           string
 	CustomDomains          []string
 }
 
-func ServiceDiscovery(ctx context.Context, clientProducer client.Producer, options *ServiceDiscoveryOptions, brokerInfo *broker.Info,
-	brokerSecret *v1.Secret, repositoryInfo *image.RepositoryInfo, status reporter.Interface,
+func ServiceDiscovery(ctx context.Context, clientProducer client.Producer, options *ServiceDiscoveryOptions,
+	brokerInfo *broker.Info, brokerSecret *v1.Secret, clustersetConfig clustersetip.Config,
+	repositoryInfo *image.RepositoryInfo, status reporter.Interface,
 ) error {
-	serviceDiscoverySpec := populateServiceDiscoverySpec(options, brokerInfo, brokerSecret, repositoryInfo)
+	serviceDiscoverySpec := populateServiceDiscoverySpec(options, brokerInfo, brokerSecret, clustersetConfig,
+		repositoryInfo)
 
 	err := ServiceDiscoveryFromSpec(ctx, clientProducer.ForGeneral(), serviceDiscoverySpec)
 	if err != nil {
@@ -65,7 +69,7 @@ func ServiceDiscoveryFromSpec(ctx context.Context, cc controllerClient.Client,
 }
 
 func populateServiceDiscoverySpec(options *ServiceDiscoveryOptions, brokerInfo *broker.Info, brokerSecret *v1.Secret,
-	repositoryInfo *image.RepositoryInfo,
+	clustersetConfig clustersetip.Config, repositoryInfo *image.RepositoryInfo,
 ) *operatorv1alpha1.ServiceDiscoverySpec {
 	brokerURL := removeSchemaPrefix(brokerInfo.BrokerURL)
 
@@ -82,6 +86,8 @@ func populateServiceDiscoverySpec(options *ServiceDiscoveryOptions, brokerInfo *
 		ClusterID:                options.ClusterID,
 		Namespace:                constants.OperatorNamespace,
 		ImageOverrides:           repositoryInfo.Overrides,
+		ClustersetIPEnabled:      options.ClustersetIPEnabled,
+		ClustersetIPCIDR:         clustersetConfig.ClustersetIPCIDR,
 	}
 
 	if options.CoreDNSCustomConfigMap != "" {

pkg/deploy/submariner.go

@@ -31,6 +31,7 @@ import (
 	"github.com/submariner-io/subctl/pkg/secret"
 	"github.com/submariner-io/subctl/pkg/submarinercr"
 	operatorv1alpha1 "github.com/submariner-io/submariner-operator/api/v1alpha1"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/clustersetip"
 	"github.com/submariner-io/submariner-operator/pkg/discovery/globalnet"
 	v1 "k8s.io/api/core/v1"
 	controllerClient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -46,6 +47,7 @@ type SubmarinerOptions struct {
 	LoadBalancerEnabled           bool
 	HealthCheckEnabled            bool
 	BrokerK8sInsecure             bool
+	ClustersetIPEnabled           bool
 	NATTPort                      int
 	HealthCheckInterval           uint64
 	HealthCheckMaxPacketLossCount uint64
@@ -60,14 +62,15 @@ type SubmarinerOptions struct {
 }
 
 func Submariner(ctx context.Context, clientProducer client.Producer, options *SubmarinerOptions, brokerInfo *broker.Info,
-	brokerSecret *v1.Secret, netconfig globalnet.Config, repositoryInfo *image.RepositoryInfo, status reporter.Interface,
+	brokerSecret *v1.Secret, netconfig globalnet.Config, clustersetConfig clustersetip.Config,
+	repositoryInfo *image.RepositoryInfo, status reporter.Interface,
 ) error {
 	pskSecret, err := secret.Ensure(ctx, clientProducer.ForKubernetes(), constants.OperatorNamespace, brokerInfo.IPSecPSK)
 	if err != nil {
 		return status.Error(err, "Error creating PSK secret for cluster")
 	}
 
-	submarinerSpec := populateSubmarinerSpec(options, brokerInfo, brokerSecret, pskSecret, netconfig, repositoryInfo)
+	submarinerSpec := populateSubmarinerSpec(options, brokerInfo, brokerSecret, pskSecret, netconfig, clustersetConfig, repositoryInfo)
 
 	err = SubmarinerFromSpec(ctx, clientProducer.ForGeneral(), submarinerSpec)
 	if err != nil {
@@ -84,7 +87,7 @@ func SubmarinerFromSpec(ctx context.Context, ctlClient controllerClient.Client,
 }
 
 func populateSubmarinerSpec(options *SubmarinerOptions, brokerInfo *broker.Info, brokerSecret *v1.Secret, pskSecret *v1.Secret,
-	netconfig globalnet.Config, repositoryInfo *image.RepositoryInfo,
+	netconfig globalnet.Config, clustersetConfig clustersetip.Config, repositoryInfo *image.RepositoryInfo,
 ) *operatorv1alpha1.SubmarinerSpec {
 	brokerURL := removeSchemaPrefix(brokerInfo.BrokerURL)
 
@@ -105,6 +108,8 @@ func populateSubmarinerSpec(options *SubmarinerOptions, brokerInfo *broker.Info,
 		BrokerK8sApiServer:       brokerURL,
 		BrokerK8sSecret:          brokerSecret.ObjectMeta.Name,
 		BrokerK8sInsecure:        options.BrokerK8sInsecure,
+		ClustersetIPEnabled:      options.ClustersetIPEnabled,
+		ClustersetIPCIDR:         clustersetConfig.ClustersetIPCIDR,
 		NatEnabled:               options.NATTraversal,
 		Debug:                    options.SubmarinerDebug,
 		ClusterID:                options.ClusterID,

pkg/join/join.go

@@ -36,6 +36,7 @@ import (
 	"github.com/submariner-io/subctl/pkg/operator"
 	"github.com/submariner-io/subctl/pkg/secret"
 	"github.com/submariner-io/subctl/pkg/version"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/clustersetip"
 	"github.com/submariner-io/submariner-operator/pkg/discovery/globalnet"
 	submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
 	v1 "k8s.io/api/core/v1"
@@ -83,6 +84,11 @@ func ClusterToBroker(ctx context.Context, brokerInfo *broker.Info, options *Opti
 		ClusterSize: options.GlobalnetClusterSize,
 	}
 
+	clustersetConfig := clustersetip.Config{
+		ClusterID:        options.ClusterID,
+		ClustersetIPCIDR: options.ClustersetIPCIDR,
+	}
+
 	operatorNamespace := constants.OperatorNamespace
 
 	err = ensureUniqueCluster(ctx, options.ClusterID, brokerClientProducer, brokerNamespace, clientProducer, operatorNamespace, status)
@@ -98,6 +104,12 @@ func ClusterToBroker(ctx context.Context, brokerInfo *broker.Info, options *Opti
 		}
 	}
 
+	err = clustersetip.AllocateCIDRFromConfigMap(ctx, brokerClientProducer.ForGeneral(), brokerNamespace,
+		&clustersetConfig, status)
+	if err != nil {
+		return errors.Wrap(err, "unable to determine the clusterset IP CIDR")
+	}
+
 	status.Start("Deploying the Submariner operator")
 
 	repositoryInfo := image.NewRepositoryInfo(options.Repository, options.ImageVersion, imageOverrides)
@@ -127,7 +139,7 @@ func ClusterToBroker(ctx context.Context, brokerInfo *broker.Info, options *Opti
 		status.Start("Deploying submariner")
 
 		err := deploy.Submariner(ctx, clientProducer, submarinerOptionsFrom(options), brokerInfo, brokerSecret, netconfig,
-			repositoryInfo, status)
+			clustersetConfig, repositoryInfo, status)
 		if err != nil {
 			return status.Error(err, "Error deploying the Submariner resource")
 		}
@@ -137,7 +149,7 @@ func ClusterToBroker(ctx context.Context, brokerInfo *broker.Info, options *Opti
 		status.Start("Deploying service discovery only")
 
 		err := deploy.ServiceDiscovery(ctx, clientProducer, serviceDiscoveryOptionsFrom(options), brokerInfo, brokerSecret,
-			repositoryInfo, status)
+			clustersetConfig, repositoryInfo, status)
 		if err != nil {
 			return status.Error(err, "Error deploying the ServiceDiscovery resource")
 		}
@@ -170,6 +182,7 @@ func submarinerOptionsFrom(joinOptions *Options) *deploy.SubmarinerOptions {
 		ServiceCIDR:                   joinOptions.ServiceCIDR,
 		ClusterCIDR:                   joinOptions.ClusterCIDR,
 		BrokerK8sInsecure:             !joinOptions.BrokerK8sSecure,
+		ClustersetIPEnabled:           joinOptions.EnableClustersetIP,
 	}
 }
 
@@ -182,6 +195,7 @@ func serviceDiscoveryOptionsFrom(joinOptions *Options) *deploy.ServiceDiscoveryO
 		ImageVersion:           joinOptions.ImageVersion,
 		CustomDomains:          joinOptions.CustomDomains,
 		BrokerK8sInsecure:      !joinOptions.BrokerK8sSecure,
+		ClustersetIPEnabled:    joinOptions.EnableClustersetIP,
 	}
 }
 

pkg/join/options.go

@@ -33,6 +33,7 @@ type Options struct {
 	LoadBalancerEnabled           bool
 	HealthCheckEnabled            bool
 	BrokerK8sSecure               bool
+	EnableClustersetIP            bool
 	NATTPort                      int
 	GlobalnetClusterSize          uint
 	HealthCheckInterval           uint64
@@ -46,6 +47,7 @@ type Options struct {
 	CableDriver                   string
 	CoreDNSCustomConfigMap        string
 	BrokerURL                     string
+	ClustersetIPCIDR              string
 	CustomDomains                 []string
 	ImageOverrideArr              []string
 	HTTPProxyConfig               httpproxy.Config