Add support for VPC mixed mode

1. Add controller for CR Network to maintain the default Network resource in
system Namespaces.
2. Add watch in Pod/Namepsace/Service/NetworkPolicy controller to requeue
resources when Network type is switch to VPC.
3. Add precheck on if the default Network type is VPC or not in resource create
events.
4. Add webhook in NCP owned resource creations to check if the default Network
type is VPC or not in a Namespace.

build/yaml/webhook/manifests.yaml

@@ -30,3 +30,36 @@ webhooks:
     resources:
     - subnetsets
   sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: subnetset
+      namespace: vmware-system-nsx
+      path: /validate-vpc-enablement
+  failurePolicy: Fail
+  name: vpcnetwork.validating.nsx.vmware.com
+  rules:
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+      resources:
+        - networkinfos
+        - nsxserviceaccount
+        - securitypolicies
+        - staticroutes
+        - subnetports
+        - subnets
+        - subnetsets
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha2
+      operations:
+        - CREATE
+      resources:
+        - ippools
+  sideEffects: None

cmd/main.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
@@ -35,6 +36,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
@@ -133,12 +135,13 @@ func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
 	}
 }
 
-func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider) {
+func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
 	nsReconciler := &namespacecontroller.NamespaceReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		NSXConfig:  cf,
-		VPCService: vpcService,
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		NSXConfig:       cf,
+		VPCService:      vpcService,
+		NetworkProvider: networkProvider,
 	}
 
 	if err := nsReconciler.Start(mgr); err != nil {
@@ -149,14 +152,26 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp
 
 func main() {
 	log.Info("starting NSX Operator")
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	mgrOptions := ctrl.Options{
 		Scheme:                  scheme,
 		HealthProbeBindAddress:  config.ProbeAddr,
 		Metrics:                 metricsserver.Options{BindAddress: config.MetricsAddr},
 		LeaderElection:          cf.HAEnabled(),
 		LeaderElectionNamespace: nsxOperatorNamespace,
 		LeaderElectionID:        "nsx-operator",
-	})
+	}
+
+	enableWebhook := true
+	if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
+		log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
+		enableWebhook = false
+	} else {
+		mgrOptions.WebhookServer = webhook.NewServer(webhook.Options{
+			Port:    config.WebhookServerPort,
+			CertDir: config.WebhookCertDir,
+		})
+	}
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
 	if err != nil {
 		log.Error(err, "failed to init manager")
 		os.Exit(1)
@@ -181,6 +196,10 @@ func main() {
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork {
+		if !enableWebhook {
+			log.Error(nil, "Webhook cert is not provided, can't filter out the CRs in a non-VPC namespace")
+			os.Exit(1)
+		}
 		// Check NSX version for VPC networking mode
 		if !commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
 			log.Error(nil, "VPC mode cannot be enabled if NSX version is lower than 4.1.1")
@@ -218,28 +237,27 @@ func main() {
 			os.Exit(1)
 		}
 		// Start controllers which only supports VPC
+		var vpcNetworkProvider vpcnetwork.VPCNetworkProvider
+		if cf.CoeConfig.EnableVPCMixedMode {
+			vpcNetworkProvider = vpcnetwork.StartNetworkController(mgr)
+		}
 		StartNetworkInfoController(mgr, vpcService)
-		StartNamespaceController(mgr, cf, vpcService)
+		StartNamespaceController(mgr, cf, vpcService, vpcNetworkProvider)
 		// Start subnet/subnetset controller.
 		if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {
 			os.Exit(1)
 		}
-		enableWebhook := true
-		if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
-			log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
-			enableWebhook = false
-		}
 		if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil {
 			os.Exit(1)
 		}
 
 		node.StartNodeController(mgr, nodeService)
 		staticroutecontroller.StartStaticRouteController(mgr, staticRouteService)
 		subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
-		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
+		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService, vpcNetworkProvider)
 		StartIPPoolController(mgr, ipPoolService, vpcService)
-		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
-		service.StartServiceLbController(mgr, commonService)
+		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService, vpcNetworkProvider)
+		service.StartServiceLbController(mgr, commonService, vpcNetworkProvider)
 	}
 	// Start controllers which can run in non-VPC mode
 	securitypolicycontroller.StartSecurityPolicyController(mgr, commonService, vpcService)

go.mod

@@ -6,14 +6,15 @@ replace (
 	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
-        github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
 )
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/deckarep/golang-set v1.8.0
 	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/zapr v1.2.4
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -23,6 +24,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.4
+	github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240305035435-c992c623aad3
 	github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f
 	github.com/vmware-tanzu/vm-operator/api v1.8.2
@@ -53,7 +55,6 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gibson042/canonicaljson-go v1.0.3 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect

go.sum

@@ -177,8 +177,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1 h1:r8RuSJnLEStOdGTfaZeOjH4rvRB4Gm/N1+qtU16VrI0=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo=
 github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
 github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
 github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=

pkg/config/config.go

@@ -94,8 +94,9 @@ type DefaultConfig struct {
 }
 
 type CoeConfig struct {
-	Cluster          string `ini:"cluster"`
-	EnableVPCNetwork bool   `ini:"enable_vpc_network"`
+	Cluster            string `ini:"cluster"`
+	EnableVPCNetwork   bool   `ini:"enable_vpc_network"`
+	EnableVPCMixedMode bool   `ini:"enable_vpc_mixed_mode"`
 }
 
 type NsxConfig struct {
@@ -238,6 +239,10 @@ func (operatorConfig *NSXOperatorConfig) validate() error {
 	if err := operatorConfig.NsxConfig.validate(operatorConfig.CoeConfig.EnableVPCNetwork); err != nil {
 		return err
 	}
+	if operatorConfig.CoeConfig.EnableVPCMixedMode && !operatorConfig.CoeConfig.EnableVPCNetwork {
+		configLog.Error("VPC mixed mode is enabled but VPC is not enabled")
+		return errors.New("VPC mixed mode can't be enabled without VPC network enablement")
+	}
 	// TODO, verify if user&pwd, cert, jwt has any of them provided
 	return nil
 }

pkg/controllers/namespace/namespace_controller.go

@@ -8,17 +8,21 @@ import (
 	"errors"
 	"fmt"
 
+	netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -35,10 +39,11 @@ var (
 // Using vpcservice provider instead of vpc service to prevent
 // invoking method that should be exposed to other module.
 type NamespaceReconciler struct {
-	Client     client.Client
-	Scheme     *apimachineryruntime.Scheme
-	NSXConfig  *config.NSXOperatorConfig
-	VPCService types.VPCServiceProvider
+	Client          client.Client
+	Scheme          *apimachineryruntime.Scheme
+	NSXConfig       *config.NSXOperatorConfig
+	VPCService      types.VPCServiceProvider
+	NetworkProvider vpcnetwork.VPCNetworkProvider
 }
 
 func (r *NamespaceReconciler) getDefaultNetworkConfigName() (string, error) {
@@ -179,6 +184,13 @@ func (r *NamespaceReconciler) insertNamespaceNetworkconfigBinding(ns string, ann
 	return nil
 }
 
+func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	if r.NetworkProvider != nil {
+		return r.NetworkProvider.ReconcileWithVPCFilters("namespace", ctx, req, r.reconcile)
+	}
+	return r.reconcile(ctx, req)
+}
+
 /*
 	VPC creation strategy:
 
@@ -193,7 +205,7 @@ We suppose namespace should have following annotations:
   - If the ns do not have either of the annotation above, then we believe it is using default VPC, try to search
     default VPC in network config CR store. The default VPC network config CR's name is "default".
 */
-func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *NamespaceReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	obj := &v1.Namespace{}
 	log.Info("reconciling K8s namespace", "namespace", req.NamespacedName)
 	metrics.CounterInc(r.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNamespace)
@@ -251,6 +263,14 @@ func (r *NamespaceReconciler) setupWithManager(mgr ctrl.Manager) error {
 			controller.Options{
 				MaxConcurrentReconciles: common.NumReconcile(),
 			}).
+		Watches(
+			&netopv1alpha1.Network{},
+			&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: func(namespace string) ([]apitypes.NamespacedName, error) {
+				obj := apitypes.NamespacedName{Name: namespace, Namespace: namespace}
+				return []apitypes.NamespacedName{obj}, nil
+			}},
+			builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
+		).
 		Complete(r)
 }
 

pkg/controllers/namespace/namespace_controller_test.go

@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	vpcnetworktesting "github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork/testing"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
@@ -40,9 +41,10 @@ func createNameSpaceReconciler() *NamespaceReconciler {
 	}
 
 	return &NamespaceReconciler{
-		Client:     fake.NewClientBuilder().Build(),
-		Scheme:     fake.NewClientBuilder().Build().Scheme(),
-		VPCService: service,
+		Client:          fake.NewClientBuilder().Build(),
+		Scheme:          fake.NewClientBuilder().Build().Scheme(),
+		VPCService:      service,
+		NetworkProvider: &vpcnetworktesting.FakeVPCNetworkProvider{},
 	}
 }