Skip to content

Commit

Permalink
Add support for VPC mixed mode
Browse files Browse the repository at this point in the history
1. Add controller for CR Network to maintain the default Network resource in
system Namespaces.
2. Add watch in Pod/Namepsace/Service/NetworkPolicy controller to requeue
resources when Network type is switch to VPC.
3. Add precheck on if the default Network type is VPC or not in resource create
events.
4. Add webhook in NCP owned resource creations to check if the default Network
type is VPC or not in a Namespace.
  • Loading branch information
wenyingd committed Jun 27, 2024
1 parent d1ad217 commit e670ae4
Show file tree
Hide file tree
Showing 18 changed files with 1,484 additions and 88 deletions.
33 changes: 33 additions & 0 deletions build/yaml/webhook/manifests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,36 @@ webhooks:
resources:
- subnetsets
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
service:
name: subnetset
namespace: vmware-system-nsx
path: /validate-vpc-enablement
failurePolicy: Fail
name: vpcnetwork.validating.nsx.vmware.com
rules:
- apiGroups:
- nsx.vmware.com
apiVersions:
- v1alpha1
operations:
- CREATE
resources:
- networkinfos
- nsxserviceaccount
- securitypolicies
- staticroutes
- subnetports
- subnets
- subnetsets
- apiGroups:
- nsx.vmware.com
apiVersions:
- v1alpha2
operations:
- CREATE
resources:
- ippools
sideEffects: None
47 changes: 31 additions & 16 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/healthz"
logf "sigs.k8s.io/controller-runtime/pkg/log"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"

"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
Expand All @@ -35,6 +36,7 @@ import (
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
Expand Down Expand Up @@ -133,12 +135,13 @@ func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
}
}

func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider) {
func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
nsReconciler := &namespacecontroller.NamespaceReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
NSXConfig: cf,
VPCService: vpcService,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
NSXConfig: cf,
VPCService: vpcService,
NetworkProvider: networkProvider,
}

if err := nsReconciler.Start(mgr); err != nil {
Expand All @@ -149,14 +152,26 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp

func main() {
log.Info("starting NSX Operator")
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
mgrOptions := ctrl.Options{
Scheme: scheme,
HealthProbeBindAddress: config.ProbeAddr,
Metrics: metricsserver.Options{BindAddress: config.MetricsAddr},
LeaderElection: cf.HAEnabled(),
LeaderElectionNamespace: nsxOperatorNamespace,
LeaderElectionID: "nsx-operator",
})
}

enableWebhook := true
if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
enableWebhook = false
} else {
mgrOptions.WebhookServer = webhook.NewServer(webhook.Options{
Port: config.WebhookServerPort,
CertDir: config.WebhookCertDir,
})
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
if err != nil {
log.Error(err, "failed to init manager")
os.Exit(1)
Expand All @@ -181,6 +196,10 @@ func main() {
var vpcService *vpc.VPCService

if cf.CoeConfig.EnableVPCNetwork {
if !enableWebhook {
log.Error(nil, "Webhook cert is not provided, can't filter out the CRs in a non-VPC namespace")
os.Exit(1)
}
// Check NSX version for VPC networking mode
if !commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
log.Error(nil, "VPC mode cannot be enabled if NSX version is lower than 4.1.1")
Expand Down Expand Up @@ -218,28 +237,24 @@ func main() {
os.Exit(1)
}
// Start controllers which only supports VPC
vpcNetworkProvider := vpcnetwork.StartNetworkController(mgr)
StartNetworkInfoController(mgr, vpcService)
StartNamespaceController(mgr, cf, vpcService)
StartNamespaceController(mgr, cf, vpcService, vpcNetworkProvider)
// Start subnet/subnetset controller.
if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {
os.Exit(1)
}
enableWebhook := true
if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
enableWebhook = false
}
if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil {
os.Exit(1)
}

node.StartNodeController(mgr, nodeService)
staticroutecontroller.StartStaticRouteController(mgr, staticRouteService)
subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService, vpcNetworkProvider)
StartIPPoolController(mgr, ipPoolService, vpcService)
networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
service.StartServiceLbController(mgr, commonService)
networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService, vpcNetworkProvider)
service.StartServiceLbController(mgr, commonService, vpcNetworkProvider)
}
// Start controllers which can run in non-VPC mode
securitypolicycontroller.StartSecurityPolicyController(mgr, commonService, vpcService)
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@ replace (
github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
)

require (
github.com/agiledragon/gomonkey/v2 v2.9.0
github.com/apparentlymart/go-cidr v1.1.0
github.com/deckarep/golang-set v1.8.0
github.com/go-logr/logr v1.3.0
github.com/go-logr/zapr v1.2.4
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang/mock v1.6.0
github.com/google/uuid v1.3.0
Expand All @@ -23,6 +24,7 @@ require (
github.com/prometheus/client_golang v1.16.0
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.4
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1
github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240305035435-c992c623aad3
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f
github.com/vmware-tanzu/vm-operator/api v1.8.2
Expand Down Expand Up @@ -53,7 +55,6 @@ require (
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gibson042/canonicaljson-go v1.0.3 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1 h1:r8RuSJnLEStOdGTfaZeOjH4rvRB4Gm/N1+qtU16VrI0=
github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo=
github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=
Expand Down
27 changes: 22 additions & 5 deletions pkg/controllers/namespace/namespace_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,21 @@ import (
"errors"
"fmt"

netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
apitypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"

"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
"github.com/vmware-tanzu/nsx-operator/pkg/config"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
Expand All @@ -35,10 +39,11 @@ var (
// Using vpcservice provider instead of vpc service to prevent
// invoking method that should be exposed to other module.
type NamespaceReconciler struct {
Client client.Client
Scheme *apimachineryruntime.Scheme
NSXConfig *config.NSXOperatorConfig
VPCService types.VPCServiceProvider
Client client.Client
Scheme *apimachineryruntime.Scheme
NSXConfig *config.NSXOperatorConfig
VPCService types.VPCServiceProvider
NetworkProvider vpcnetwork.VPCNetworkProvider
}

func (r *NamespaceReconciler) getDefaultNetworkConfigName() (string, error) {
Expand Down Expand Up @@ -179,6 +184,10 @@ func (r *NamespaceReconciler) insertNamespaceNetworkconfigBinding(ns string, ann
return nil
}

func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
return r.NetworkProvider.ReconcileWithVPCFilters("namespace", ctx, req, r.reconcile)
}

/*
VPC creation strategy:
Expand All @@ -193,7 +202,7 @@ We suppose namespace should have following annotations:
- If the ns do not have either of the annotation above, then we believe it is using default VPC, try to search
default VPC in network config CR store. The default VPC network config CR's name is "default".
*/
func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
func (r *NamespaceReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
obj := &v1.Namespace{}
log.Info("reconciling K8s namespace", "namespace", req.NamespacedName)
metrics.CounterInc(r.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNamespace)
Expand Down Expand Up @@ -251,6 +260,14 @@ func (r *NamespaceReconciler) setupWithManager(mgr ctrl.Manager) error {
controller.Options{
MaxConcurrentReconciles: common.NumReconcile(),
}).
Watches(
&netopv1alpha1.Network{},
&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: func(namespace string) ([]apitypes.NamespacedName, error) {
obj := apitypes.NamespacedName{Name: namespace, Namespace: namespace}
return []apitypes.NamespacedName{obj}, nil
}},
builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
).
Complete(r)
}

Expand Down
8 changes: 5 additions & 3 deletions pkg/controllers/namespace/namespace_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"

"github.com/vmware-tanzu/nsx-operator/pkg/config"
vpcnetworktesting "github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork/testing"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
Expand All @@ -40,9 +41,10 @@ func createNameSpaceReconciler() *NamespaceReconciler {
}

return &NamespaceReconciler{
Client: fake.NewClientBuilder().Build(),
Scheme: fake.NewClientBuilder().Build().Scheme(),
VPCService: service,
Client: fake.NewClientBuilder().Build(),
Scheme: fake.NewClientBuilder().Build().Scheme(),
VPCService: service,
NetworkProvider: &vpcnetworktesting.FakeVPCNetworkProvider{},
}
}

Expand Down
46 changes: 38 additions & 8 deletions pkg/controllers/networkpolicy/networkpolicy_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,21 @@ import (
"os"
"sync"

"github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
Expand All @@ -41,10 +44,11 @@ var (

// NetworkPolicyReconciler reconciles a NetworkPolicy object
type NetworkPolicyReconciler struct {
Client client.Client
Scheme *apimachineryruntime.Scheme
Service *securitypolicy.SecurityPolicyService
Recorder record.EventRecorder
Client client.Client
Scheme *apimachineryruntime.Scheme
Service *securitypolicy.SecurityPolicyService
Recorder record.EventRecorder
NetworkProvider vpcnetwork.VPCNetworkProvider
}

func updateFail(r *NetworkPolicyReconciler, c *context.Context, o *networkingv1.NetworkPolicy, e *error) {
Expand All @@ -70,6 +74,10 @@ func deleteSuccess(r *NetworkPolicyReconciler, _ *context.Context, o *networking
func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Use once.Do to ensure gc is called only once
common.GcOnce(r, &once)
return r.NetworkProvider.ReconcileWithVPCFilters("networkpolicy", ctx, req, r.reconcile)
}

func (r *NetworkPolicyReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
networkPolicy := &networkingv1.NetworkPolicy{}
log.Info("reconciling networkpolicy", "networkpolicy", req.NamespacedName)
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType)
Expand Down Expand Up @@ -134,6 +142,11 @@ func (r *NetworkPolicyReconciler) setupWithManager(mgr ctrl.Manager) error {
controller.Options{
MaxConcurrentReconciles: common.NumReconcile(),
}).
Watches(
&v1alpha1.Network{},
&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: r.listNetworkPolicies},
builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
).
Complete(r)
}

Expand Down Expand Up @@ -180,11 +193,28 @@ func (r *NetworkPolicyReconciler) CollectGarbage(ctx context.Context) {
}
}

func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider) {
func (r *NetworkPolicyReconciler) listNetworkPolicies(ns string) ([]types.NamespacedName, error) {
npList := &networkingv1.NetworkPolicyList{}
err := r.Client.List(context.Background(), npList, client.InNamespace(ns))
if err != nil {
return nil, err
}
nsNames := make([]types.NamespacedName, 0)
for _, np := range npList.Items {
nsNames = append(nsNames, types.NamespacedName{
Namespace: np.Namespace,
Name: np.Name,
})
}
return nsNames, nil
}

func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
networkPolicyReconcile := NetworkPolicyReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("networkpolicy-controller"),
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("networkpolicy-controller"),
NetworkProvider: networkProvider,
}
networkPolicyReconcile.Service = securitypolicy.GetSecurityService(commonService, vpcService)
if err := networkPolicyReconcile.Start(mgr); err != nil {
Expand Down
Loading

0 comments on commit e670ae4

Please sign in to comment.