Support FRR-K8s running externally

We allow setting the external mode from the MetalLB crd, and we also
allow to specify the namespace it is running on.

Signed-off-by: Federico Paolinelli <[email protected]>

api/v1beta1/metallb_types.go

@@ -37,9 +37,10 @@ const (
 )
 
 const (
-	FRRMode    BGPType = "frr"
-	NativeMode BGPType = "native"
-	FRRK8sMode BGPType = "frr-k8s"
+	FRRMode            BGPType = "frr"
+	NativeMode         BGPType = "native"
+	FRRK8sMode         BGPType = "frr-k8s"
+	FRRK8sExternalMode BGPType = "frr-k8s-external"
 )
 
 type BGPType string
@@ -103,6 +104,8 @@ type MetalLBSpec struct {
 type FRRK8SConfig struct {
 	// A list of cidrs we want always to block for incoming routes
 	AlwaysBlock []string `json:"alwaysBlock,omitempty"`
+	// The namespace frr-k8s is running on in case of frr-k8s external mode
+	Namespace string `json:"namespace,omitempty"`
 }
 
 type Config struct {

api/v1beta1/metallb_webhook.go

@@ -29,7 +29,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
-func (metallb *MetalLB) SetupWebhookWithManager(mgr ctrl.Manager) error {
+var ExternalFRRK8sNamespace string
+
+func (metallb *MetalLB) SetupWebhookWithManager(mgr ctrl.Manager, externalFRRK8sNamespace string) error {
+	ExternalFRRK8sNamespace = externalFRRK8sNamespace
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(metallb).
 		Complete()
@@ -109,22 +112,24 @@ func (metallb *MetalLB) Validate() error {
 	if metallb.Spec.BGPBackend != "" &&
 		metallb.Spec.BGPBackend != NativeMode &&
 		metallb.Spec.BGPBackend != FRRK8sMode &&
+		metallb.Spec.BGPBackend != FRRK8sExternalMode &&
 		metallb.Spec.BGPBackend != FRRMode {
 		return errors.New("Invalid BGP Backend, must be one of native, frr, frr-k8s")
 	}
 
-	if metallb.Spec.BGPBackend != FRRK8sMode &&
-		metallb.Spec.FRRK8SConfig != nil {
-		return fmt.Errorf("can't apply frrk8s config while running in %s mode", metallb.Spec.BGPBackend)
-	}
-
-	if err := validateFRRK8sConfig(metallb.Spec.FRRK8SConfig); err != nil {
+	if err := validateFRRK8sConfig(metallb.Spec); err != nil {
 		return err
 	}
 	return nil
 }
 
-func validateFRRK8sConfig(config *FRRK8SConfig) error {
+func validateFRRK8sConfig(spec MetalLBSpec) error {
+	config := spec.FRRK8SConfig
+	if spec.BGPBackend == FRRK8sExternalMode &&
+		ExternalFRRK8sNamespace == "" && (config == nil || config.Namespace == "") {
+		return errors.New("bgp backend: frrk8s external and no default or user provided namespace")
+	}
+
 	if config == nil {
 		return nil
 	}

api/v1beta1/metallb_webhook_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestValidateFRRK8sConfig(t *testing.T) {
 	t.Run("NilConfig", func(t *testing.T) {
-		err := validateFRRK8sConfig(nil)
+		err := validateFRRK8sConfig(MetalLBSpec{})
 		if err != nil {
 			t.Errorf("Expected nil error, got: %v", err)
 		}
@@ -17,7 +17,9 @@ func TestValidateFRRK8sConfig(t *testing.T) {
 		config := &FRRK8SConfig{
 			AlwaysBlock: []string{"192.168.0.0/24", "10.0.0.0/16"},
 		}
-		err := validateFRRK8sConfig(config)
+		err := validateFRRK8sConfig(MetalLBSpec{
+			FRRK8SConfig: config,
+		})
 		if err != nil {
 			t.Errorf("Expected nil error, got: %v", err)
 		}
@@ -27,7 +29,9 @@ func TestValidateFRRK8sConfig(t *testing.T) {
 		config := &FRRK8SConfig{
 			AlwaysBlock: []string{"192.168.0.0/24", "invalid_cidr"},
 		}
-		err := validateFRRK8sConfig(config)
+		err := validateFRRK8sConfig(MetalLBSpec{
+			FRRK8SConfig: config,
+		})
 		expectedErr := errors.New("invalid CIDR invalid_cidr in AlwaysBlock")
 		if err == nil || err.Error() != expectedErr.Error() {
 			t.Errorf("Expected error: %v, got: %v", expectedErr, err)
@@ -38,9 +42,52 @@ func TestValidateFRRK8sConfig(t *testing.T) {
 		config := &FRRK8SConfig{
 			AlwaysBlock: []string{"2001:db8::/32", "2001:db8:85a3::/48"},
 		}
-		err := validateFRRK8sConfig(config)
+		err := validateFRRK8sConfig(MetalLBSpec{
+			FRRK8SConfig: config,
+		})
+
 		if err != nil {
 			t.Errorf("Expected nil error, got: %v", err)
 		}
 	})
+
+	t.Run("External, no config, no default", func(t *testing.T) {
+		oldNs := ExternalFRRK8sNamespace
+		defer func() { ExternalFRRK8sNamespace = oldNs }()
+		ExternalFRRK8sNamespace = ""
+		err := validateFRRK8sConfig(MetalLBSpec{
+			BGPBackend: FRRK8sExternalMode,
+		})
+
+		if err == nil {
+			t.Errorf("Expected error, got no error")
+		}
+	})
+
+	t.Run("External, config, no ns,  no default", func(t *testing.T) {
+		oldNs := ExternalFRRK8sNamespace
+		defer func() { ExternalFRRK8sNamespace = oldNs }()
+		ExternalFRRK8sNamespace = ""
+		err := validateFRRK8sConfig(MetalLBSpec{
+			BGPBackend:   FRRK8sExternalMode,
+			FRRK8SConfig: &FRRK8SConfig{},
+		})
+
+		if err == nil {
+			t.Errorf("Expected error, got no error")
+		}
+	})
+
+	t.Run("External, default namespace", func(t *testing.T) {
+		oldNs := ExternalFRRK8sNamespace
+		defer func() { ExternalFRRK8sNamespace = oldNs }()
+		ExternalFRRK8sNamespace = "foo"
+		err := validateFRRK8sConfig(MetalLBSpec{
+			BGPBackend: FRRK8sExternalMode,
+		})
+
+		if err != nil {
+			t.Errorf("Expected no error, got: %v", err)
+		}
+	})
 }

bin/metallb-operator.yaml

@@ -3979,6 +3979,8 @@ spec:
           value: "false"
         - name: FRRK8S_IMAGE
           value: quay.io/metallb/frr-k8s:v0.0.14
+        - name: FRRK8S_EXTERNAL_NAMESPACE
+          value: frr-k8s-system
         - name: OPERATOR_NAMESPACE
           valueFrom:
             fieldRef:

bundle/manifests/metallb-operator.clusterserviceversion.yaml

@@ -432,7 +432,7 @@ metadata:
     categories: Networking
     certified: "false"
     containerImage: quay.io/metallb/metallb-operator
-    createdAt: "2024-07-19T12:33:20Z"
+    createdAt: "2024-07-22T10:18:02Z"
     description: An operator for deploying MetalLB on a kubernetes cluster.
     operators.operatorframework.io/builder: operator-sdk-v1.34.1
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
@@ -875,6 +875,8 @@ spec:
                         value: "false"
                       - name: FRRK8S_IMAGE
                         value: quay.io/metallb/frr-k8s:v0.0.14
+                      - name: FRRK8S_EXTERNAL_NAMESPACE
+                        value: frr-k8s-system
                       - name: OPERATOR_NAMESPACE
                         valueFrom:
                           fieldRef:

config/manager/env.yaml

@@ -23,3 +23,5 @@ spec:
               value: "false"
             - name: FRRK8S_IMAGE
               value: "quay.io/metallb/frr-k8s:v0.0.14"
+            - name: FRRK8S_EXTERNAL_NAMESPACE
+              value: "frr-k8s-system"

controllers/metallb_controller.go

@@ -59,6 +59,8 @@ type MetalLBReconciler struct {
 var MetalLBChartPath = MetalLBChartPathController
 var FRRK8SChartPath = FRRK8SChartPathController
 
+var EmbeddedFRRK8sSupportNotAvailable = errors.New("current CNO version does not support deploying frr-k8s")
+
 // Namespace Scoped
 // +kubebuilder:rbac:groups=apps,namespace=metallb-system,resources=deployments;daemonsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete
@@ -124,7 +126,10 @@ func (r *MetalLBReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 }
 
 func (r *MetalLBReconciler) reconcileResource(ctx context.Context, req ctrl.Request, instance *metallbv1beta1.MetalLB) (ctrl.Result, string, error) {
-	err := r.syncMetalLBResources(instance)
+	err := r.syncMetalLBResources(ctx, instance)
+	if errors.Is(err, EmbeddedFRRK8sSupportNotAvailable) {
+		return ctrl.Result{RequeueAfter: 2 * time.Minute}, "", nil
+	}
 	if err != nil {
 		return ctrl.Result{}, status.ConditionDegraded, errors.Wrapf(err, "FailedToSyncMetalLBResources")
 	}
@@ -154,10 +159,12 @@ func (r *MetalLBReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *MetalLBReconciler) syncMetalLBResources(config *metallbv1beta1.MetalLB) error {
+func (r *MetalLBReconciler) syncMetalLBResources(ctx context.Context, config *metallbv1beta1.MetalLB) error {
 	logger := r.Log.WithName("syncMetalLBResources")
 	logger.Info("Start")
 
+	bgpType := params.BGPType(config, r.EnvConfig.IsOpenshift)
+
 	err := config.Validate()
 	if err != nil {
 		r.Log.Error(err, "Invalid MetalLB resource")
@@ -175,7 +182,7 @@ func (r *MetalLBReconciler) syncMetalLBResources(config *metallbv1beta1.MetalLB)
 		return err
 	}
 
-	if params.BGPType(config, r.EnvConfig.IsOpenshift) == metallbv1beta1.FRRK8sMode {
+	if bgpType == metallbv1beta1.FRRK8sMode {
 		objs = append(objs, frrk8sObjs...)
 	} else {
 		toDel = append(toDel, frrk8sObjs...)

go.mod

@@ -1,6 +1,8 @@
 module github.com/metallb/metallb-operator
 
-go 1.21
+go 1.22.0
+
+toolchain go1.22.5
 
 require (
 	github.com/go-logr/logr v1.4.1
@@ -11,9 +13,9 @@ require (
 	github.com/open-policy-agent/cert-controller v0.8.0
 	github.com/pkg/errors v0.9.1
 	helm.sh/helm/v3 v3.14.4
-	k8s.io/api v0.29.1
+	k8s.io/api v0.30.1
 	k8s.io/apiextensions-apiserver v0.29.1
-	k8s.io/apimachinery v0.29.1
+	k8s.io/apimachinery v0.30.1
 	k8s.io/client-go v1.5.2
 	k8s.io/kubernetes v1.25.4
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
@@ -77,7 +79,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -153,7 +155,7 @@ require (
 	k8s.io/cli-runtime v0.29.0 // indirect
 	k8s.io/component-base v0.29.1 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240126223410-2919ad4fcfec // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kubectl v0.29.0 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -840,8 +840,9 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
@@ -1907,8 +1908,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-aggregator v0.27.4 h1:WdK9iiBr32G8bWfpUEFVQl70RZO2dU19ZAktUXL5JFc=
 k8s.io/kube-aggregator v0.27.4/go.mod h1:+eG83gkAyh0uilQEAOgheeQW4hr+PkyV+5O1nLGsjlM=
-k8s.io/kube-openapi v0.0.0-20240126223410-2919ad4fcfec h1:iGTel2aR8vCZdxJDgmbeY0zrlXy9Qcvyw4R2sB4HLrA=
-k8s.io/kube-openapi v0.0.0-20240126223410-2919ad4fcfec/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
 k8s.io/kubectl v0.27.4 h1:RV1TQLIbtL34+vIM+W7HaS3KfAbqvy9lWn6pWB9els4=
 k8s.io/kubectl v0.27.4/go.mod h1:qtc1s3BouB9KixJkriZMQqTsXMc+OAni6FeKAhq7q14=
 k8s.io/kubernetes v1.27.4 h1:js5bonPoe7jgVPduNcWo6IjPTUdLzlnfhRgGmC7isM0=

main.go

@@ -154,7 +154,7 @@ func main() {
 		setupLog.Info("waiting to create operator webhook for MetalLB CR")
 		<-setupFinished
 		setupLog.Info("creating operator webhook for MetalLB CR")
-		if err = (&metallbv1beta1.MetalLB{}).SetupWebhookWithManager(mgr); err != nil {
+		if err = (&metallbv1beta1.MetalLB{}).SetupWebhookWithManager(mgr, envParams.FRRK8sExternalNamespace); err != nil {
 			setupLog.Error(err, "unable to create webhook", "operator webhook", "MetalLB")
 			os.Exit(1)
 		}

pkg/helm/metallb.go

@@ -337,8 +337,20 @@ func metalLBFrrk8sValues(envConfig params.EnvConfig, crdConfig *metallbv1beta1.M
 	if params.BGPType(crdConfig, envConfig.IsOpenshift) == metallbv1beta1.FRRK8sMode {
 		enabled = true
 	}
+
+	frrK8sNamespace := envConfig.FRRK8sExternalNamespace
+	if crdConfig.Spec.FRRK8SConfig != nil && crdConfig.Spec.FRRK8SConfig.Namespace != "" {
+		frrK8sNamespace = crdConfig.Spec.FRRK8SConfig.Namespace
+	}
+
+	external := false
+	if params.BGPType(crdConfig, envConfig.IsOpenshift) == metallbv1beta1.FRRK8sExternalMode {
+		external = true
+	}
 	frrk8sValuesMap := map[string]interface{}{
-		"enabled": enabled,
+		"enabled":   enabled,
+		"external":  external,
+		"namespace": frrK8sNamespace,
 	}
 	return frrk8sValuesMap
 }

pkg/params/params.go

@@ -27,6 +27,7 @@ func BGPType(m *v1beta1.MetalLB, isOpenshift bool) v1beta1.BGPType {
 
 type EnvConfig struct {
 	Namespace                  string
+	FRRK8sExternalNamespace    string
 	ControllerImage            ImageInfo
 	SpeakerImage               ImageInfo
 	FRRImage                   ImageInfo
@@ -121,6 +122,8 @@ func FromEnvironment(isOpenshift bool) (EnvConfig, error) {
 		res.DeployServiceMonitors = true
 	}
 
+	res.FRRK8sExternalNamespace = os.Getenv("FRRK8S_EXTERNAL_NAMESPACE")
+
 	// Ignoring the error, if not set we'll consume the image from the chart
 	res.FRRK8sImage, _ = imageFromEnv("FRRK8S_IMAGE")