(feat) CAPI clusters

event-manager can be passed a new arg: `capi-onboard-annotation`. It allows administrators to specify a custom annotation key. When this annotation is set, Sveltos will only manage CAPI clusters that have this specific annotation with any value. CAPI clusters without this annotation will be ignored by Sveltos.
projectsveltos · Feb 23, 2025 · fd120c8 · fd120c8
1 parent d859151
commit fd120c8
Show file tree

Hide file tree

Showing 8 changed files with 48 additions and 38 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -60,18 +60,19 @@ import (
 )
 
 var (
-	setupLog             = ctrl.Log.WithName("setup")
-	shardKey             string
-	version              string
-	diagnosticsAddress   string
-	insecureDiagnostics  bool
-	workers              int
-	concurrentReconciles int
-	restConfigQPS        float32
-	restConfigBurst      int
-	webhookPort          int
-	syncPeriod           time.Duration
-	healthAddr           string
+	setupLog              = ctrl.Log.WithName("setup")
+	shardKey              string
+	version               string
+	diagnosticsAddress    string
+	insecureDiagnostics   bool
+	workers               int
+	concurrentReconciles  int
+	restConfigQPS         float32
+	restConfigBurst       int
+	webhookPort           int
+	syncPeriod            time.Duration
+	healthAddr            string
+	capiOnboardAnnotation string
 )
 
 const (
@@ -184,6 +185,9 @@ func initFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&shardKey, "shard-key", "",
 		"If set this deployment will reconcile only clusters matching this shard")
 
+	fs.StringVar(&capiOnboardAnnotation, "capi-onboard-annotation", "",
+		"If provided, Sveltos will only manage CAPI clusters that have this exact annotation.")
+
 	fs.StringVar(&version, "version", "", "current sveltos version")
 
 	fs.IntVar(&workers, "worker-number", defaultWorkers,
@@ -309,18 +313,19 @@ func getDiagnosticsOptions() metricsserver.Options {
 
 func getEventTriggerReconciler(mgr manager.Manager) *controllers.EventTriggerReconciler {
 	return &controllers.EventTriggerReconciler{
-		Client:               mgr.GetClient(),
-		Scheme:               mgr.GetScheme(),
-		ConcurrentReconciles: concurrentReconciles,
-		ShardKey:             shardKey,
-		Mux:                  sync.Mutex{},
-		Logger:               ctrl.Log.WithName("eventTriggerReconciler"),
-		ClusterMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
-		ToClusterMap:         make(map[types.NamespacedName]*libsveltosset.Set),
-		EventTriggers:        make(map[corev1.ObjectReference]libsveltosv1beta1.Selector),
-		ClusterLabels:        make(map[corev1.ObjectReference]map[string]string),
-		EventSourceMap:       make(map[corev1.ObjectReference]*libsveltosset.Set),
-		ToEventSourceMap:     make(map[types.NamespacedName]*libsveltosset.Set),
-		ClusterSetMap:        make(map[corev1.ObjectReference]*libsveltosset.Set),
+		Client:                mgr.GetClient(),
+		Scheme:                mgr.GetScheme(),
+		ConcurrentReconciles:  concurrentReconciles,
+		ShardKey:              shardKey,
+		CapiOnboardAnnotation: capiOnboardAnnotation,
+		Mux:                   sync.Mutex{},
+		Logger:                ctrl.Log.WithName("eventTriggerReconciler"),
+		ClusterMap:            make(map[corev1.ObjectReference]*libsveltosset.Set),
+		ToClusterMap:          make(map[types.NamespacedName]*libsveltosset.Set),
+		EventTriggers:         make(map[corev1.ObjectReference]libsveltosv1beta1.Selector),
+		ClusterLabels:         make(map[corev1.ObjectReference]map[string]string),
+		EventSourceMap:        make(map[corev1.ObjectReference]*libsveltosset.Set),
+		ToEventSourceMap:      make(map[types.NamespacedName]*libsveltosset.Set),
+		ClusterSetMap:         make(map[corev1.ObjectReference]*libsveltosset.Set),
 	}
 }
diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
@@ -13,5 +13,6 @@ spec:
         args:
         - "--diagnostics-address=:8443"
         - "--shard-key="
+        - "--capi-onboard-annotation="
         - "--v=5"
         - "--version=main"
diff --git a/controllers/eventreport_collection.go b/controllers/eventreport_collection.go
@@ -177,7 +177,7 @@ func buildEventTriggersForClusterMap(eventTriggers *v1beta1.EventTriggerList,
 
 // Periodically collects EventReports from each CAPI cluster.
 func collectEventReports(config *rest.Config, c client.Client, s *runtime.Scheme,
-	shardKey, version string, logger logr.Logger) {
+	shardKey, capiOnboardAnnotation, version string, logger logr.Logger) {
 
 	interval := 10 * time.Second
 	if shardKey != "" {
@@ -204,7 +204,8 @@ func collectEventReports(config *rest.Config, c client.Client, s *runtime.Scheme
 		eventTriggerMap := buildEventTriggersForClusterMap(eventTriggers)
 
 		logger.V(logs.LogDebug).Info("collecting managed clusters")
-		clusterList, err := clusterproxy.GetListOfClustersForShardKey(ctx, c, "", shardKey, logger)
+		clusterList, err := clusterproxy.GetListOfClustersForShardKey(ctx, c, "", capiOnboardAnnotation,
+			shardKey, logger)
 		if err != nil {
 			logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get clusters: %v", err))
 			time.Sleep(interval)

diff --git a/controllers/eventtrigger_controller.go b/controllers/eventtrigger_controller.go
@@ -76,12 +76,13 @@ const (
 // EventTriggerReconciler reconciles a EventTrigger object
 type EventTriggerReconciler struct {
 	client.Client
-	Scheme               *runtime.Scheme
-	ConcurrentReconciles int
-	Deployer             deployer.DeployerInterface
-	EventReportMode      ReportMode
-	ShardKey             string
-	Logger               logr.Logger
+	Scheme                *runtime.Scheme
+	ConcurrentReconciles  int
+	Deployer              deployer.DeployerInterface
+	EventReportMode       ReportMode
+	ShardKey              string
+	CapiOnboardAnnotation string // when set, only capi clusters with this annotation are considered
+	Logger                logr.Logger
 
 	// use a Mutex to update Map as MaxConcurrentReconciles is higher than one
 	Mux sync.Mutex
@@ -248,7 +249,7 @@ func (r *EventTriggerReconciler) reconcileNormal(
 	}
 
 	matchingCluster, err := clusterproxy.GetMatchingClusters(ctx, r.Client, eventTriggerScope.GetSelector(), "",
-		eventTriggerScope.Logger)
+		r.CapiOnboardAnnotation, eventTriggerScope.Logger)
 	if err != nil {
 		return reconcile.Result{Requeue: true, RequeueAfter: normalRequeueAfter}
 	}
@@ -355,7 +356,7 @@ func (r *EventTriggerReconciler) SetupWithManager(mgr ctrl.Manager) (controller.
 
 	if r.EventReportMode == CollectFromManagementCluster {
 		go collectEventReports(mgr.GetConfig(), mgr.GetClient(), mgr.GetScheme(), r.ShardKey,
-			getVersion(), mgr.GetLogger())
+			r.CapiOnboardAnnotation, getVersion(), mgr.GetLogger())
 	}
 
 	return c, nil

diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/onsi/gomega v1.36.2
 	github.com/pkg/errors v0.9.1
 	github.com/projectsveltos/addon-controller v0.48.1
-	github.com/projectsveltos/libsveltos v0.48.1
+	github.com/projectsveltos/libsveltos v0.48.2-0.20250223075846-d9184ce7344f
 	github.com/prometheus/client_golang v1.20.5
 	github.com/spf13/pflag v1.0.6
 	golang.org/x/text v0.22.0

diff --git a/go.sum b/go.sum
@@ -138,8 +138,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectsveltos/addon-controller v0.48.1 h1:DN41/UJes8uYfWKkDNli8uqeMA2KmPCpsQu+qix9sPU=
 github.com/projectsveltos/addon-controller v0.48.1/go.mod h1:/DAODslGfcANpHiwgeoptZ71wVoSMwakOMw7jrlHB+o=
-github.com/projectsveltos/libsveltos v0.48.1 h1:SWtACXeVNehWNxh/jEeFB/Z1QqMd4HeSh5Z60czwJbQ=
-github.com/projectsveltos/libsveltos v0.48.1/go.mod h1:9z2AUhSE2qzi+m5tqeQUMm+c4whMtbKH6oYOYY+0tbw=
+github.com/projectsveltos/libsveltos v0.48.2-0.20250223075846-d9184ce7344f h1:QyRB2GW8b2D4d9a5fT8spSSW5NzDzOdz/cQsg7toehs=
+github.com/projectsveltos/libsveltos v0.48.2-0.20250223075846-d9184ce7344f/go.mod h1:9z2AUhSE2qzi+m5tqeQUMm+c4whMtbKH6oYOYY+0tbw=
 github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
 github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=

diff --git a/manifest/deployment-shard.yaml b/manifest/deployment-shard.yaml
@@ -21,6 +21,7 @@ spec:
       - args:
         - --diagnostics-address=:8443
         - --shard-key={{.SHARD}}
+        - --capi-onboard-annotation=
         - --v=5
         - --version=main
         command:

diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml
@@ -2463,6 +2463,7 @@ spec:
       - args:
         - --diagnostics-address=:8443
         - --shard-key=
+        - --capi-onboard-annotation=
         - --v=5
         - --version=main
         command: