Update CAPRKE2 template to work with CAPI 1.9+

Signed-off-by: Danil-Grigorev <[email protected]>
rancher · Jan 2, 2025 · 674cac5 · 674cac5
1 parent 886c780
commit 674cac5
Show file tree

Hide file tree

Showing 16 changed files with 214 additions and 172 deletions.
diff --git a/exp/README.md b/exp/README.md
@@ -42,12 +42,9 @@ export WORKER_MACHINE_COUNT=1
 export CLUSTER_NAME=rke2
 export KUBERNETES_VERSION=v1.30.0
 export RKE2_VERSION=v1.31.0+rke2r1
+export RKE2_CNI=calico
 
-# Prevent replacing lb-config variables by envsubst
-export address='$address'
-export server='$server'
-
-envsubst < test/e2e/data/cluster-templates/docker-rke2.yaml | kubectl apply -f -
+envsubst '${CLUSTER_NAME} ${WORKER_MACHINE_COUNT} ${RKE2_VERSION} ${CONTROL_PLANE_MACHINE_COUNT} ${KUBERNETES_VERSION} ${RKE2_CNI}' < test/e2e/data/cluster-templates/docker-rke2.yaml | kubectl apply -f -
 ```
 
 ## Performing a manual snapshot

diff --git a/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go b/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go
@@ -1,5 +1,5 @@
 /*
-Copyright © 2023 - 2024 SUSE LLC
+Copyright © 2023 - 2025 SUSE LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/controllers/remote"
+	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -49,7 +49,7 @@ type ETCDMachineSnapshotReconciler struct {
 	WatchFilterValue string
 
 	controller controller.Controller
-	Tracker    *remote.ClusterCacheTracker
+	Tracker    clustercache.ClusterCache
 	Scheme     *runtime.Scheme
 }
 

diff --git a/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go b/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go
@@ -1,5 +1,5 @@
 /*
-Copyright © 2023 - 2024 SUSE LLC
+Copyright © 2023 - 2025 SUSE LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/controllers/remote"
+	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	capiutil "sigs.k8s.io/cluster-api/util"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -46,7 +46,7 @@ type EtcdSnapshotSyncReconciler struct {
 	WatchFilterValue string
 
 	controller controller.Controller
-	Tracker    *remote.ClusterCacheTracker
+	Tracker    clustercache.ClusterCache
 }
 
 func (r *EtcdSnapshotSyncReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error {
@@ -131,13 +131,12 @@ func (r *EtcdSnapshotSyncReconciler) watchEtcdSnapshotFiles(ctx context.Context,
 
 	log.V(5).Info("Setting up watch on ETCDSnapshotFile")
 
-	return r.Tracker.Watch(ctx, remote.WatchInput{
+	return r.Tracker.Watch(ctx, capiutil.ObjectKey(cluster), clustercache.NewWatcher(clustercache.WatcherOptions{
 		Name:         "ETCDSnapshotFiles-watcher",
-		Cluster:      capiutil.ObjectKey(cluster),
 		Watcher:      r.controller,
 		Kind:         &k3sv1.ETCDSnapshotFile{},
 		EventHandler: handler.EnqueueRequestsFromMapFunc(r.etcdSnapshotFile(ctx, cluster)),
-	})
+	}))
 }
 
 func (r *EtcdSnapshotSyncReconciler) etcdSnapshotFile(ctx context.Context, cluster *clusterv1.Cluster) handler.MapFunc {

diff --git a/exp/etcdrestore/main.go b/exp/etcdrestore/main.go
@@ -1,5 +1,5 @@
 /*
-Copyright © 2023 - 2024 SUSE LLC
+Copyright © 2023 - 2025 SUSE LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -40,6 +40,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -208,16 +209,14 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
 		os.Exit(1)
 	}
 
-	// Set up a ClusterCacheTracker and ClusterCacheReconciler to provide to controllers
+	// Set up a ClusterCacheTracker reconciler to provide to controllers
 	// requiring a connection to a remote cluster
-	tracker, err := remote.NewClusterCacheTracker(
-		mgr,
-		remote.ClusterCacheTrackerOptions{
-			SecretCachingClient: secretCachingClient,
-			ControllerName:      "etcd-restore-controller",
-			Log:                 &ctrl.Log,
+	tracker, err := clustercache.SetupWithManager(ctx, mgr, clustercache.Options{
+		SecretClient: secretCachingClient,
+		Client: clustercache.ClientOptions{
+			UserAgent: remote.DefaultClusterAPIUserAgent("etcd-restore-controller"),
 		},
-	)
+	}, controller.Options{})
 	if err != nil {
 		setupLog.Error(err, "unable to create cluster cache tracker")
 		os.Exit(1)

diff --git a/test/e2e/const.go b/test/e2e/const.go
@@ -96,6 +96,9 @@ var (
 	//go:embed data/cluster-templates/vsphere-rke2.yaml
 	CAPIvSphereRKE2 []byte
 
+	//go:embed data/cni/kindnet.yaml
+	CAPIKindnet []byte
+
 	//go:embed data/cluster-api-addon-provider-fleet/host-network-patch.yaml
 	AddonProviderFleetHostNetworkPatch []byte
 

diff --git a/test/e2e/data/cluster-templates/docker-kubeadm.yaml b/test/e2e/data/cluster-templates/docker-kubeadm.yaml
@@ -145,133 +145,3 @@ spec:
         kind: DockerMachineTemplate
         name: ${CLUSTER_NAME}-md-0
       version: ${KUBERNETES_VERSION}
----
-kind: Bundle
-apiVersion: fleet.cattle.io/v1alpha1
-metadata:
-  name: kindnet-cni-${CLUSTER_NAME}
-spec:
-  resources:
-  # List of all resources that will be deployed
-  - content: |-
-      # kindnetd networking manifest
-      ---
-      kind: ClusterRole
-      apiVersion: rbac.authorization.k8s.io/v1
-      metadata:
-        name: kindnet
-      rules:
-        - apiGroups:
-            - ""
-          resources:
-            - nodes
-          verbs:
-            - list
-            - watch
-            - patch
-        - apiGroups:
-            - ""
-          resources:
-            - configmaps
-          verbs:
-            - get
-      ---
-      kind: ClusterRoleBinding
-      apiVersion: rbac.authorization.k8s.io/v1
-      metadata:
-        name: kindnet
-      roleRef:
-        apiGroup: rbac.authorization.k8s.io
-        kind: ClusterRole
-        name: kindnet
-      subjects:
-        - kind: ServiceAccount
-          name: kindnet
-          namespace: kube-system
-      ---
-      apiVersion: v1
-      kind: ServiceAccount
-      metadata:
-        name: kindnet
-        namespace: kube-system
-      ---
-      apiVersion: apps/v1
-      kind: DaemonSet
-      metadata:
-        name: kindnet
-        namespace: kube-system
-        labels:
-          tier: node
-          app: kindnet
-          k8s-app: kindnet
-      spec:
-        selector:
-          matchLabels:
-            app: kindnet
-        template:
-          metadata:
-            labels:
-              tier: node
-              app: kindnet
-              k8s-app: kindnet
-          spec:
-            hostNetwork: true
-            tolerations:
-              - operator: Exists
-                effect: NoSchedule
-            serviceAccountName: kindnet
-            containers:
-              - name: kindnet-cni
-                image: kindest/kindnetd:v20230511-dc714da8
-                env:
-                  - name: HOST_IP
-                    valueFrom:
-                      fieldRef:
-                        fieldPath: status.hostIP
-                  - name: POD_IP
-                    valueFrom:
-                      fieldRef:
-                        fieldPath: status.podIP
-                  - name: POD_SUBNET
-                    value: '10.1.0.0/16'
-                volumeMounts:
-                  - name: cni-cfg
-                    mountPath: /etc/cni/net.d
-                  - name: xtables-lock
-                    mountPath: /run/xtables.lock
-                    readOnly: false
-                  - name: lib-modules
-                    mountPath: /lib/modules
-                    readOnly: true
-                resources:
-                  requests:
-                    cpu: "100m"
-                    memory: "50Mi"
-                  limits:
-                    cpu: "100m"
-                    memory: "50Mi"
-                securityContext:
-                  privileged: false
-                  capabilities:
-                    add: ["NET_RAW", "NET_ADMIN"]
-            volumes:
-              - name: cni-bin
-                hostPath:
-                  path: /opt/cni/bin
-                  type: DirectoryOrCreate
-              - name: cni-cfg
-                hostPath:
-                  path: /etc/cni/net.d
-                  type: DirectoryOrCreate
-              - name: xtables-lock
-                hostPath:
-                  path: /run/xtables.lock
-                  type: FileOrCreate
-              - name: lib-modules
-                hostPath:
-                  path: /lib/modules
-    name: kindnet.yaml
-  targets:
-  - clusterSelector:
-      matchLabels:
-        cni: ${CLUSTER_NAME}-crs-0
diff --git a/test/e2e/data/cluster-templates/docker-rke2.yaml b/test/e2e/data/cluster-templates/docker-rke2.yaml
@@ -48,13 +48,18 @@ metadata:
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${RKE2_VERSION}
-  registrationMethod: internal-first
   rolloutStrategy:
     rollingUpdate:
       maxSurge: 1
     type: RollingUpdate
   serverConfig:
+    cni: ${RKE2_CNI}
+    kubeAPIServer:
+      extraArgs:
+      - --anonymous-auth=true
     disableComponents:
+      pluginComponents:
+       - rke2-ingress-nginx
       kubernetesComponents:
       - cloudController
     etcd:
@@ -137,10 +142,12 @@ data:
       default-server init-addr none
 
     frontend stats
+      mode http
       bind *:8404
       stats enable
-      stats uri /
-      stats refresh 10s
+      stats uri /stats
+      stats refresh 1s
+      stats admin if TRUE
 
     frontend control-plane
       bind *:{{ .FrontendControlPlanePort }}
@@ -151,10 +158,9 @@ data:
 
     backend kube-apiservers
       option httpchk GET /healthz
-      http-check expect status 401
-      # TODO: we should be verifying (!)
-      {{range $server, $address := .BackendServers}}
-      server {{ $server }} {{ JoinHostPort $address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
+
+      {{range $server, $backend := .BackendServers }}
+      server {{ $server }} {{ JoinHostPort $backend.Address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
       {{- end}}
 
     frontend rke2-join
@@ -167,8 +173,8 @@ data:
     backend rke2-servers
       option httpchk GET /v1-rke2/readyz
       http-check expect status 403
-      {{range $server, $address := .BackendServers}}
-      server {{ $server }} {{ $address }}:9345 check check-ssl verify none
+      {{range $server, $backend := .BackendServers }}
+      server {{ $server }} {{ $backend.Address }}:9345 check check-ssl verify none
       {{- end}}
 kind: ConfigMap
 metadata: