Merge pull request #5720 from jabellard/external_etcd_part2

External `etcd` Support for Karmada Operator - Part 2

operator/pkg/constants/constants.go

@@ -77,6 +77,16 @@ const (
 	KarmadaAPIserverListenClientPort = 5443
 	// EtcdDataVolumeName defines the name to etcd data volume
 	EtcdDataVolumeName = "etcd-data"
+	// EtcdClientCredentialsVolumeName defines the name of the volume for the etcd client credentials
+	EtcdClientCredentialsVolumeName = "etcd-client-cert" // #nosec G101
+	// EtcdClientCredentialsMountPath defines the mount path for the etcd client credentials data
+	EtcdClientCredentialsMountPath = "/etc/karmada/pki/etcd-client" // #nosec G101
+	// CaCertDataKey defines the data key for a CA cert
+	CaCertDataKey = "ca.crt"
+	// TLSCertDataKey defines the data key for a TLS cert
+	TLSCertDataKey = "tls.crt"
+	// TLSPrivateKeyDataKey defines the data key for a TLS cert private key
+	TLSPrivateKeyDataKey = "tls.key"
 
 	// CertificateValidity Certificate validity period
 	CertificateValidity = time.Hour * 24 * 365
@@ -125,9 +135,6 @@ const (
 
 	// APIServiceName defines the karmada aggregated apiserver APIService resource name.
 	APIServiceName = "v1alpha1.cluster.karmada.io"
-
-	// KarmadaApiserverEtcdClientCertNameSuffix defines the suffix for the Karmada API server etcd client cert name
-	KarmadaApiserverEtcdClientCertNameSuffix = "karmada-apiserver-etcd-client-cert"
 )
 
 var (

operator/pkg/controller/karmada/controller.go

@@ -40,8 +40,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
-	"github.com/karmada-io/karmada/operator/pkg/constants"
 	operatorscheme "github.com/karmada-io/karmada/operator/pkg/scheme"
+	"github.com/karmada-io/karmada/operator/pkg/util"
 )
 
 const (
@@ -112,7 +112,7 @@ func (ctrl *Controller) Reconcile(ctx context.Context, req controllerruntime.Req
 // validateKarmada ensures the Karmada resource adheres to validation rules
 func (ctrl *Controller) validateKarmada(karmada *operatorv1alpha1.Karmada) error {
 	if karmada.Spec.Components.Etcd != nil && karmada.Spec.Components.Etcd.External != nil {
-		expectedSecretName := fmt.Sprintf("%s-%s", karmada.Name, constants.KarmadaApiserverEtcdClientCertNameSuffix)
+		expectedSecretName := util.EtcdCertSecretName(karmada.Name)
 		if karmada.Spec.Components.Etcd.External.SecretRef.Name != expectedSecretName {
 			errorMessage := fmt.Sprintf("Secret name for external etcd client must be %s, but got %s", expectedSecretName, karmada.Spec.Components.Etcd.External.SecretRef.Name)
 			ctrl.EventRecorder.Event(karmada, corev1.EventTypeWarning, ValidationErrorReason, errorMessage)

operator/pkg/controller/karmada/planner_test.go

@@ -51,6 +51,13 @@ func TestNewPlannerFor(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Name: name,
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			client:     fake.NewFakeClient(),
 			config:     &rest.Config{},
@@ -65,8 +72,16 @@ func TestNewPlannerFor(t *testing.T) {
 					DeletionTimestamp: &metav1.Time{
 						Time: time.Now().Add(-5 * time.Minute),
 					},
+
 					Finalizers: []string{ControllerFinalizerName},
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			client:     fake.NewFakeClient(),
 			config:     &rest.Config{},
@@ -107,6 +122,13 @@ func TestPreRunJob(t *testing.T) {
 					Name:      name,
 					Namespace: namespace,
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config:  &rest.Config{},
 			action:  InitAction,
@@ -124,6 +146,13 @@ func TestPreRunJob(t *testing.T) {
 					},
 					Finalizers: []string{ControllerFinalizerName},
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config:  &rest.Config{},
 			action:  DeInitAction,
@@ -137,6 +166,13 @@ func TestPreRunJob(t *testing.T) {
 					Name:      name,
 					Namespace: namespace,
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config: &rest.Config{},
 			action: "UnknownAction",
@@ -197,7 +233,13 @@ func TestAfterRunJob(t *testing.T) {
 					Name:      name,
 					Namespace: namespace,
 				},
-				Spec: operatorv1alpha1.KarmadaSpec{},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config: &rest.Config{},
 			action: InitAction,
@@ -233,6 +275,13 @@ func TestAfterRunJob(t *testing.T) {
 					},
 					Finalizers: []string{ControllerFinalizerName},
 				},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config: &rest.Config{},
 			action: DeInitAction,
@@ -288,7 +337,13 @@ func TestRunJobErr(t *testing.T) {
 					Name:      name,
 					Namespace: namespace,
 				},
-				Spec: operatorv1alpha1.KarmadaSpec{},
+				Spec: operatorv1alpha1.KarmadaSpec{
+					Components: &operatorv1alpha1.KarmadaComponents{
+						Etcd: &operatorv1alpha1.Etcd{
+							Local: &operatorv1alpha1.LocalEtcd{},
+						},
+					},
+				},
 			},
 			config:  &rest.Config{},
 			jobErr:  errors.New("test error"),

operator/pkg/controlplane/apiserver/apiserver.go

@@ -27,15 +27,15 @@ import (
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 
 	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
-	"github.com/karmada-io/karmada/operator/pkg/constants"
+	"github.com/karmada-io/karmada/operator/pkg/controlplane/etcd"
 	"github.com/karmada-io/karmada/operator/pkg/util"
 	"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
 	"github.com/karmada-io/karmada/operator/pkg/util/patcher"
 )
 
 // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
 func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
-	if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace, featureGates); err != nil {
+	if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil {
 		return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
 	}
 
@@ -44,29 +44,25 @@ func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.Ka
 
 // EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource
 func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
-	if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace, featureGates); err != nil {
+	if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil {
 		return err
 	}
 	return createKarmadaAggregatedAPIServerService(client, name, namespace)
 }
 
-func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string, _ map[string]bool) error {
+func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error {
 	apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct {
-		DeploymentName, Namespace, Image, ImagePullPolicy, EtcdClientService string
-		ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret                   string
-		Replicas                                                             *int32
-		EtcdListenClientPort                                                 int32
+		DeploymentName, Namespace, Image, ImagePullPolicy string
+		ServiceSubnet, KarmadaCertsSecret                 string
+		Replicas                                          *int32
 	}{
-		DeploymentName:       util.KarmadaAPIServerName(name),
-		Namespace:            namespace,
-		Image:                cfg.Image.Name(),
-		ImagePullPolicy:      string(cfg.ImagePullPolicy),
-		EtcdClientService:    util.KarmadaEtcdClientName(name),
-		ServiceSubnet:        *cfg.ServiceSubnet,
-		KarmadaCertsSecret:   util.KarmadaCertSecretName(name),
-		EtcdCertsSecret:      util.EtcdCertSecretName(name),
-		Replicas:             cfg.Replicas,
-		EtcdListenClientPort: constants.EtcdListenClientPort,
+		DeploymentName:     util.KarmadaAPIServerName(name),
+		Namespace:          namespace,
+		Image:              cfg.Image.Name(),
+		ImagePullPolicy:    string(cfg.ImagePullPolicy),
+		ServiceSubnet:      *cfg.ServiceSubnet,
+		KarmadaCertsSecret: util.KarmadaCertSecretName(name),
+		Replicas:           cfg.Replicas,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing karmadaApiserver deployment template: %w", err)
@@ -76,6 +72,12 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
 	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), apiserverDeploymentBytes, apiserverDeployment); err != nil {
 		return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err)
 	}
+
+	err = etcd.ConfigureClientCredentials(apiserverDeployment, etcdCfg, name, namespace)
+	if err != nil {
+		return err
+	}
+
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithExtraArgs(cfg.ExtraArgs).WithExtraVolumeMounts(cfg.ExtraVolumeMounts).
 		WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).ForDeployment(apiserverDeployment)
@@ -112,23 +114,19 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al
 	return nil
 }
 
-func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string, featureGates map[string]bool) error {
+func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
 	aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct {
-		DeploymentName, Namespace, Image, ImagePullPolicy, EtcdClientService string
-		KubeconfigSecret, KarmadaCertsSecret, EtcdCertsSecret                string
-		Replicas                                                             *int32
-		EtcdListenClientPort                                                 int32
+		DeploymentName, Namespace, Image, ImagePullPolicy string
+		KubeconfigSecret, KarmadaCertsSecret              string
+		Replicas                                          *int32
 	}{
-		DeploymentName:       util.KarmadaAggregatedAPIServerName(name),
-		Namespace:            namespace,
-		Image:                cfg.Image.Name(),
-		ImagePullPolicy:      string(cfg.ImagePullPolicy),
-		EtcdClientService:    util.KarmadaEtcdClientName(name),
-		KubeconfigSecret:     util.AdminKubeconfigSecretName(name),
-		KarmadaCertsSecret:   util.KarmadaCertSecretName(name),
-		EtcdCertsSecret:      util.EtcdCertSecretName(name),
-		Replicas:             cfg.Replicas,
-		EtcdListenClientPort: constants.EtcdListenClientPort,
+		DeploymentName:     util.KarmadaAggregatedAPIServerName(name),
+		Namespace:          namespace,
+		Image:              cfg.Image.Name(),
+		ImagePullPolicy:    string(cfg.ImagePullPolicy),
+		KubeconfigSecret:   util.AdminKubeconfigSecretName(name),
+		KarmadaCertsSecret: util.KarmadaCertSecretName(name),
+		Replicas:           cfg.Replicas,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing karmadaAggregatedAPIServer deployment template: %w", err)
@@ -139,6 +137,11 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
 		return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err)
 	}
 
+	err = etcd.ConfigureClientCredentials(aggregatedAPIServerDeployment, etcdCfg, name, namespace)
+	if err != nil {
+		return err
+	}
+
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(aggregatedAPIServerDeployment)
 

operator/pkg/controlplane/apiserver/apiserver_test.go

@@ -54,6 +54,9 @@ func TestEnsureKarmadaAPIServer(t *testing.T) {
 			ServiceSubnet: ptr.To(serviceSubnet),
 			ExtraArgs:     map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
 		},
+		Etcd: &operatorv1alpha1.Etcd{
+			Local: &operatorv1alpha1.LocalEtcd{},
+		},
 	}
 
 	fakeClient := fakeclientset.NewSimpleClientset()
@@ -90,6 +93,9 @@ func TestEnsureKarmadaAggregatedAPIServer(t *testing.T) {
 			},
 			ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
 		},
+		Etcd: &operatorv1alpha1.Etcd{
+			Local: &operatorv1alpha1.LocalEtcd{},
+		},
 	}
 
 	featureGates := map[string]bool{"FeatureA": true}
@@ -133,11 +139,13 @@ func TestInstallKarmadaAPIServer(t *testing.T) {
 		ServiceSubnet: ptr.To(serviceSubnet),
 		ExtraArgs:     map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
 	}
-
+	etcdCfg := &operatorv1alpha1.Etcd{
+		Local: &operatorv1alpha1.LocalEtcd{},
+	}
 	featureGates := map[string]bool{"FeatureA": true}
 
 	// Call the function under test.
-	err := installKarmadaAPIServer(fakeClient, cfg, name, namespace, featureGates)
+	err := installKarmadaAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
 	if err != nil {
 		t.Fatalf("expected no error, but got: %v", err)
 	}
@@ -228,8 +236,10 @@ func TestInstallKarmadaAggregatedAPIServer(t *testing.T) {
 	}
 
 	featureGates := map[string]bool{"FeatureA": true}
-
-	err := installKarmadaAggregatedAPIServer(fakeClient, cfg, name, namespace, featureGates)
+	etcdCfg := &operatorv1alpha1.Etcd{
+		Local: &operatorv1alpha1.LocalEtcd{},
+	}
+	err := installKarmadaAggregatedAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
 	if err != nil {
 		t.Fatalf("Failed to install Karmada Aggregated API Server: %v", err)
 	}

operator/pkg/controlplane/apiserver/mainfests.go

@@ -50,10 +50,6 @@ spec:
         - --disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount
         - --enable-admission-plugins=NodeRestriction
         - --enable-bootstrap-token-auth=true
-        - --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
-        - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
-        - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
-        - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
         - --bind-address=0.0.0.0
         - --secure-port=5443
         - --service-account-issuer=https://kubernetes.default.svc.cluster.local
@@ -112,17 +108,11 @@ spec:
         - mountPath: /etc/karmada/pki
           name: apiserver-cert
           readOnly: true
-        - mountPath: /etc/etcd/pki
-          name: etcd-cert
-          readOnly: true
       priorityClassName: system-node-critical
       volumes:
       - name: apiserver-cert
         secret:
           secretName: {{ .KarmadaCertsSecret }}
-      - name: etcd-cert
-        secret:
-          secretName: {{ .EtcdCertsSecret }}
 `
 
 	// KarmadaApiserverService is karmada apiserver service manifest
@@ -176,10 +166,6 @@ spec:
         - --kubeconfig=/etc/karmada/kubeconfig
         - --authentication-kubeconfig=/etc/karmada/kubeconfig
         - --authorization-kubeconfig=/etc/karmada/kubeconfig
-        - --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
-        - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
-        - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
-        - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
         - --tls-cert-file=/etc/karmada/pki/karmada.crt
         - --tls-private-key-file=/etc/karmada/pki/karmada.key
         - --tls-min-version=VersionTLS13
@@ -190,9 +176,6 @@ spec:
         - mountPath: /etc/karmada/kubeconfig
           name: kubeconfig
           subPath: kubeconfig
-        - mountPath: /etc/etcd/pki
-          name: etcd-cert
-          readOnly: true
         - mountPath: /etc/karmada/pki
           name: apiserver-cert
           readOnly: true
@@ -203,9 +186,6 @@ spec:
       - name: apiserver-cert
         secret:
           secretName: {{ .KarmadaCertsSecret }}
-      - name: etcd-cert
-        secret:
-          secretName: {{ .EtcdCertsSecret }}
 `
 	// KarmadaAggregatedAPIServerService is karmada aggregated APIServer Service manifest
 	KarmadaAggregatedAPIServerService = `