Configure url and cabundle for self managed cluster

Signed-off-by: Tamal Saha <[email protected]>

Author: tamalsaha

charts/multicluster-controlplane/templates/deployment.yaml

@@ -32,6 +32,9 @@ spec:
         {{- if eq .Values.enableSelfManagement true }}
         - "--self-management"
         {{- end }}
+        {{- if .Values.selfManagementClusterName }}
+        - "--self-management-cluster-name={{ .Values.selfManagementClusterName }}"
+        {{- end }}
         {{- if eq .Values.enableDelegatingAuthentication true }}
         - "--delegating-authentication"
         {{- end }}

charts/multicluster-controlplane/values.yaml

@@ -13,6 +13,7 @@ autoApprovalBootstrapUsers: ""
 
 # TODO: should add restriction while enable selfmanagement
 enableSelfManagement: false
+selfManagementClusterName: ""
 
 enableDelegatingAuthentication: false
 

go.mod

@@ -28,12 +28,15 @@ require (
 	k8s.io/metrics v0.29.2
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	open-cluster-management.io/api v0.13.0
+	open-cluster-management.io/clusteradm v0.8.0
 	open-cluster-management.io/ocm v0.13.0
 	sigs.k8s.io/controller-runtime v0.17.2
 )
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
@@ -45,6 +48,7 @@ require (
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudevents/sdk-go/protocol/mqtt_paho/v2 v2.0.0-20231030012137-0836a524e995 // indirect
 	github.com/cloudevents/sdk-go/v2 v2.14.0 // indirect
 	github.com/coreos/go-oidc v2.2.1+incompatible // indirect
@@ -58,10 +62,12 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -79,8 +85,10 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
@@ -91,19 +99,24 @@ require (
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/openshift/api v0.0.0-20231218131639-7a5aa77cc72d // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkg/profile v1.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -114,6 +127,7 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
@@ -126,6 +140,7 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.10 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
@@ -142,6 +157,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
@@ -166,10 +182,12 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.14.1 // indirect
+	k8s.io/cli-runtime v0.29.2 // indirect
 	k8s.io/cloud-provider v0.29.2 // indirect
 	k8s.io/component-helpers v0.29.2 // indirect
 	k8s.io/dynamic-resource-allocation v0.29.2 // indirect
 	k8s.io/kms v0.29.2 // indirect
+	k8s.io/kubectl v0.29.0 // indirect
 	k8s.io/kubelet v0.29.2 // indirect
 	k8s.io/mount-utils v0.29.2 // indirect
 	k8s.io/pod-security-admission v0.29.2 // indirect
@@ -178,6 +196,8 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -264,6 +264,17 @@ func InitKubeconfig(
 		return err
 	}
 
+	// save controlplane in-cluster kubeconfig to the data directory for self management
+	if err := util.KubeconfigWriteToFile(
+		InclusterKubeconfigFile(certDir),
+		fmt.Sprintf("https://multicluster-controlplane.%s.svc/", util.GetComponentNamespace()),
+		inClusterTrustBundlePEM,
+		kubeconfigCertPEM,
+		kubeconfigKeyPEM,
+	); err != nil {
+		return err
+	}
+
 	// expose controlplane in-cluster kubeconfig in a secret
 	if err := util.KubeconfigWroteToSecret(
 		config,

pkg/certificate/certificate.go

@@ -8,8 +8,9 @@ import (
 )
 
 const (
-	ServiceAccountKeyFileName = "kube-serviceaccount.key"
-	KubeconfigFileName        = "kube-aggregator.kubeconfig"
+	ServiceAccountKeyFileName   = "kube-serviceaccount.key"
+	KubeconfigFileName          = "kube-aggregator.kubeconfig"
+	InclusterKubeconfigFileName = "incluster.kubeconfig"
 	// client user info
 	UserAdmin         = "system:admin"
 	UserKubeApiserver = "kube-apiserver"
@@ -53,6 +54,9 @@ func ServiceAccountKeyFile(certsDir string) string {
 func KubeConfigFile(certsDir string) string {
 	return filepath.Join(certsDir, KubeconfigFileName)
 }
+func InclusterKubeconfigFile(certsDir string) string {
+	return filepath.Join(certsDir, InclusterKubeconfigFileName)
+}
 func DefaultRootCAFile(certsDir string) string {
 	return filepath.Join(certsDir, RootCACertDirName, certchains.CACertFileName)
 }

pkg/certificate/certificateinfo.go

@@ -16,17 +16,24 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
-
 	clusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
+	"open-cluster-management.io/clusteradm/pkg/helpers"
 
 	"open-cluster-management.io/multicluster-controlplane/pkg/agent"
+	"open-cluster-management.io/multicluster-controlplane/pkg/certificate"
 	"open-cluster-management.io/multicluster-controlplane/pkg/servers/options"
 	"open-cluster-management.io/multicluster-controlplane/pkg/util"
 )
 
 const SelfManagementClusterLabel = "multicluster-controlplane.open-cluster-management.io/selfmanagement"
 
+type ClusterInfo struct {
+	ClusterName string
+	URL         string
+	CABundle    []byte
+}
+
 func InstallSelfManagementCluster(options options.ServerRunOptions) func(<-chan struct{}, *aggregatorapiserver.Config) error {
 	return func(stopCh <-chan struct{}, aggregatorConfig *aggregatorapiserver.Config) error {
 		inClusterConfig, err := rest.InClusterConfig()
@@ -52,13 +59,32 @@ func InstallSelfManagementCluster(options options.ServerRunOptions) func(<-chan
 			}
 		}
 
-		go EnableSelfManagement(ctx, hubRestConfig, options.ControlplaneDataDir, clusterName)
+		kubeClient, err := kubernetes.NewForConfig(inClusterConfig)
+		if err != nil {
+			return err
+		}
+		apiserverURL, err := helpers.GetAPIServer(kubeClient)
+		if err != nil {
+			return err
+		}
+		caBundle, err := helpers.GetCACert(kubeClient)
+		if err != nil {
+			return err
+		}
+
+		selfClusterInfo := ClusterInfo{
+			ClusterName: clusterName,
+			URL:         apiserverURL,
+			CABundle:    caBundle,
+		}
+
+		go EnableSelfManagement(ctx, hubRestConfig, options.ControlplaneDataDir, &selfClusterInfo)
 
 		return nil
 	}
 }
 
-func EnableSelfManagement(ctx context.Context, hubRestConfig *rest.Config, controlplaneCertDir, selfClusterName string) {
+func EnableSelfManagement(ctx context.Context, hubRestConfig *rest.Config, controlplaneCertDir string, selfClusterInfo *ClusterInfo) {
 	kubeClient, err := kubernetes.NewForConfig(hubRestConfig)
 	if err != nil {
 		klog.Fatalf("Failed to kube client, %v", err)
@@ -69,24 +95,24 @@ func EnableSelfManagement(ctx context.Context, hubRestConfig *rest.Config, contr
 		klog.Fatalf("Failed to cluster client, %v", err)
 	}
 
-	if err := createNamespace(ctx, kubeClient, selfClusterName); err != nil {
+	if err := createNamespace(ctx, kubeClient, selfClusterInfo.ClusterName); err != nil {
 		klog.Fatalf("Failed to create self managed cluster namespace, %v", err)
 	}
 
 	// TODO need a controller to maintain the self managed cluster
-	if err := waitForSelfManagedCluster(ctx, clusterClient, selfClusterName); err != nil {
+	if err := waitForSelfManagedCluster(ctx, clusterClient, selfClusterInfo); err != nil {
 		klog.Fatalf("Failed to create self managed cluster, %v", err)
 	}
 
-	bootstrapKubeConfig := path.Join(controlplaneCertDir, "cert", "kube-aggregator.kubeconfig")
+	bootstrapKubeConfig := path.Join(controlplaneCertDir, "cert", certificate.InclusterKubeconfigFileName)
 	agentHubKubeconfigDir := path.Join(controlplaneCertDir, "agent", "hub-kubeconfig")
 	if err := os.MkdirAll(agentHubKubeconfigDir, os.ModePerm); err != nil {
 		klog.Fatalf("Failed to create dir %s, %v", agentHubKubeconfigDir, err)
 	}
 
 	// TODO also need provide feature gates
 	klusterletAgent := agent.NewAgentOptions().
-		WithClusterName(selfClusterName).
+		WithClusterName(selfClusterInfo.ClusterName).
 		WithBootstrapKubeconfig(bootstrapKubeConfig).
 		WithHubKubeconfigDir(agentHubKubeconfigDir).
 		WithWorkloadSourceDriverConfig(agentHubKubeconfigDir + "/kubeconfig")
@@ -114,21 +140,27 @@ func createNamespace(ctx context.Context, kubeClient kubernetes.Interface, ns st
 	return err
 }
 
-func waitForSelfManagedCluster(ctx context.Context, clusterClient clusterclient.Interface, selfClusterName string) error {
+func waitForSelfManagedCluster(ctx context.Context, clusterClient clusterclient.Interface, selfClusterInfo *ClusterInfo) error {
 	return wait.PollUntilContextCancel(ctx, 5*time.Second, true, func(ctx context.Context) (bool, error) {
-		selfCluster, err := clusterClient.ClusterV1().ManagedClusters().Get(ctx, selfClusterName, metav1.GetOptions{})
+		selfCluster, err := clusterClient.ClusterV1().ManagedClusters().Get(ctx, selfClusterInfo.ClusterName, metav1.GetOptions{})
 		if errors.IsNotFound(err) {
 			_, err := clusterClient.ClusterV1().ManagedClusters().Create(
 				ctx,
 				&clusterv1.ManagedCluster{
 					ObjectMeta: metav1.ObjectMeta{
-						Name: selfClusterName,
+						Name: selfClusterInfo.ClusterName,
 						Labels: map[string]string{
 							SelfManagementClusterLabel: "",
 						},
 					},
 					Spec: clusterv1.ManagedClusterSpec{
 						HubAcceptsClient: true,
+						ManagedClusterClientConfigs: []clusterv1.ClientConfig{
+							{
+								URL:      selfClusterInfo.URL,
+								CABundle: selfClusterInfo.CABundle,
+							},
+						},
 					},
 				},
 				metav1.CreateOptions{},

pkg/controllers/ocmcontroller/ocmagent.go

@@ -78,7 +78,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
 
 // DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver.
 func DefaultOffAdmissionPlugins() sets.Set[string] {
-	defaultOnPlugins := sets.New(
+	defaultOnPlugins := sets.New[string](
 		lifecycle.PluginName,              // NamespaceLifecycle
 		serviceaccount.PluginName,         // ServiceAccount
 		mutatingwebhook.PluginName,        // MutatingAdmissionWebhook

pkg/servers/options/plugins.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	clientcmd "k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"