Allow API server to update CA bundle on its service (#266)

build/charts/nephe/templates/controller/clusterrole.yaml

@@ -150,3 +150,10 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - apiregistration.k8s.io
+  resources:
+  - apiservices
+  verbs:
+  - list
+  - update

config/nephe.yml

@@ -441,6 +441,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - apiregistration.k8s.io
+  resources:
+  - apiservices
+  verbs:
+  - list
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

config/rbac/role.yaml

@@ -151,3 +151,10 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - apiregistration.k8s.io
+  resources:
+  - apiservices
+  verbs:
+  - list
+  - update

go.mod

@@ -27,6 +27,7 @@ require (
 	k8s.io/apimachinery v0.26.4
 	k8s.io/apiserver v0.26.4
 	k8s.io/client-go v0.26.4
+	k8s.io/kube-aggregator v0.26.4
 	sigs.k8s.io/controller-runtime v0.14.6
 )
 

go.sum

@@ -792,6 +792,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kms v0.26.4 h1:mQ+DeOvgAHC6+heZcozPkEd3rWtP4DVVjo1hLSih9w4=
 k8s.io/kms v0.26.4/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg=
+k8s.io/kube-aggregator v0.26.4 h1:iGljhq5exQkbuc3bnkwUx95RPCBDExg7DkX9XaYhg6w=
+k8s.io/kube-aggregator v0.26.4/go.mod h1:eWfg4tU0+l57ebWiS5THOANIJUrKRxudSVDJ+63bqvQ=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=

pkg/apiserver/apiserver.go

@@ -15,6 +15,7 @@
 package apiserver
 
 import (
+	"bytes"
 	"context"
 	"net"
 	"os"
@@ -29,6 +30,7 @@ import (
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
+	aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 
 	runtimev1alpha1 "antrea.io/nephe/apis/runtime/v1alpha1"
@@ -43,6 +45,10 @@ var (
 	apiServerPort = 5443
 	// Match Nephe Controller Service Name
 	nepheControllerSvcName = "nephe-controller-service"
+	// nepheServedLabel includes the labels used to select resources served by nephe-controller.
+	nepheServedLabel = map[string]string{
+		"served-by": "nephe-controller",
+	}
 )
 
 // ExtraConfig holds custom apiserver config.
@@ -134,18 +140,69 @@ func (s *NepheControllerAPIServer) SetupWithManager(
 		s.logger.Error(err, "unable to create APIServer config")
 		return err
 	}
-
-	s.genericAPIServer, err = apiConfig.Complete().New(mgr.GetScheme(), codecs, s.logger)
-	if err != nil {
+	if s.genericAPIServer, err = apiConfig.Complete().New(mgr.GetScheme(), codecs, s.logger); err != nil {
 		s.logger.Error(err, "unable to create APIServer")
 		return err
 	}
+	if err = s.syncAPIServices(certDir); err != nil {
+		s.logger.Error(err, "failed to sync CA cert with APIService")
+		return err
+	}
 	if err = mgr.Add(s); err != nil {
 		return err
 	}
 	return nil
 }
 
+// syncAPIServices updates nephe controller APIService with CA bundle.
+func (s *NepheControllerAPIServer) syncAPIServices(certDir string) error {
+	clientset, err := aggregatorclientset.NewForConfig(controllerruntime.GetConfigOrDie())
+	if err != nil {
+		return err
+	}
+
+	listOption := metav1.ListOptions{
+		LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{
+			MatchLabels: nepheServedLabel})}
+	nepheAPIServices, err := clientset.ApiregistrationV1().APIServices().List(context.TODO(), listOption)
+	if err != nil {
+		return err
+	}
+
+	if len(nepheAPIServices.Items) == 0 {
+		return nil
+	}
+
+	s.logger.Info("Syncing CA certificate with APIServices")
+	caCert, err := getCaCert(certDir)
+	if err != nil {
+		return err
+	}
+	for i := range nepheAPIServices.Items {
+		apiService := nepheAPIServices.Items[i]
+		if bytes.Equal(apiService.Spec.CABundle, caCert) {
+			continue
+		}
+		apiService.Spec.CABundle = caCert
+		if _, err := clientset.ApiregistrationV1().APIServices().Update(context.TODO(), &apiService, metav1.UpdateOptions{}); err != nil {
+			s.logger.Error(err, "failed to update CA cert of APIService", "name", apiService.Name)
+			return err
+		}
+		s.logger.Info("Updated CA cert of APIService", "name", apiService.Name)
+	}
+	return nil
+}
+
+// getCaCert gets the content of CA bundle from cert file.
+func getCaCert(certDir string) ([]byte, error) {
+	filePath := certDir + "/ca.crt"
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}
+
 type completedConfig struct {
 	GenericConfig genericapiserver.CompletedConfig
 	ExtraConfig   *ExtraConfig