From b8750525e0c558e35601b551af65e0d709927b94 Mon Sep 17 00:00:00 2001 From: Rahul Jain Date: Wed, 28 Jun 2023 12:57:10 -0700 Subject: [PATCH] Allow API server to update CA bundle on its service (#266) --- .../templates/controller/clusterrole.yaml | 7 +++ config/nephe.yml | 7 +++ config/rbac/role.yaml | 7 +++ go.mod | 1 + go.sum | 2 + pkg/apiserver/apiserver.go | 63 ++++++++++++++++++- 6 files changed, 84 insertions(+), 3 deletions(-) diff --git a/build/charts/nephe/templates/controller/clusterrole.yaml b/build/charts/nephe/templates/controller/clusterrole.yaml index c39a4ae5..9e6a932c 100644 --- a/build/charts/nephe/templates/controller/clusterrole.yaml +++ b/build/charts/nephe/templates/controller/clusterrole.yaml @@ -150,3 +150,10 @@ rules: - get - list - watch +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - list + - update diff --git a/config/nephe.yml b/config/nephe.yml index f1cf1925..286a9d42 100644 --- a/config/nephe.yml +++ b/config/nephe.yml @@ -441,6 +441,13 @@ rules: - get - list - watch +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - list + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4b595981..43345038 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -151,3 +151,10 @@ rules: - get - list - watch +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - list + - update diff --git a/go.mod b/go.mod index 85bac33b..919adfe3 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( k8s.io/apimachinery v0.26.4 k8s.io/apiserver v0.26.4 k8s.io/client-go v0.26.4 + k8s.io/kube-aggregator v0.26.4 sigs.k8s.io/controller-runtime v0.14.6 ) diff --git a/go.sum b/go.sum index 42548f7a..84d6519f 100644 --- a/go.sum +++ b/go.sum @@ -792,6 +792,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.26.4 h1:mQ+DeOvgAHC6+heZcozPkEd3rWtP4DVVjo1hLSih9w4= k8s.io/kms v0.26.4/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= +k8s.io/kube-aggregator v0.26.4 h1:iGljhq5exQkbuc3bnkwUx95RPCBDExg7DkX9XaYhg6w= +k8s.io/kube-aggregator v0.26.4/go.mod h1:eWfg4tU0+l57ebWiS5THOANIJUrKRxudSVDJ+63bqvQ= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 0067362c..5b53981b 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -15,6 +15,7 @@ package apiserver import ( + "bytes" "context" "net" "os" @@ -29,6 +30,7 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" controllerruntime "sigs.k8s.io/controller-runtime" runtimev1alpha1 "antrea.io/nephe/apis/runtime/v1alpha1" @@ -43,6 +45,10 @@ var ( apiServerPort = 5443 // Match Nephe Controller Service Name nepheControllerSvcName = "nephe-controller-service" + // nepheServedLabel includes the labels used to select resources served by nephe-controller. + nepheServedLabel = map[string]string{ + "served-by": "nephe-controller", + } ) // ExtraConfig holds custom apiserver config. @@ -134,18 +140,69 @@ func (s *NepheControllerAPIServer) SetupWithManager( s.logger.Error(err, "unable to create APIServer config") return err } - - s.genericAPIServer, err = apiConfig.Complete().New(mgr.GetScheme(), codecs, s.logger) - if err != nil { + if s.genericAPIServer, err = apiConfig.Complete().New(mgr.GetScheme(), codecs, s.logger); err != nil { s.logger.Error(err, "unable to create APIServer") return err } + if err = s.syncAPIServices(certDir); err != nil { + s.logger.Error(err, "failed to sync CA cert with APIService") + return err + } if err = mgr.Add(s); err != nil { return err } return nil } +// syncAPIServices updates nephe controller APIService with CA bundle. +func (s *NepheControllerAPIServer) syncAPIServices(certDir string) error { + clientset, err := aggregatorclientset.NewForConfig(controllerruntime.GetConfigOrDie()) + if err != nil { + return err + } + + listOption := metav1.ListOptions{ + LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{ + MatchLabels: nepheServedLabel})} + nepheAPIServices, err := clientset.ApiregistrationV1().APIServices().List(context.TODO(), listOption) + if err != nil { + return err + } + + if len(nepheAPIServices.Items) == 0 { + return nil + } + + s.logger.Info("Syncing CA certificate with APIServices") + caCert, err := getCaCert(certDir) + if err != nil { + return err + } + for i := range nepheAPIServices.Items { + apiService := nepheAPIServices.Items[i] + if bytes.Equal(apiService.Spec.CABundle, caCert) { + continue + } + apiService.Spec.CABundle = caCert + if _, err := clientset.ApiregistrationV1().APIServices().Update(context.TODO(), &apiService, metav1.UpdateOptions{}); err != nil { + s.logger.Error(err, "failed to update CA cert of APIService", "name", apiService.Name) + return err + } + s.logger.Info("Updated CA cert of APIService", "name", apiService.Name) + } + return nil +} + +// getCaCert gets the content of CA bundle from cert file. +func getCaCert(certDir string) ([]byte, error) { + filePath := certDir + "/ca.crt" + content, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + return content, nil +} + type completedConfig struct { GenericConfig genericapiserver.CompletedConfig ExtraConfig *ExtraConfig