Skip to content

Commit

Permalink
Inject cdi-uploadproxy CA cert into user created routes
Browse files Browse the repository at this point in the history 
Routes should exist in the installation namespace and have

annotations:
  operator.cdi.kubevirt.io/injectUploadProxyCert: "true"

Signed-off-by: Michael Henriksen <[email protected]>
  • Loading branch information
@mhenriks
mhenriks committed Aug 26, 2024
1 parent 99c3cdb commit 66e7c09
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 27 deletions.
10 changes: 8 additions & 2 deletions pkg/operator/controller/callbacks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,16 @@ import (
func addReconcileCallbacks(r *ReconcileCDI) {
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileDeleteControllerDeployment)
r.reconciler.AddCallback(&corev1.ServiceAccount{}, reconcileSCC)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileCreateRoute)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileCreatePrometheusInfra)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileRemainingRelationshipLabels)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileDeleteDeprecatedResources)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileCDICRD)
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcilePvcMutatingWebhook)
r.reconciler.AddCallback(&extv1.CustomResourceDefinition{}, reconcileSetConfigAuthority)
r.reconciler.AddCallback(&extv1.CustomResourceDefinition{}, reconcileHandleOldVersion)
if r.haveRoutes {
r.reconciler.AddCallback(&appsv1.Deployment{}, reconcileRoute)
}
}

func isControllerDeployment(d *appsv1.Deployment) bool {
Expand Down Expand Up @@ -109,7 +111,7 @@ func reconcileDeleteControllerDeployment(args *callbacks.ReconcileCallbackArgs)
return nil
}

func reconcileCreateRoute(args *callbacks.ReconcileCallbackArgs) error {
func reconcileRoute(args *callbacks.ReconcileCallbackArgs) error {
if args.State != callbacks.ReconcileStatePostRead {
return nil
}
Expand All @@ -126,6 +128,10 @@ func reconcileCreateRoute(args *callbacks.ReconcileCallbackArgs) error {
}
args.Recorder.Event(cr, corev1.EventTypeNormal, createResourceSuccess, "Successfully ensured upload proxy route exists")

if err := updateUserRoutes(context.TODO(), args.Logger, args.Client, args.Recorder); err != nil {
return err
}

return nil
}

Expand Down
10 changes: 10 additions & 0 deletions pkg/operator/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ const (
deleteResourceFailed = "DeleteResourceFailed"
deleteResourceSuccess = "DeleteResourceSuccess"
dumpInstallStrategyKey = "DUMP_INSTALL_STRATEGY"

annInjectUploadProxyCert = "operator.cdi.kubevirt.io/injectUploadProxyCert"
updateUserRouteSuccess = "UploadProxyRouteInjectSuccess"
)

var (
Expand Down Expand Up @@ -139,6 +142,11 @@ func newReconciler(mgr manager.Manager) (*ReconcileCDI, error) {

recorder := mgr.GetEventRecorderFor("operator-controller")

haveRoutes, err := haveRoutes(uncachedClient)
if err != nil {
return nil, err
}

r := &ReconcileCDI{
client: restClient,
uncachedClient: uncachedClient,
Expand All @@ -149,6 +157,7 @@ func newReconciler(mgr manager.Manager) (*ReconcileCDI, error) {
clusterArgs: clusterArgs,
namespacedArgs: &namespacedArgs,
dumpInstallStrategy: dumpInstallStrategy,
haveRoutes: haveRoutes,
}
callbackDispatcher := callbacks.NewCallbackDispatcher(log, restClient, uncachedClient, scheme, namespace)
r.reconciler = sdkr.NewReconciler(r, log, restClient, callbackDispatcher, scheme, mgr.GetCache, createVersionLabel, updateVersionLabel, LastAppliedConfigAnnotation, certPollInterval, finalizerName, false, recorder)
Expand Down Expand Up @@ -181,6 +190,7 @@ type ReconcileCDI struct {
certManager CertManager
reconciler *sdkr.Reconciler
dumpInstallStrategy bool
haveRoutes bool
}

// SetController sets the controller dependency
Expand Down
35 changes: 33 additions & 2 deletions pkg/operator/controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,35 @@ var _ = Describe("Controller", func() {
validateEvents(args.reconciler, createReadyEventValidationMap())
})

It("should update existing route", func() {
route := &routev1.Route{
ObjectMeta: metav1.ObjectMeta{
Name: "user-route",
Namespace: cdiNamespace,
Annotations: map[string]string{
"operator.cdi.kubevirt.io/injectUploadProxyCert": "true",
},
},
Spec: routev1.RouteSpec{
TLS: &routev1.TLSConfig{},
},
}

args := createArgs(route)
doReconcile(args)
Expect(setDeploymentsReady(args)).To(BeTrue())

obj, err := getObject(args.client, route)
Expect(err).ToNot(HaveOccurred())
route = obj.(*routev1.Route)

Expect(route.Spec.TLS.DestinationCACertificate).Should(Equal(testCertData))

eventMap := createReadyEventValidationMap()
eventMap["Normal UploadProxyRouteInjectSuccess Successfully updated Route destination CA certificate"] = false
validateEvents(args.reconciler, eventMap)
})

It("should have CDIOperatorDown", func() {
args := createArgs()
doReconcile(args)
Expand Down Expand Up @@ -1635,9 +1664,10 @@ func createFromArgs(version string) *args {
}
}

func createArgs() *args {
func createArgs(objs ...client.Object) *args {
cdi := createCDI("cdi", "good uid")
client := createClient(cdi)
objs = append(objs, cdi)
client := createClient(objs...)
reconciler := createReconciler(client)

return &args{
Expand Down Expand Up @@ -1753,6 +1783,7 @@ func createReconciler(client client.Client) *ReconcileCDI {
clusterArgs: clusterArgs,
namespacedArgs: namespacedArgs,
certManager: newFakeCertManager(client, namespace),
haveRoutes: true,
}
callbackDispatcher := callbacks.NewCallbackDispatcher(log, client, client, scheme.Scheme, namespace)
getCache := func() cache.Cache {
Expand Down
92 changes: 69 additions & 23 deletions pkg/operator/controller/route.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand All @@ -49,21 +50,15 @@ func ensureUploadProxyRouteExists(ctx context.Context, logger logr.Logger, c cli
return fmt.Errorf("cluster scoped owner not supported")
}

cm := &corev1.ConfigMap{}
key := client.ObjectKey{Namespace: namespace, Name: uploadProxyCABundle}
if err := c.Get(ctx, key, cm); err != nil {
cert, err := getUploadProxyCABundle(ctx, c)
if err != nil {
if errors.IsNotFound(err) {
logger.V(3).Info("upload proxy ca cert doesn't exist yet")
logger.V(3).Info("ensureUploadProxyRouteExists() upload proxy ca cert doesn't exist")
return nil
}
return err
}

cert, exists := cm.Data["ca-bundle.crt"]
if !exists {
return fmt.Errorf("unexpected ConfigMap format, 'ca-bundle.crt' key missing")
}

cr, err := cc.GetActiveCDI(ctx, c)
if err != nil {
return err
Expand Down Expand Up @@ -100,7 +95,7 @@ func ensureUploadProxyRouteExists(ctx context.Context, logger logr.Logger, c cli
util.SetRecommendedLabels(desiredRoute, installerLabels, "cdi-operator")

currentRoute := &routev1.Route{}
key = client.ObjectKey{Namespace: namespace, Name: uploadProxyRouteName}
key := client.ObjectKey{Namespace: namespace, Name: uploadProxyRouteName}
err = c.Get(ctx, key, currentRoute)
if err == nil {
if currentRoute.Spec.To.Kind != desiredRoute.Spec.To.Kind ||
Expand All @@ -115,12 +110,6 @@ func ensureUploadProxyRouteExists(ctx context.Context, logger logr.Logger, c cli
return nil
}

if meta.IsNoMatchError(err) {
// not in openshift
logger.V(3).Info("No match error for Route, must not be in openshift")
return nil
}

if !errors.IsNotFound(err) {
return err
}
Expand All @@ -132,18 +121,75 @@ func ensureUploadProxyRouteExists(ctx context.Context, logger logr.Logger, c cli
return c.Create(ctx, desiredRoute)
}

func (r *ReconcileCDI) watchRoutes() error {
err := r.uncachedClient.List(context.TODO(), &routev1.RouteList{}, &client.ListOptions{
func updateUserRoutes(ctx context.Context, logger logr.Logger, c client.Client, recorder record.EventRecorder) error {
cert, err := getUploadProxyCABundle(ctx, c)
if err != nil {
if errors.IsNotFound(err) {
logger.V(3).Info("updateUserRoutes() upload proxy ca cert doesn't exist")
return nil
}
return err
}

routes := &routev1.RouteList{}
err = c.List(ctx, routes, &client.ListOptions{
Namespace: util.GetNamespace(),
Limit: 1,
})
if err == nil {
return r.controller.Watch(source.Kind(r.getCache(), &routev1.Route{}), enqueueCDI(r.client))
if err != nil {
return err
}
if meta.IsNoMatchError(err) {

for _, r := range routes.Items {
route := r.DeepCopy()
if route.Annotations[annInjectUploadProxyCert] != "true" {
continue
}

if route.Spec.TLS == nil {
logger.V(1).Info("Route has no TLS config, skipping", "route", route.Name)
continue
}

if route.Spec.TLS.DestinationCACertificate != cert {
logger.V(1).Info("Updating route with new CA cert", "route", route.Name)
route.Spec.TLS.DestinationCACertificate = cert
if err := c.Update(ctx, route); err != nil {
return err
}
recorder.Event(route, corev1.EventTypeNormal, updateUserRouteSuccess, "Successfully updated Route destination CA certificate")
}
}

return nil
}

func getUploadProxyCABundle(ctx context.Context, c client.Client) (string, error) {
cm := &corev1.ConfigMap{}
key := client.ObjectKey{Namespace: util.GetNamespace(), Name: uploadProxyCABundle}
if err := c.Get(ctx, key, cm); err != nil {
return "", err
}
return cm.Data["ca-bundle.crt"], nil
}

func (r *ReconcileCDI) watchRoutes() error {
if !r.haveRoutes {
log.Info("Not watching Routes")
return nil
}
return r.controller.Watch(source.Kind(r.getCache(), &routev1.Route{}), enqueueCDI(r.client))
}

return err
func haveRoutes(c client.Client) (bool, error) {
err := c.List(context.TODO(), &routev1.RouteList{}, &client.ListOptions{
Namespace: util.GetNamespace(),
Limit: 1,
})
if err != nil {
if meta.IsNoMatchError(err) {
return false, nil
}
return false, err
}
return true, nil
}

0 comments on commit 66e7c09

Please sign in to comment.