From cf477d2d1d2404f575b990a0fddab3c66e231e97 Mon Sep 17 00:00:00 2001 From: Mitali Salvi <44349099+mitali-salvi@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:13:31 -0500 Subject: [PATCH] Implementing workload webhook for auto-annotation (#73) --- config/rbac/role.yaml | 1 + config/webhook/manifests.yaml | 22 +++ .../workloadmutation/webhookhandler.go | 92 +++++++++++ .../workloadmutation/webhookhandler_test.go | 149 ++++++++++++++++++ main.go | 45 +++--- pkg/instrumentation/auto/annotation.go | 14 +- pkg/instrumentation/auto/annotation_test.go | 4 + 7 files changed, 301 insertions(+), 26 deletions(-) create mode 100644 internal/webhook/workloadmutation/webhookhandler.go create mode 100644 internal/webhook/workloadmutation/webhookhandler_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2e8b0609b..262d9d50e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,6 +32,7 @@ rules: - namespaces verbs: - list + - update - watch - apiGroups: - apps diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ce347fbcf..6bf2cde85 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -64,6 +64,28 @@ webhooks: resources: - pods sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-v1-workload + failurePolicy: Ignore + name: mworkload.kb.io + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + - deployments + - statefulsets + sideEffects: None --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go new file mode 100644 index 000000000..f43e7dca8 --- /dev/null +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -0,0 +1,92 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package workloadmutation contains the webhook that injects annotations into daemon-sets, deployments and stateful-sets. +package workloadmutation + +import ( + "context" + "encoding/json" + "errors" + "net/http" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/aws/amazon-cloudwatch-agent-operator/internal/config" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +// +kubebuilder:webhook:path=/mutate-v1-workload,mutating=true,failurePolicy=ignore,groups="apps",resources=daemonsets;deployments;statefulsets,verbs=create;update,versions=v1,name=mworkload.kb.io,sideEffects=none,admissionReviewVersions=v1 +// +kubebuilder:rbac:groups="apps",resources=daemonsets;deployments;statefulsets,verbs=get;list;watch + +var _ WebhookHandler = (*workloadMutationWebhook)(nil) + +// WebhookHandler is a webhook handler that analyzes new daemon-sets and injects appropriate annotations into it. +type WebhookHandler interface { + admission.Handler +} + +// the implementation. +type workloadMutationWebhook struct { + client client.Client + decoder *admission.Decoder + logger logr.Logger + config config.Config + annotationMutator *auto.AnnotationMutators +} + +// NewWebhookHandler creates a new WorkloadWebhookHandler. +func NewWebhookHandler(cfg config.Config, logger logr.Logger, decoder *admission.Decoder, cl client.Client, annotationMutation *auto.AnnotationMutators) WebhookHandler { + return &workloadMutationWebhook{ + config: cfg, + decoder: decoder, + logger: logger, + client: cl, + annotationMutator: annotationMutation, + } +} + +func (p *workloadMutationWebhook) Handle(ctx context.Context, req admission.Request) admission.Response { + var err error + var marshaledObject []byte + var object runtime.Object + switch objectKind := req.Kind.Kind; objectKind { + case "DaemonSet": + ds := appsv1.DaemonSet{} + err = p.decoder.Decode(req, &ds) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + object = &ds + case "Deployment": + d := appsv1.Deployment{} + err = p.decoder.Decode(req, &d) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + object = &d + case "StatefulSet": + ss := appsv1.StatefulSet{} + err = p.decoder.Decode(req, &ss) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + object = &ss + default: + return admission.Errored(http.StatusBadRequest, errors.New("failed to unmarshal request object")) + } + + p.annotationMutator.Mutate(object) + marshaledObject, err = json.Marshal(object) + if err != nil { + res := admission.Errored(http.StatusInternalServerError, err) + res.Allowed = true + return res + } + + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledObject) +} diff --git a/internal/webhook/workloadmutation/webhookhandler_test.go b/internal/webhook/workloadmutation/webhookhandler_test.go new file mode 100644 index 000000000..d94e6d4a6 --- /dev/null +++ b/internal/webhook/workloadmutation/webhookhandler_test.go @@ -0,0 +1,149 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package workloadmutation + +import ( + "context" + "encoding/json" + "github.com/stretchr/testify/require" + admv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "net/http" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "k8s.io/kubectl/pkg/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/aws/amazon-cloudwatch-agent-operator/internal/config" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" +) + +var ( + k8sClient client.Client + logger = logf.Log.WithName("unit-tests") +) + +func TestHandle(t *testing.T) { + for _, tt := range []struct { + req admission.Request + name string + expected int32 + allowed bool + }{ + { + name: "empty payload", + req: admission.Request{}, + expected: http.StatusBadRequest, + allowed: false, + }, + { + name: "invalid empty daemon-set payload", + req: func() admission.Request { + ds := appsv1.DaemonSet{} + encoded, err := json.Marshal(ds) + require.NoError(t, err) + + return admission.Request{ + AdmissionRequest: admv1.AdmissionRequest{ + Namespace: "testing", + Object: runtime.RawExtension{ + Raw: encoded, + }, + }, + } + }(), + expected: http.StatusBadRequest, + allowed: false, + }, + { + name: "invalid pod payload", + req: func() admission.Request { + pod := corev1.Pod{} + encoded, err := json.Marshal(pod) + require.NoError(t, err) + + return admission.Request{ + AdmissionRequest: admv1.AdmissionRequest{ + Namespace: "testing", + Object: runtime.RawExtension{ + Raw: encoded, + }, + }, + } + }(), + expected: http.StatusBadRequest, + allowed: false, + }, + { + name: "valid workload payload", + req: func() admission.Request { + ds := appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + }, + } + encoded, err := json.Marshal(ds) + require.NoError(t, err) + + return admission.Request{ + AdmissionRequest: admv1.AdmissionRequest{ + Kind: metav1.GroupVersionKind{ + Kind: "DaemonSet", + }, + Namespace: "testing", + Object: runtime.RawExtension{ + Raw: encoded, + }, + }, + } + }(), + expected: http.StatusOK, + allowed: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // prepare + cfg := config.New() + decoder := admission.NewDecoder(scheme.Scheme) + autoAnnotationConfig := auto.AnnotationConfig{ + Java: auto.AnnotationResources{ + Namespaces: []string{"keep-auto-java"}, + }, + } + mutators := auto.NewAnnotationMutators( + k8sClient, + k8sClient, + logr.Logger{}, + autoAnnotationConfig, + instrumentation.NewTypeSet(instrumentation.TypeJava), + ) + injector := NewWebhookHandler(cfg, logger, decoder, k8sClient, mutators) + + // test + res := injector.Handle(context.Background(), tt.req) + + // verify + assert.Equal(t, tt.allowed, res.Allowed) + if !tt.allowed { + assert.NotNil(t, res.AdmissionResponse.Result) + assert.Equal(t, tt.expected, res.AdmissionResponse.Result.Code) + } + }) + } +} diff --git a/main.go b/main.go index 987f34bb2..f0a934204 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent-operator/internal/config" "github.com/aws/amazon-cloudwatch-agent-operator/internal/version" "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/podmutation" + "github.com/aws/amazon-cloudwatch-agent-operator/internal/webhook/workloadmutation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/featuregate" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/auto" @@ -185,6 +186,29 @@ func main() { os.Exit(1) } + decoder := admission.NewDecoder(mgr.GetScheme()) + + if os.Getenv("DISABLE_AUTO_ANNOTATION") != "true" { + var autoAnnotationConfig auto.AnnotationConfig + if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { + setupLog.Error(err, "unable to unmarshal auto-annotation config") + } else { + autoAnnotationMutator := auto.NewAnnotationMutators( + mgr.GetClient(), + mgr.GetAPIReader(), + logger, + autoAnnotationConfig, + instrumentation.NewTypeSet( + instrumentation.TypeJava, + instrumentation.TypePython, + ), + ) + mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ + Handler: workloadmutation.NewWebhookHandler(cfg, ctrl.Log.WithName("workload-webhook"), decoder, mgr.GetClient(), autoAnnotationMutator)}) + go autoAnnotationMutator.MutateAll(ctx) + } + } + if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = otelv1alpha1.SetupCollectorWebhook(mgr, cfg); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AmazonCloudWatchAgent") @@ -194,7 +218,6 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "Instrumentation") os.Exit(1) } - decoder := admission.NewDecoder(mgr.GetScheme()) mgr.GetWebhookServer().Register("/mutate-v1-pod", &webhook.Admission{ Handler: podmutation.NewWebhookHandler(cfg, ctrl.Log.WithName("pod-webhook"), decoder, mgr.GetClient(), []podmutation.PodMutator{ @@ -202,7 +225,6 @@ func main() { instrumentation.NewMutator(logger, mgr.GetClient(), mgr.GetEventRecorderFor("amazon-cloudwatch-agent-operator")), }), }) - } else { ctrl.Log.Info("Webhooks are disabled, operator is running an unsupported mode", "ENABLE_WEBHOOKS", "false") } @@ -217,25 +239,6 @@ func main() { os.Exit(1) } - if os.Getenv("DISABLE_AUTO_ANNOTATION") != "true" { - var autoAnnotationConfig auto.AnnotationConfig - if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { - setupLog.Error(err, "unable to unmarshal auto-annotation config") - } else { - setupLog.Info("starting auto-annotator") - autoAnnotation := auto.NewAnnotationMutators( - mgr.GetClient(), - logger, - autoAnnotationConfig, - instrumentation.NewTypeSet( - instrumentation.TypeJava, - instrumentation.TypePython, - ), - ) - go autoAnnotation.MutateAll(ctx) - } - } - setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index d42c78806..1cc32803d 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -24,9 +24,12 @@ const ( defaultAnnotationValue = "true" ) +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch;update + // AnnotationMutators has an AnnotationMutator resource name type AnnotationMutators struct { client client.Client + clientReader client.Reader logger logr.Logger namespaceMutators map[string]instrumentation.AnnotationMutator deploymentMutators map[string]instrumentation.AnnotationMutator @@ -46,7 +49,7 @@ func (m *AnnotationMutators) MutateAll(ctx context.Context) { // MutateNamespaces lists all namespaces and runs MutateNamespace on each. func (m *AnnotationMutators) MutateNamespaces(ctx context.Context) { namespaces := &corev1.NamespaceList{} - if err := m.client.List(ctx, namespaces); err != nil { + if err := m.clientReader.List(ctx, namespaces); err != nil { m.logger.Error(err, "Unable to list namespaces") return } @@ -66,7 +69,7 @@ func (m *AnnotationMutators) MutateNamespaces(ctx context.Context) { // MutateDeployments lists all deployments and runs MutateDeployment on each. func (m *AnnotationMutators) MutateDeployments(ctx context.Context) { deployments := &appsv1.DeploymentList{} - if err := m.client.List(ctx, deployments); err != nil { + if err := m.clientReader.List(ctx, deployments); err != nil { m.logger.Error(err, "Unable to list deployments") return } @@ -86,7 +89,7 @@ func (m *AnnotationMutators) MutateDeployments(ctx context.Context) { // MutateDaemonSets lists all daemonsets and runs MutateDaemonSet on each. func (m *AnnotationMutators) MutateDaemonSets(ctx context.Context) { daemonSets := &appsv1.DaemonSetList{} - if err := m.client.List(ctx, daemonSets); err != nil { + if err := m.clientReader.List(ctx, daemonSets); err != nil { m.logger.Error(err, "Unable to list daemonsets") return } @@ -106,7 +109,7 @@ func (m *AnnotationMutators) MutateDaemonSets(ctx context.Context) { // MutateStatefulSets lists all statefulsets and runs MutateStatefulSet on each. func (m *AnnotationMutators) MutateStatefulSets(ctx context.Context) { statefulSets := &appsv1.StatefulSetList{} - if err := m.client.List(ctx, statefulSets); err != nil { + if err := m.clientReader.List(ctx, statefulSets); err != nil { m.logger.Error(err, "Unable to list statefulsets") return } @@ -154,10 +157,11 @@ func namespacedName(obj metav1.Object) string { // NewAnnotationMutators creates mutators based on the AnnotationConfig provided and enabled instrumentation.TypeSet. // The default mutator, which is used for non-configured resources, removes all auto-annotated annotations in the type // set. -func NewAnnotationMutators(client client.Client, logger logr.Logger, cfg AnnotationConfig, typeSet instrumentation.TypeSet) *AnnotationMutators { +func NewAnnotationMutators(client client.Client, clientReader client.Reader, logger logr.Logger, cfg AnnotationConfig, typeSet instrumentation.TypeSet) *AnnotationMutators { builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ client: client, + clientReader: clientReader, logger: logger, namespaceMutators: builder.buildMutators(getResources(cfg, typeSet, getNamespaces)), deploymentMutators: builder.buildMutators(getResources(cfg, typeSet, getDeployments)), diff --git a/pkg/instrumentation/auto/annotation_test.go b/pkg/instrumentation/auto/annotation_test.go index 1d3b87d9b..130e32c5c 100644 --- a/pkg/instrumentation/auto/annotation_test.go +++ b/pkg/instrumentation/auto/annotation_test.go @@ -106,6 +106,7 @@ func TestAnnotationMutators_Namespaces(t *testing.T) { ctx := context.Background() client := fake.NewClientBuilder().WithLists(&corev1.NamespaceList{Items: namespaces}).Build() mutators := NewAnnotationMutators( + client, client, logr.Logger{}, testCase.cfg, @@ -172,6 +173,7 @@ func TestAnnotationMutators_Deployments(t *testing.T) { ctx := context.Background() client := fake.NewClientBuilder().WithLists(&appv1.DeploymentList{Items: deployments}).Build() mutators := NewAnnotationMutators( + client, client, logr.Logger{}, testCase.cfg, @@ -239,6 +241,7 @@ func TestAnnotationMutators_DaemonSets(t *testing.T) { ctx := context.Background() client := fake.NewClientBuilder().WithLists(&appv1.DaemonSetList{Items: daemonSets}).Build() mutators := NewAnnotationMutators( + client, client, logr.Logger{}, testCase.cfg, @@ -306,6 +309,7 @@ func TestAnnotationMutators_StatefulSets(t *testing.T) { ctx := context.Background() client := fake.NewClientBuilder().WithLists(&appv1.StatefulSetList{Items: statefulSets}).Build() mutators := NewAnnotationMutators( + client, client, logr.Logger{}, testCase.cfg,