From d608cf32c752abce775950d37cad71607e555d6e Mon Sep 17 00:00:00 2001 From: hysyeah Date: Mon, 23 Dec 2024 22:10:16 +0800 Subject: [PATCH] fix: patch deploy/sts pod template cause pod restart (#119) --- pkg/apiserver/filters.go | 1 + pkg/apiserver/handler.go | 4 + pkg/apiserver/handler_webhook.go | 122 +++++++++++++++++++++++++++++++ pkg/apiserver/webservice.go | 7 ++ pkg/appinstaller/helm.go | 9 --- pkg/webhook/setup.go | 88 ++++++++++++++++++++++ 6 files changed, 222 insertions(+), 9 deletions(-) diff --git a/pkg/apiserver/filters.go b/pkg/apiserver/filters.go index f42d18b..146f25b 100644 --- a/pkg/apiserver/filters.go +++ b/pkg/apiserver/filters.go @@ -94,6 +94,7 @@ func (h *Handler) authenticate(req *restful.Request, resp *restful.Response, cha "/app-service/v1/workflow/inject", "/app-service/v1/runasuser/inject", "/app-service/v1/terminus/version", + "/app-service/v1/app-label/inject", } needAuth := true diff --git a/pkg/apiserver/handler.go b/pkg/apiserver/handler.go index 8bbf573..771f03d 100644 --- a/pkg/apiserver/handler.go +++ b/pkg/apiserver/handler.go @@ -83,6 +83,10 @@ func (b *handlerBuilder) Build() (*Handler, error) { if err != nil { return nil, err } + err = wh.CreateOrUpdateAppLabelMutatingWebhook() + if err != nil { + return nil, err + } return &Handler{ kubeHost: b.ksHost, diff --git a/pkg/apiserver/handler_webhook.go b/pkg/apiserver/handler_webhook.go index 0dc205c..44cf876 100644 --- a/pkg/apiserver/handler_webhook.go +++ b/pkg/apiserver/handler_webhook.go @@ -777,3 +777,125 @@ func (h *Handler) runAsUserInject(ctx context.Context, pod *corev1.Pod, namespac return pod, nil } + +func (h *Handler) appLabelInject(req *restful.Request, resp *restful.Response) { + klog.Infof("Received mutating webhook[app-label inject] request: Method=%v, URL=%v", req.Request.Method, req.Request.URL) + admissionRequestBody, ok := h.sidecarWebhook.GetAdmissionRequestBody(req, resp) + if !ok { + return + } + var admissionReq, admissionResp admissionv1.AdmissionReview + proxyUUID := uuid.New() + if _, _, err := webhook.Deserializer.Decode(admissionRequestBody, nil, &admissionReq); err != nil { + klog.Errorf("Failed to decode admission request body err=%v", err) + admissionResp.Response = h.sidecarWebhook.AdmissionError("", err) + } else { + admissionResp.Response = h.appLabelMutate(req.Request.Context(), admissionReq.Request, proxyUUID) + } + admissionResp.TypeMeta = admissionReq.TypeMeta + admissionResp.Kind = admissionReq.Kind + + requestForNamespace := "unknown" + if admissionReq.Request != nil { + requestForNamespace = admissionReq.Request.Namespace + } + err := resp.WriteAsJson(&admissionResp) + if err != nil { + klog.Error("Failed to write response[app-label inject] admin review in namespace=%s err=%v", requestForNamespace, err) + return + } + klog.Infof("Done[app-label inject] with uuid=%s in namespace=%s", proxyUUID, requestForNamespace) +} + +func (h *Handler) appLabelMutate(ctx context.Context, req *admissionv1.AdmissionRequest, proxyUUID uuid.UUID) *admissionv1.AdmissionResponse { + if req == nil { + klog.Error("Failed to get admission Request, err=admission request is nil") + return h.sidecarWebhook.AdmissionError("", errNilAdmissionRequest) + } + klog.Infof("Enter appLabelMutate namespace=%s name=%s kind=%s", req.Namespace, req.Name, req.Kind.Kind) + + object := struct { + metav1.ObjectMeta `json:"metadata,omitempty"` + }{} + raw := req.Object.Raw + err := json.Unmarshal(raw, &object) + if err != nil { + klog.Errorf("Error unmarshalling request with UUID %s in namespace %s, error %v ", proxyUUID, req.Namespace, err) + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + + resp := &admissionv1.AdmissionResponse{ + Allowed: true, + UID: req.UID, + } + + appcfg, _ := h.sidecarWebhook.GetAppConfig(req.Namespace) + if appcfg == nil { + klog.Error("get appcfg is empty") + return resp + } + + appName := appcfg.AppName + if len(appName) == 0 || appName != object.Name { + return resp + } + + patchBytes, err := makePatches(req) + if err != nil { + klog.Errorf("make patches err=%v", patchBytes) + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + + klog.Info("patchBytes:", string(patchBytes)) + h.sidecarWebhook.PatchAdmissionResponse(resp, patchBytes) + return resp +} + +func makePatches(req *admissionv1.AdmissionRequest) ([]byte, error) { + original := req.Object.Raw + var patchBytes []byte + var tpl *corev1.PodTemplateSpec + switch req.Kind.Kind { + case "Deployment": + var deploy *appsv1.Deployment + if err := json.Unmarshal(req.Object.Raw, &deploy); err != nil { + klog.Errorf("Error unmarshaling request with UUID %s in namespace %s, %v", req.Namespace, err) + return []byte{}, err + } + tpl = &deploy.Spec.Template + if tpl.ObjectMeta.Labels == nil { + tpl.ObjectMeta.Labels = make(map[string]string) + } + tpl.ObjectMeta.Labels["io.bytetrade.app"] = "true" + current, err := json.Marshal(deploy) + if err != nil { + return []byte{}, err + } + admissionResponse := admission.PatchResponseFromRaw(original, current) + patchBytes, err = json.Marshal(admissionResponse.Patches) + if err != nil { + return []byte{}, err + } + case "StatefulSet": + var sts *appsv1.StatefulSet + if err := json.Unmarshal(req.Object.Raw, &sts); err != nil { + klog.Errorf("Error unmarshaling request with UUID %s in namespace %s, %v", req.Namespace, err) + return []byte{}, err + } + tpl = &sts.Spec.Template + if tpl.ObjectMeta.Labels == nil { + tpl.ObjectMeta.Labels = make(map[string]string) + } + tpl.ObjectMeta.Labels["io.bytetrade.app"] = "true" + current, err := json.Marshal(sts) + if err != nil { + return []byte{}, err + } + admissionResponse := admission.PatchResponseFromRaw(original, current) + patchBytes, err = json.Marshal(admissionResponse.Patches) + if err != nil { + return []byte{}, err + } + } + return patchBytes, nil +} diff --git a/pkg/apiserver/webservice.go b/pkg/apiserver/webservice.go index 0fc4597..b2b50b5 100644 --- a/pkg/apiserver/webservice.go +++ b/pkg/apiserver/webservice.go @@ -342,6 +342,13 @@ func addServiceToContainer(c *restful.Container, handler *Handler) error { Returns(http.StatusOK, "add limit success", nil)). Consumes(restful.MIME_JSON) + ws.Route(ws.POST("/app-label/inject"). + To(handler.appLabelInject). + Doc("add resources limits for deployment/statefulset"). + Metadata(restfulspec.KeyOpenAPITags, MODULE_TAGS). + Returns(http.StatusOK, "add limit success", nil)). + Consumes(restful.MIME_JSON) + ws.Route(ws.POST("/provider-registry/validate"). To(handler.providerRegistryValidate). Doc("validating webhook for validate app install namespace"). diff --git a/pkg/appinstaller/helm.go b/pkg/appinstaller/helm.go index 5cc7e10..a132e99 100644 --- a/pkg/appinstaller/helm.go +++ b/pkg/appinstaller/helm.go @@ -217,15 +217,6 @@ func (h *HelmOps) addApplicationLabelsToDeployment() error { constants.ApplicationSourceLabel: h.options.Source, }, }, - "spec": map[string]interface{}{ - "template": map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]string{ - "io.bytetrade.app": "true", - }, - }, - }, - }, } patchByte, err := json.Marshal(patchData) diff --git a/pkg/webhook/setup.go b/pkg/webhook/setup.go index f72a8bf..ef7f23b 100644 --- a/pkg/webhook/setup.go +++ b/pkg/webhook/setup.go @@ -23,9 +23,11 @@ const ( providerRegistryValidatingWebhookName = "provider-registry-validating-webhook.bytetrade.io" validatingWebhookName = "appns-validating-webhook.bytetrade.io" gpuLimitWebhookName = "gpu-limit-webhook" + appPodLabelWebhookName = "app-label-webhook" runAsUserWebhookName = "run-as-user-webhook" mutatingWebhookRunAsUserName = "run-as-user.bytetrade.io" mutatingWebhookGpuLimitName = "gpu-limit-inject-webhook.bytetrade.io" + mutatingWebhookAppLabelName = "app-label-inject-webhook.bytetrade.io" webhookServiceName = "app-service" webhookServiceNamespace = "os-system" defaultCaPath = "/etc/certs/ca.crt" @@ -661,3 +663,89 @@ func (wh *Webhook) CreateOrUpdateRunAsUserMutatingWebhook() error { klog.Infof("Finished creating MutatingWebhookConfiguration %s", runAsUserWebhookName) return nil } + +// CreateOrUpdateAppLabelMutatingWebhook creates or updates app mutating webhook. +func (wh *Webhook) CreateOrUpdateAppLabelMutatingWebhook() error { + webhookPath := "/app-service/v1/app-label/inject" + port, err := strconv.Atoi(strings.Split(constants.WebhookServerListenAddress, ":")[1]) + if err != nil { + return err + } + webhookPort := int32(port) + failurePolicy := admissionregv1.Fail + matchPolicy := admissionregv1.Exact + webhookTimeout := int32(30) + + mwhLabels := map[string]string{"velero.io/exclude-from-backup": "true"} + caBundle, err := ioutil.ReadFile(defaultCaPath) + if err != nil { + return err + } + mwh := admissionregv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: appPodLabelWebhookName, + Labels: mwhLabels, + }, + Webhooks: []admissionregv1.MutatingWebhook{ + { + Name: mutatingWebhookAppLabelName, + ClientConfig: admissionregv1.WebhookClientConfig{ + CABundle: caBundle, + Service: &admissionregv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &webhookPath, + Port: &webhookPort, + }, + }, + FailurePolicy: &failurePolicy, + MatchPolicy: &matchPolicy, + ObjectSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "tier", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"app-service"}, + }, + }, + }, + Rules: []admissionregv1.RuleWithOperations{ + { + Operations: []admissionregv1.OperationType{admissionregv1.Create}, + Rule: admissionregv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"v1"}, + Resources: []string{"deployments", "statefulsets"}, + }, + }, + }, + SideEffects: func() *admissionregv1.SideEffectClass { + sideEffect := admissionregv1.SideEffectClassNoneOnDryRun + return &sideEffect + }(), + TimeoutSeconds: &webhookTimeout, + AdmissionReviewVersions: []string{"v1"}}}, + } + if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), &mwh, metav1.CreateOptions{}); err != nil { + // Webhook already exists, update the webhook in this scenario + if apierrors.IsAlreadyExists(err) { + existing, err := wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.Background(), mwh.Name, metav1.GetOptions{}) + if err != nil { + klog.Errorf("Failed to get MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + mwh.ObjectMeta.ResourceVersion = existing.ObjectMeta.ResourceVersion + if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.Background(), &mwh, metav1.UpdateOptions{}); err != nil { + if !apierrors.IsConflict(err) { + klog.Errorf("Failed to update MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + } + } else { + klog.Errorf("Failed to create MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + } + klog.Infof("Finished creating MutatingWebhookConfiguration %s", appPodLabelWebhookName) + return nil +}