fix: patch deploy/sts pod template cause pod restart (#119)

pkg/apiserver/filters.go

@@ -94,6 +94,7 @@ func (h *Handler) authenticate(req *restful.Request, resp *restful.Response, cha
 		"/app-service/v1/workflow/inject",
 		"/app-service/v1/runasuser/inject",
 		"/app-service/v1/terminus/version",
+		"/app-service/v1/app-label/inject",
 	}
 
 	needAuth := true

pkg/apiserver/handler.go

@@ -83,6 +83,10 @@ func (b *handlerBuilder) Build() (*Handler, error) {
 	if err != nil {
 		return nil, err
 	}
+	err = wh.CreateOrUpdateAppLabelMutatingWebhook()
+	if err != nil {
+		return nil, err
+	}
 
 	return &Handler{
 		kubeHost:         b.ksHost,

pkg/apiserver/handler_webhook.go

@@ -777,3 +777,125 @@ func (h *Handler) runAsUserInject(ctx context.Context, pod *corev1.Pod, namespac
 
 	return pod, nil
 }
+
+func (h *Handler) appLabelInject(req *restful.Request, resp *restful.Response) {
+	klog.Infof("Received mutating webhook[app-label inject] request: Method=%v, URL=%v", req.Request.Method, req.Request.URL)
+	admissionRequestBody, ok := h.sidecarWebhook.GetAdmissionRequestBody(req, resp)
+	if !ok {
+		return
+	}
+	var admissionReq, admissionResp admissionv1.AdmissionReview
+	proxyUUID := uuid.New()
+	if _, _, err := webhook.Deserializer.Decode(admissionRequestBody, nil, &admissionReq); err != nil {
+		klog.Errorf("Failed to decode admission request body err=%v", err)
+		admissionResp.Response = h.sidecarWebhook.AdmissionError("", err)
+	} else {
+		admissionResp.Response = h.appLabelMutate(req.Request.Context(), admissionReq.Request, proxyUUID)
+	}
+	admissionResp.TypeMeta = admissionReq.TypeMeta
+	admissionResp.Kind = admissionReq.Kind
+
+	requestForNamespace := "unknown"
+	if admissionReq.Request != nil {
+		requestForNamespace = admissionReq.Request.Namespace
+	}
+	err := resp.WriteAsJson(&admissionResp)
+	if err != nil {
+		klog.Error("Failed to write response[app-label inject] admin review in namespace=%s err=%v", requestForNamespace, err)
+		return
+	}
+	klog.Infof("Done[app-label inject] with uuid=%s in namespace=%s", proxyUUID, requestForNamespace)
+}
+
+func (h *Handler) appLabelMutate(ctx context.Context, req *admissionv1.AdmissionRequest, proxyUUID uuid.UUID) *admissionv1.AdmissionResponse {
+	if req == nil {
+		klog.Error("Failed to get admission Request, err=admission request is nil")
+		return h.sidecarWebhook.AdmissionError("", errNilAdmissionRequest)
+	}
+	klog.Infof("Enter appLabelMutate namespace=%s name=%s kind=%s", req.Namespace, req.Name, req.Kind.Kind)
+
+	object := struct {
+		metav1.ObjectMeta `json:"metadata,omitempty"`
+	}{}
+	raw := req.Object.Raw
+	err := json.Unmarshal(raw, &object)
+	if err != nil {
+		klog.Errorf("Error unmarshalling request with UUID %s in namespace %s, error %v ", proxyUUID, req.Namespace, err)
+		return h.sidecarWebhook.AdmissionError(req.UID, err)
+	}
+
+	resp := &admissionv1.AdmissionResponse{
+		Allowed: true,
+		UID:     req.UID,
+	}
+
+	appcfg, _ := h.sidecarWebhook.GetAppConfig(req.Namespace)
+	if appcfg == nil {
+		klog.Error("get appcfg is empty")
+		return resp
+	}
+
+	appName := appcfg.AppName
+	if len(appName) == 0 || appName != object.Name {
+		return resp
+	}
+
+	patchBytes, err := makePatches(req)
+	if err != nil {
+		klog.Errorf("make patches err=%v", patchBytes)
+		return h.sidecarWebhook.AdmissionError(req.UID, err)
+	}
+
+	klog.Info("patchBytes:", string(patchBytes))
+	h.sidecarWebhook.PatchAdmissionResponse(resp, patchBytes)
+	return resp
+}
+
+func makePatches(req *admissionv1.AdmissionRequest) ([]byte, error) {
+	original := req.Object.Raw
+	var patchBytes []byte
+	var tpl *corev1.PodTemplateSpec
+	switch req.Kind.Kind {
+	case "Deployment":
+		var deploy *appsv1.Deployment
+		if err := json.Unmarshal(req.Object.Raw, &deploy); err != nil {
+			klog.Errorf("Error unmarshaling request with UUID %s in namespace %s, %v", req.Namespace, err)
+			return []byte{}, err
+		}
+		tpl = &deploy.Spec.Template
+		if tpl.ObjectMeta.Labels == nil {
+			tpl.ObjectMeta.Labels = make(map[string]string)
+		}
+		tpl.ObjectMeta.Labels["io.bytetrade.app"] = "true"
+		current, err := json.Marshal(deploy)
+		if err != nil {
+			return []byte{}, err
+		}
+		admissionResponse := admission.PatchResponseFromRaw(original, current)
+		patchBytes, err = json.Marshal(admissionResponse.Patches)
+		if err != nil {
+			return []byte{}, err
+		}
+	case "StatefulSet":
+		var sts *appsv1.StatefulSet
+		if err := json.Unmarshal(req.Object.Raw, &sts); err != nil {
+			klog.Errorf("Error unmarshaling request with UUID %s in namespace %s, %v", req.Namespace, err)
+			return []byte{}, err
+		}
+		tpl = &sts.Spec.Template
+		if tpl.ObjectMeta.Labels == nil {
+			tpl.ObjectMeta.Labels = make(map[string]string)
+		}
+		tpl.ObjectMeta.Labels["io.bytetrade.app"] = "true"
+		current, err := json.Marshal(sts)
+		if err != nil {
+			return []byte{}, err
+		}
+		admissionResponse := admission.PatchResponseFromRaw(original, current)
+		patchBytes, err = json.Marshal(admissionResponse.Patches)
+		if err != nil {
+			return []byte{}, err
+		}
+	}
+	return patchBytes, nil
+}

pkg/apiserver/webservice.go

@@ -342,6 +342,13 @@ func addServiceToContainer(c *restful.Container, handler *Handler) error {
 		Returns(http.StatusOK, "add limit success", nil)).
 		Consumes(restful.MIME_JSON)
 
+	ws.Route(ws.POST("/app-label/inject").
+		To(handler.appLabelInject).
+		Doc("add resources limits for deployment/statefulset").
+		Metadata(restfulspec.KeyOpenAPITags, MODULE_TAGS).
+		Returns(http.StatusOK, "add limit success", nil)).
+		Consumes(restful.MIME_JSON)
+
 	ws.Route(ws.POST("/provider-registry/validate").
 		To(handler.providerRegistryValidate).
 		Doc("validating webhook for validate app install namespace").

pkg/appinstaller/helm.go

@@ -217,15 +217,6 @@ func (h *HelmOps) addApplicationLabelsToDeployment() error {
 				constants.ApplicationSourceLabel:  h.options.Source,
 			},
 		},
-		"spec": map[string]interface{}{
-			"template": map[string]interface{}{
-				"metadata": map[string]interface{}{
-					"labels": map[string]string{
-						"io.bytetrade.app": "true",
-					},
-				},
-			},
-		},
 	}
 
 	patchByte, err := json.Marshal(patchData)

pkg/webhook/setup.go

@@ -23,9 +23,11 @@ const (
 	providerRegistryValidatingWebhookName = "provider-registry-validating-webhook.bytetrade.io"
 	validatingWebhookName                 = "appns-validating-webhook.bytetrade.io"
 	gpuLimitWebhookName                   = "gpu-limit-webhook"
+	appPodLabelWebhookName                = "app-label-webhook"
 	runAsUserWebhookName                  = "run-as-user-webhook"
 	mutatingWebhookRunAsUserName          = "run-as-user.bytetrade.io"
 	mutatingWebhookGpuLimitName           = "gpu-limit-inject-webhook.bytetrade.io"
+	mutatingWebhookAppLabelName           = "app-label-inject-webhook.bytetrade.io"
 	webhookServiceName                    = "app-service"
 	webhookServiceNamespace               = "os-system"
 	defaultCaPath                         = "/etc/certs/ca.crt"
@@ -661,3 +663,89 @@ func (wh *Webhook) CreateOrUpdateRunAsUserMutatingWebhook() error {
 	klog.Infof("Finished creating MutatingWebhookConfiguration %s", runAsUserWebhookName)
 	return nil
 }
+
+// CreateOrUpdateAppLabelMutatingWebhook creates or updates app mutating webhook.
+func (wh *Webhook) CreateOrUpdateAppLabelMutatingWebhook() error {
+	webhookPath := "/app-service/v1/app-label/inject"
+	port, err := strconv.Atoi(strings.Split(constants.WebhookServerListenAddress, ":")[1])
+	if err != nil {
+		return err
+	}
+	webhookPort := int32(port)
+	failurePolicy := admissionregv1.Fail
+	matchPolicy := admissionregv1.Exact
+	webhookTimeout := int32(30)
+
+	mwhLabels := map[string]string{"velero.io/exclude-from-backup": "true"}
+	caBundle, err := ioutil.ReadFile(defaultCaPath)
+	if err != nil {
+		return err
+	}
+	mwh := admissionregv1.MutatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   appPodLabelWebhookName,
+			Labels: mwhLabels,
+		},
+		Webhooks: []admissionregv1.MutatingWebhook{
+			{
+				Name: mutatingWebhookAppLabelName,
+				ClientConfig: admissionregv1.WebhookClientConfig{
+					CABundle: caBundle,
+					Service: &admissionregv1.ServiceReference{
+						Namespace: webhookServiceNamespace,
+						Name:      webhookServiceName,
+						Path:      &webhookPath,
+						Port:      &webhookPort,
+					},
+				},
+				FailurePolicy: &failurePolicy,
+				MatchPolicy:   &matchPolicy,
+				ObjectSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "tier",
+							Operator: metav1.LabelSelectorOpNotIn,
+							Values:   []string{"app-service"},
+						},
+					},
+				},
+				Rules: []admissionregv1.RuleWithOperations{
+					{
+						Operations: []admissionregv1.OperationType{admissionregv1.Create},
+						Rule: admissionregv1.Rule{
+							APIGroups:   []string{"*"},
+							APIVersions: []string{"v1"},
+							Resources:   []string{"deployments", "statefulsets"},
+						},
+					},
+				},
+				SideEffects: func() *admissionregv1.SideEffectClass {
+					sideEffect := admissionregv1.SideEffectClassNoneOnDryRun
+					return &sideEffect
+				}(),
+				TimeoutSeconds:          &webhookTimeout,
+				AdmissionReviewVersions: []string{"v1"}}},
+	}
+	if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), &mwh, metav1.CreateOptions{}); err != nil {
+		// Webhook already exists, update the webhook in this scenario
+		if apierrors.IsAlreadyExists(err) {
+			existing, err := wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.Background(), mwh.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.Errorf("Failed to get MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err)
+				return err
+			}
+			mwh.ObjectMeta.ResourceVersion = existing.ObjectMeta.ResourceVersion
+			if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.Background(), &mwh, metav1.UpdateOptions{}); err != nil {
+				if !apierrors.IsConflict(err) {
+					klog.Errorf("Failed to update MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err)
+					return err
+				}
+			}
+		} else {
+			klog.Errorf("Failed to create MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err)
+			return err
+		}
+	}
+	klog.Infof("Finished creating MutatingWebhookConfiguration %s", appPodLabelWebhookName)
+	return nil
+}