diff --git a/pkg/apiserver/filters.go b/pkg/apiserver/filters.go index 9870ba3..66d0b4a 100644 --- a/pkg/apiserver/filters.go +++ b/pkg/apiserver/filters.go @@ -92,6 +92,7 @@ func (h *Handler) authenticate(req *restful.Request, resp *restful.Response, cha "/app-service/v1/provider-registry/validate", "/app-service/v1/pods/kubelet/eviction", "/app-service/v1/workflow/inject", + "/app-service/v1/runasuser/inject", } needAuth := true diff --git a/pkg/apiserver/handler.go b/pkg/apiserver/handler.go index 08ccfa3..8bbf573 100644 --- a/pkg/apiserver/handler.go +++ b/pkg/apiserver/handler.go @@ -79,6 +79,10 @@ func (b *handlerBuilder) Build() (*Handler, error) { if err != nil { return nil, err } + err = wh.CreateOrUpdateRunAsUserMutatingWebhook() + if err != nil { + return nil, err + } return &Handler{ kubeHost: b.ksHost, diff --git a/pkg/apiserver/handler_webhook.go b/pkg/apiserver/handler_webhook.go index cc11897..6f246cb 100644 --- a/pkg/apiserver/handler_webhook.go +++ b/pkg/apiserver/handler_webhook.go @@ -40,8 +40,9 @@ func init() { } const ( - deployment = "Deployment" - statefulSet = "StatefulSet" + deployment = "Deployment" + statefulSet = "StatefulSet" + applicationNameKey = "applications.app.bytetrade.io/name" ) func (h *Handler) sandboxInject(req *restful.Request, resp *restful.Response) { @@ -639,3 +640,130 @@ func (h *Handler) cronWorkflowMutate(ctx context.Context, req *admissionv1.Admis h.sidecarWebhook.PatchAdmissionResponse(resp, patchBytes) return resp } + +func (h *Handler) handleRunAsUser(req *restful.Request, resp *restful.Response) { + klog.Infof("Received run as user mutate webhook request: Method=%v, URL=%v", req.Request.Method, req.Request.URL) + admissionRequestBody, ok := h.sidecarWebhook.GetAdmissionRequestBody(req, resp) + if !ok { + klog.Errorf("Failed to get admission request body") + return + } + var admissionReq, admissionResp admissionv1.AdmissionReview + proxyUUID := uuid.New() + if _, _, err := webhook.Deserializer.Decode(admissionRequestBody, nil, &admissionReq); err != nil { + klog.Errorf("Failed to decoding admission request body err=%v", err) + admissionResp.Response = h.sidecarWebhook.AdmissionError("", err) + } else { + admissionResp.Response = h.handleRunAsUserMutate(req.Request.Context(), admissionReq.Request, proxyUUID) + } + admissionResp.TypeMeta = admissionReq.TypeMeta + admissionResp.Kind = admissionReq.Kind + + requestForNamespace := "unknown" + if admissionReq.Request != nil { + requestForNamespace = admissionReq.Request.Namespace + } + + err := resp.WriteAsJson(&admissionResp) + if err != nil { + klog.Infof("handleRunAsUserMutate: write response failed namespace=%s, err=%v", requestForNamespace, err) + return + } + klog.Infof("Done handleRunAsUserMutate admission request with uuid=%s, namespace=%s", proxyUUID, requestForNamespace) +} + +func (h *Handler) handleRunAsUserMutate(ctx context.Context, req *admissionv1.AdmissionRequest, proxyUUID uuid.UUID) *admissionv1.AdmissionResponse { + if req == nil { + klog.Error("Failed to get admission request err=admission request is nil") + return h.sidecarWebhook.AdmissionError("", errNilAdmissionRequest) + } + resp := &admissionv1.AdmissionResponse{ + Allowed: true, + UID: req.UID, + } + var pod corev1.Pod + err := json.Unmarshal(req.Object.Raw, &pod) + if err != nil { + klog.Errorf("Failed to unmarshal request object raw with uuid=%s namespace=%s", proxyUUID, req.Namespace) + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + curPod, err := h.runAsUserInject(ctx, &pod, req.Namespace) + if err != nil { + klog.Infof("run runAsUserInject err=%v", err) + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + current, err := json.Marshal(curPod) + if err != nil { + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + admissionResp := admission.PatchResponseFromRaw(req.Object.Raw, current) + patchBytes, err := json.Marshal(admissionResp.Patches) + if err != nil { + return h.sidecarWebhook.AdmissionError(req.UID, err) + } + h.sidecarWebhook.PatchAdmissionResponse(resp, patchBytes) + return resp +} + +func (h *Handler) runAsUserInject(ctx context.Context, pod *corev1.Pod, namespace string) (*corev1.Pod, error) { + if len(pod.OwnerReferences) == 0 || pod == nil { + return pod, nil + } + var err error + var kind, name string + ownerRef := pod.OwnerReferences[0] + switch ownerRef.Kind { + case "ReplicaSet": + key := types.NamespacedName{Namespace: namespace, Name: ownerRef.Name} + var rs appsv1.ReplicaSet + err = h.ctrlClient.Get(ctx, key, &rs) + if err != nil { + klog.Infof("get replicaset err=%v", err) + return nil, err + } + if len(rs.OwnerReferences) > 0 && rs.OwnerReferences[0].Kind == deployment { + kind = deployment + name = rs.OwnerReferences[0].Name + } + case statefulSet: + kind = statefulSet + name = ownerRef.Name + } + if kind == "" { + return pod, nil + } + labels := make(map[string]string) + switch kind { + case deployment: + var deploy appsv1.Deployment + key := types.NamespacedName{Name: name, Namespace: namespace} + err = h.ctrlClient.Get(ctx, key, &deploy) + if err != nil { + return nil, err + } + labels = deploy.Labels + + case statefulSet: + var sts appsv1.StatefulSet + key := types.NamespacedName{Name: name, Namespace: namespace} + err = h.ctrlClient.Get(ctx, key, &sts) + if err != nil { + return nil, err + } + labels = sts.Labels + } + userID := int64(1000) + if appName, ok := labels[applicationNameKey]; ok && !userspace.IsSysApp(appName) { + if pod.Spec.SecurityContext == nil { + pod.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsUser: &userID, + } + } else { + if pod.Spec.SecurityContext.RunAsUser == nil || *pod.Spec.SecurityContext.RunAsUser != 1000 { + pod.Spec.SecurityContext.RunAsUser = &userID + } + } + return pod, nil + } + return pod, nil +} diff --git a/pkg/apiserver/webservice.go b/pkg/apiserver/webservice.go index 3416c86..0fc4597 100644 --- a/pkg/apiserver/webservice.go +++ b/pkg/apiserver/webservice.go @@ -321,6 +321,13 @@ func addServiceToContainer(c *restful.Container, handler *Handler) error { Returns(http.StatusOK, "pod eviction validated success", nil)). Consumes(restful.MIME_JSON) + ws.Route(ws.POST("/runasuser/inject"). + To(handler.handleRunAsUser). + Doc("mutating webhook for inject runasuser 1000 for third party app's pod"). + Metadata(restfulspec.KeyOpenAPITags, MODULE_TAGS). + Returns(http.StatusOK, "inject runasuser success", nil)). + Consumes(restful.MIME_JSON) + ws.Route(ws.POST("/workflow/inject"). To(handler.cronWorkflowInject). Doc("mutating webhook for cron workflow"). diff --git a/pkg/webhook/setup.go b/pkg/webhook/setup.go index 8f0716e..f72a8bf 100644 --- a/pkg/webhook/setup.go +++ b/pkg/webhook/setup.go @@ -23,6 +23,8 @@ const ( providerRegistryValidatingWebhookName = "provider-registry-validating-webhook.bytetrade.io" validatingWebhookName = "appns-validating-webhook.bytetrade.io" gpuLimitWebhookName = "gpu-limit-webhook" + runAsUserWebhookName = "run-as-user-webhook" + mutatingWebhookRunAsUserName = "run-as-user.bytetrade.io" mutatingWebhookGpuLimitName = "gpu-limit-inject-webhook.bytetrade.io" webhookServiceName = "app-service" webhookServiceNamespace = "os-system" @@ -563,3 +565,99 @@ func (wh *Webhook) CreateOrUpdateCronWorkflowMutatingWebhook() error { klog.Info("Finished creating MutatingWebhookConfiguration name=%s", mwc.Name) return nil } + +// CreateOrUpdateRunAsUserMutatingWebhook creates or updates gpu limit mutating webhook. +func (wh *Webhook) CreateOrUpdateRunAsUserMutatingWebhook() error { + webhookPath := "/app-service/v1/runasuser/inject" + port, err := strconv.Atoi(strings.Split(constants.WebhookServerListenAddress, ":")[1]) + if err != nil { + return err + } + webhookPort := int32(port) + failurePolicy := admissionregv1.Ignore + matchPolicy := admissionregv1.Exact + webhookTimeout := int32(30) + + mwhLabels := map[string]string{"velero.io/exclude-from-backup": "true"} + caBundle, err := ioutil.ReadFile(defaultCaPath) + if err != nil { + return err + } + mwh := admissionregv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: runAsUserWebhookName, + Labels: mwhLabels, + }, + Webhooks: []admissionregv1.MutatingWebhook{ + { + Name: mutatingWebhookRunAsUserName, + ClientConfig: admissionregv1.WebhookClientConfig{ + CABundle: caBundle, + Service: &admissionregv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &webhookPath, + Port: &webhookPort, + }, + }, + FailurePolicy: &failurePolicy, + MatchPolicy: &matchPolicy, + NamespaceSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: security.UnderLayerNamespaces, + }, + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: security.OSSystemNamespaces, + }, + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: security.GPUSystemNamespaces, + }, + }, + }, + Rules: []admissionregv1.RuleWithOperations{ + { + Operations: []admissionregv1.OperationType{admissionregv1.Create}, + Rule: admissionregv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + SideEffects: func() *admissionregv1.SideEffectClass { + sideEffect := admissionregv1.SideEffectClassNoneOnDryRun + return &sideEffect + }(), + TimeoutSeconds: &webhookTimeout, + AdmissionReviewVersions: []string{"v1"}}}, + } + if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), &mwh, metav1.CreateOptions{}); err != nil { + // Webhook already exists, update the webhook in this scenario + if apierrors.IsAlreadyExists(err) { + existing, err := wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.Background(), mwh.Name, metav1.GetOptions{}) + if err != nil { + klog.Errorf("Failed to get MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + mwh.ObjectMeta.ResourceVersion = existing.ObjectMeta.ResourceVersion + if _, err = wh.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.Background(), &mwh, metav1.UpdateOptions{}); err != nil { + if !apierrors.IsConflict(err) { + klog.Errorf("Failed to update MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + } + } else { + klog.Errorf("Failed to create MutatingWebhookConfiguration name=%s err=%v", mwh.Name, err) + return err + } + } + klog.Infof("Finished creating MutatingWebhookConfiguration %s", runAsUserWebhookName) + return nil +}