From 23901ae6adae78aded09fec3142aa3e2d1bd5b41 Mon Sep 17 00:00:00 2001 From: Ye Cao Date: Tue, 26 Dec 2023 15:44:21 +0800 Subject: [PATCH] Fix the sidecar ci error. (#1694) Fixes the sidecar part of https://github.com/v6d-io/v6d/issues/1679 Signed-off-by: Ye Cao --- k8s/cmd/README.md | 2 +- .../deploy/deploy_vineyard_deployment_test.go | 2 +- k8s/cmd/commands/inject/inject.go | 2 +- k8s/cmd/commands/inject/inject_test.go | 2 +- k8s/config/manager/kustomization.yaml | 2 +- .../templates/sidecar/injection-template.yaml | 6 +- k8s/pkg/templates/vineyardd/etcd-service.yaml | 2 +- k8s/pkg/webhook/sidecar/sidecar_webhook.go | 60 +++++++++++++------ .../sidecar-with-custom-sidecar.yaml | 8 +-- .../sidecar-with-default-sidecar.yaml | 8 +-- k8s/test/e2e/sidecar/e2e.yaml | 12 ++-- 11 files changed, 65 insertions(+), 41 deletions(-) diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md index 8bb528add..c527783a5 100644 --- a/k8s/cmd/README.md +++ b/k8s/cmd/README.md @@ -1464,7 +1464,7 @@ metadata: ownerReferences: [] spec: ports: - - name: vineyard-sidecar-etcd-for-vineyard-port + - name: etcd-for-vineyard-port port: 2379 protocol: TCP targetPort: 2379 diff --git a/k8s/cmd/commands/deploy/deploy_vineyard_deployment_test.go b/k8s/cmd/commands/deploy/deploy_vineyard_deployment_test.go index 84a6ee941..571f27421 100644 --- a/k8s/cmd/commands/deploy/deploy_vineyard_deployment_test.go +++ b/k8s/cmd/commands/deploy/deploy_vineyard_deployment_test.go @@ -476,7 +476,7 @@ func TestGetVineyardDeploymentObjectsFromTemplate_third(t *testing.T) { "spec": map[string]interface{}{ "ports": []interface{}{ map[string]interface{}{ - "name": "test-vineyardd-sample-etcd-for-vineyard-port", + "name": "etcd-for-vineyard-port", "port": int64(2379), "protocol": "TCP", "targetPort": int64(2379), diff --git a/k8s/cmd/commands/inject/inject.go b/k8s/cmd/commands/inject/inject.go index 1682de7d6..14ba077c4 100644 --- a/k8s/cmd/commands/inject/inject.go +++ b/k8s/cmd/commands/inject/inject.go @@ -167,7 +167,7 @@ var ( ownerReferences: [] spec: ports: - - name: vineyard-sidecar-etcd-for-vineyard-port + - name: etcd-for-vineyard-port port: 2379 protocol: TCP targetPort: 2379 diff --git a/k8s/cmd/commands/inject/inject_test.go b/k8s/cmd/commands/inject/inject_test.go index 081c0887f..08762204f 100644 --- a/k8s/cmd/commands/inject/inject_test.go +++ b/k8s/cmd/commands/inject/inject_test.go @@ -101,7 +101,7 @@ metadata: ownerReferences: [] spec: ports: - - name: vineyard-sidecar-etcd-for-vineyard-port + - name: etcd-for-vineyard-port port: 2379 protocol: TCP targetPort: 2379 diff --git a/k8s/config/manager/kustomization.yaml b/k8s/config/manager/kustomization.yaml index b9edd779c..f7d249d07 100644 --- a/k8s/config/manager/kustomization.yaml +++ b/k8s/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: vineyardcloudnative/vineyard-operator + newName: localhost:5001/vineyard-operator newTag: latest diff --git a/k8s/pkg/templates/sidecar/injection-template.yaml b/k8s/pkg/templates/sidecar/injection-template.yaml index 28316fb1b..1a9ae3632 100644 --- a/k8s/pkg/templates/sidecar/injection-template.yaml +++ b/k8s/pkg/templates/sidecar/injection-template.yaml @@ -9,8 +9,8 @@ spec: image: {{ .Spec.Vineyard.Image }} imagePullPolicy: {{ .Spec.Vineyard.ImagePullPolicy }} {{- if .Spec.SecurityContext }} - securityContext: -{{ toYaml .Spec.SecurityContext | indent 12 }} + securityContext: +{{ toYaml .Spec.SecurityContext | indent 8 }} {{- end }} env: - name: VINEYARDD_UID @@ -99,7 +99,7 @@ spec: emptyDir: {} {{- end }} {{- if .Spec.Volumes }} -{{ toYaml .Spec.Volumes | indent 4 }} +{{ toYaml .Spec.Volumes | indent 2 }} {{- end }} {{- if .Spec.Metric.Enable }} - name: log diff --git a/k8s/pkg/templates/vineyardd/etcd-service.yaml b/k8s/pkg/templates/vineyardd/etcd-service.yaml index f855c3095..31b92fb7d 100644 --- a/k8s/pkg/templates/vineyardd/etcd-service.yaml +++ b/k8s/pkg/templates/vineyardd/etcd-service.yaml @@ -20,7 +20,7 @@ metadata: namespace: {{ .Namespace }} spec: ports: - - name: {{ .Name }}-etcd-for-vineyard-port + - name: etcd-for-vineyard-port port: 2379 protocol: TCP targetPort: 2379 diff --git a/k8s/pkg/webhook/sidecar/sidecar_webhook.go b/k8s/pkg/webhook/sidecar/sidecar_webhook.go index c3fc0a571..4e92ae2c0 100644 --- a/k8s/pkg/webhook/sidecar/sidecar_webhook.go +++ b/k8s/pkg/webhook/sidecar/sidecar_webhook.go @@ -37,6 +37,7 @@ import ( "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "sigs.k8s.io/yaml" "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" "github.com/v6d-io/v6d/k8s/pkg/config/annotations" @@ -59,6 +60,7 @@ type Injector struct { func (r *Injector) Handle(ctx context.Context, req admission.Request) admission.Response { logger := log.FromContext(ctx).WithName("Injector") + sidecar := &v1alpha1.Sidecar{} templatePod := &corev1.Pod{} pod := &corev1.Pod{} if err := r.decoder.Decode(req, pod); err != nil { @@ -67,8 +69,6 @@ func (r *Injector) Handle(ctx context.Context, req admission.Request) admission. anno := pod.Annotations if v, ok := anno[annotations.SidecarNameAnno]; ok && v == "default" { - // create the default sidecar cr - sidecar := &v1alpha1.Sidecar{} // get the pod's label as cr's name l := pod.Labels keys := []string{} @@ -131,29 +131,53 @@ func (r *Injector) Handle(ctx context.Context, req admission.Request) admission. return admission.Errored(http.StatusInternalServerError, err) } } - - buf, err := templates.ReadFile("sidecar/injection-template.yaml") - if err != nil { - logger.Error(err, "failed to read injection template") + } else { + // get the sidecar cr + if err := r.Get( + ctx, + types.NamespacedName{Name: v, Namespace: pod.Namespace}, + sidecar, + ); err != nil { + logger.Error(err, "get custom sidecar cr failed") return admission.Errored(http.StatusInternalServerError, err) } + } + buf, err := templates.ReadFile("sidecar/injection-template.yaml") + if err != nil { + logger.Error(err, "failed to read injection template") + return admission.Errored(http.StatusInternalServerError, err) + } - if tpl, err := template.New("sidecar").Parse(string(buf)); err == nil { - var buf bytes.Buffer - if err := tpl.Execute(&buf, sidecar); err == nil { - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, _ := decode(buf.Bytes(), nil, nil) - templatePod = obj.(*corev1.Pod) - } else { - logger.Error(err, "failed to execute template") - return admission.Errored(http.StatusInternalServerError, err) + tmplFunc := map[string]interface{}{ + "toYaml": func(v interface{}) string { + bs, err := yaml.Marshal(v) + if err != nil { + logger.Error(err, "failed to marshal object %v to yaml", v) + return "" } - } - if err := r.ApplyToSidecar(sidecar, templatePod, pod, true); err != nil { - logger.Error(err, "failed to apply sidecar cr to pod") + return string(bs) + }, + "indent": func(spaces int, s string) string { + prefix := strings.Repeat(" ", spaces) + return prefix + strings.Replace(s, "\n", "\n"+prefix, -1) + }, + } + + if tpl, err := template.New("sidecar").Funcs(tmplFunc).Parse(string(buf)); err == nil { + var buf bytes.Buffer + if err := tpl.Execute(&buf, sidecar); err == nil { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, _ := decode(buf.Bytes(), nil, nil) + templatePod = obj.(*corev1.Pod) + } else { + logger.Error(err, "failed to execute template") return admission.Errored(http.StatusInternalServerError, err) } } + if err := r.ApplyToSidecar(sidecar, templatePod, pod, true); err != nil { + logger.Error(err, "failed to apply sidecar cr to pod") + return admission.Errored(http.StatusInternalServerError, err) + } marshaledPod, err := json.Marshal(pod) if err != nil { diff --git a/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml b/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml index e8bfb7def..66a494751 100644 --- a/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml +++ b/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml @@ -20,7 +20,7 @@ metadata: namespace: vineyard-job spec: replicas: 2 - selector: app=job-deployment + selector: app=job-with-custom-sidecar vineyard: image: localhost:5001/vineyardd:latest socket: /var/run/vineyard.sock @@ -28,19 +28,19 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: job-deployment + name: job-with-custom-sidecar namespace: vineyard-job spec: selector: matchLabels: - app: job-deployment + app: job-with-custom-sidecar replicas: 2 template: metadata: annotations: sidecar.v6d.io/name: "sidecar-sample" labels: - app: job-deployment + app: job-with-custom-sidecar sidecar.v6d.io/enabled: "true" spec: containers: diff --git a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml index 2e9b2f8dd..b609b234c 100644 --- a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml +++ b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml @@ -16,20 +16,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: job-deployment + name: job-with-default-sidecar namespace: vineyard-job spec: selector: matchLabels: - app: job-deployment + app: job-with-default-sidecar replicas: 2 template: metadata: annotations: sidecar.v6d.io/name: "default" labels: - app.kubernetes.io/instance: job-deployment - app: job-deployment + app.kubernetes.io/instance: job-with-default-sidecar + app: job-with-default-sidecar sidecar.v6d.io/enabled: "true" spec: containers: diff --git a/k8s/test/e2e/sidecar/e2e.yaml b/k8s/test/e2e/sidecar/e2e.yaml index d001602cf..c80fff892 100644 --- a/k8s/test/e2e/sidecar/e2e.yaml +++ b/k8s/test/e2e/sidecar/e2e.yaml @@ -28,7 +28,7 @@ setup: --sidecar.image="localhost:5001/vineyardd:latest"| kubectl apply -f - wait: - namespace: vineyard-job - resource: deployment/job-deployment + resource: deployment/job-with-default-sidecar for: condition=Available - name: install app with default sidecar via kubectl command: | @@ -40,7 +40,7 @@ setup: kubectl apply -f - wait: - namespace: vineyard-job1 - resource: deployment/job-deployment + resource: deployment/job-with-default-sidecar for: condition=Available - name: install app with custom sidecar command: | @@ -48,7 +48,7 @@ setup: kubectl apply -f k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml wait: - namespace: vineyard-job - resource: deployment/job-deployment + resource: deployment/job-with-custom-sidecar for: condition=Available timeout: 20m @@ -65,7 +65,7 @@ verify: interval: 10s cases: - query: | - kubectl get pod -l app=job-deployment-with-default-sidecar -n vineyard-job -oname | \ + kubectl get pod -l app=job-with-default-sidecar -n vineyard-job -oname | \ awk -F '/' '{print $2}' | \ head -n 1 | \ xargs kubectl logs -c job -n vineyard-job | \ @@ -73,7 +73,7 @@ verify: yq e 'to_entries' - expected: ../verify/values.yaml - query: | - kubectl get pod -l app=job-deployment-with-default-sidecar -n vineyard-job1 -oname | \ + kubectl get pod -l app=job-with-default-sidecar -n vineyard-job1 -oname | \ awk -F '/' '{print $2}' | \ head -n 1 | \ xargs kubectl logs -c job -n vineyard-job1 | \ @@ -81,7 +81,7 @@ verify: yq e 'to_entries' - expected: ../verify/values.yaml - query: | - kubectl get pod -l app=job-deployment-with-custom-sidecar -n vineyard-job -oname | \ + kubectl get pod -l app=job-with-custom-sidecar -n vineyard-job -oname | \ awk -F '/' '{print $2}' | \ head -n 1 | \ xargs kubectl logs -c job -n vineyard-job | \