feat: extra OPTEL collector configuration. (#581)

Signed-off-by: José Guilherme Vanz <[email protected]>
kubewarden · Dec 13, 2024 · 19ebe2d · 19ebe2d
1 parent fe27b38
commit 19ebe2d
Show file tree

Hide file tree

Showing 7 changed files with 425 additions and 60 deletions.
diff --git a/charts/kubewarden-controller/questions.yaml b/charts/kubewarden-controller/questions.yaml
@@ -115,47 +115,92 @@ questions:
       Number of replicas of the Controller Deployment
     group: "Controller HA"
   # Telemetry:
-  - variable: "telemetry.metrics.enabled"
+  - variable: "telemetry.mode"
+    type: enum
+    options:
+      - "sidecar"
+      - "custom"
+    default: "sidecar"
+    required: true
+    label: Telemetry mode
+    description: |
+      Choose the telemetry mode. Sidecar mode will deploy an OpenTelemetry Collector
+      as a sidecar container in the Kubewarden Controller pod. Custom mode will allow
+      you to configure the OpenTelemetry Collector.
+    group: "Telemetry"
+  - variable: "telemetry.metrics"
     type: boolean
     default: false
     required: true
     label: Enable Metrics
     description: |
       Enable metrics collection for all Policy Servers and the Kubewarden Controller.
-      Important: Requires OpenTelemetry CRDs available
+      Important: Requires OpenTelemetry CRDs available.
     group: "Telemetry"
-    subquestions:
-      - variable: "telemetry.metrics.port"
-        type: string
-        default: "8080"
-        label: Port
-        description: |
-          Port of the Prometheus exporter and PolicyServer metrics service
-        group: "Telemetry"
-        show_if: "telemetry.metrics.enabled=true"
-  - variable: "telemetry.tracing.enabled"
+  - variable: "telemetry.tracing"
     type: boolean
     default: false
     required: true
     label: Enable Tracing
     description: |
       Enable tracing collection for all PolicyServers.
-      Important: Requires OpenTelemetry CRDs available
+      Important: Requires OpenTelemetry CRDs available.
+    group: "Telemetry"
+  - variable: "telemetry.sidecar.metrics.port"
+    type: string
+    default: "8080"
+    label: Port
+    description: |
+      Port of the Prometheus exporter and PolicyServer metrics service.
+    group: "Telemetry"
+    show_if: "telemetry.mode=sidecar"
+  - variable: "telemetry.sidecar.tracing.jaeger.endpoint"
+    type: string
+    default: my-open-telemetry-collector.jaeger.svc.cluster.local:4317
+    label: Jaeger endpoint configuration
+    description: |
+      Configuration of the OTLP/Jaeger exporter.
+    group: "Telemetry"
+    show_if: "telemetry.mode=sidecar"
+  - variable: "telemetry.sidecar.tracing.jaeger.tls.insecure"
+    type: boolean
+    default: false
+    label: Jaeger endpoint insecure TLS configuration
+    description: |
+      Important: Insecure, not for production usage.
+    group: "Telemetry"
+    show_if: "telemetry.mode=sidecar"
+  - variable: "telemetry.custom.endpoint"
+    type: string
+    default: ""
+    label: OpenTelemetry endpoint
+    description: |
+      Endpoint of the OpenTelemetry collector.
+    group: "Telemetry"
+    show_if: "telemetry.mode=custom"
+  - variable: "telemetry.custom.insecure"
+    type: boolean
+    default: false
+    label: Insecure communication
+    description: |
+      Disable TLS verification for the OpenTelemetry collector.
+    group: "Telemetry"
+    show_if: "telemetry.mode=custom"
+  - variable: "telemetry.custom.otelCollectorCertificateSecret"
+    type: string
+    default: ""
+    label: Certificate secret
+    description: |
+      Secret containing the certificate for the OpenTelemetry collector.
+      The secret should contain the key `ca.crt`.
+    group: "Telemetry"
+    show_if: "telemetry.mode=custom && telemetry.custom.insecure=false"
+  - variable: "telemetry.custom.otelCollectorClientCertificateSecret"
+    type: string
+    default: ""
+    label: Client certificate secret
+    description: |
+      Secret containing the client certificate for the OpenTelemetry collector (mTLS).
+      The secret should contain the keys `tls.crt` and `tls.key`.
     group: "Telemetry"
-    subquestions:
-      - variable: "telemetry.tracing.jaeger.endpoint"
-        type: string
-        default: my-open-telemetry-collector.jaeger.svc.cluster.local:4317
-        label: Jaeger endpoint configuration
-        description: |
-          Configuration of the OTLP/Jaeger exporter
-        group: "Telemetry"
-        show_if: "telemetry.tracing.enabled=true"
-      - variable: "telemetry.tracing.jaeger.tls.insecure"
-        type: boolean
-        default: false
-        label: Jaeger endpoint insecure TLS configuration
-        description: |
-          Important: Insecure, not for production usage
-        group: "Telemetry"
-        show_if: "telemetry.tracing.enabled=true"
+    show_if: "telemetry.mode=custom && telemetry.custom.insecure=false"
diff --git a/charts/kubewarden-controller/templates/deployment.yaml b/charts/kubewarden-controller/templates/deployment.yaml
@@ -18,9 +18,11 @@ spec:
         {{- range keys .Values.podAnnotations }}
         {{ . | quote }}: {{ get $.Values.podAnnotations . | quote}}
         {{- end }}
-        {{- if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled}}
+        {{- if or .Values.telemetry.metrics .Values.telemetry.tracing }}
+        {{- if eq .Values.telemetry.mode "sidecar" }}
         "sidecar.opentelemetry.io/inject": "true"
         {{- end }}
+        {{- end }}
         {{- include "kubewarden-controller.annotations" . | nindent 8 }}
       labels:
         {{- include "kubewarden-controller.labels" . | nindent 8 }}
@@ -46,21 +48,55 @@ spec:
         - --leader-elect
         - --deployments-namespace={{ .Release.Namespace }}
         - --webhook-service-name={{ include "kubewarden-controller.fullname" . }}-webhook-service
-       {{- if .Values.telemetry.metrics.enabled }}
+        - --always-accept-admission-reviews-on-deployments-namespace
+        - --zap-log-level={{ .Values.logLevel }}
+       {{- if or .Values.telemetry.metrics .Values.telemetry.tracing }}
+       {{- if eq .Values.telemetry.mode "sidecar" }}
+        - --enable-otel-sidecar
+       {{- end }}
+       {{- if .Values.telemetry.metrics }}
         - --enable-metrics
        {{- end }}
-       {{- if .Values.telemetry.tracing.enabled }}
+       {{- if .Values.telemetry.tracing }}
         - --enable-tracing
        {{- end }}
-        - --always-accept-admission-reviews-on-deployments-namespace
-        - --zap-log-level={{ .Values.logLevel }}
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorCertificateSecret }}
+        - --opentelemetry-certificate-secret={{ .Values.telemetry.custom.otelCollectorCertificateSecret }}
+        {{- end }}
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+        - --opentelemetry-client-certificate-secret={{ .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+        {{- end }}
+       {{- end }}
         command:
         - /manager
-        {{- if .Values.telemetry.metrics.enabled }}
         env:
+        {{- if and .Values.telemetry.metrics (eq .Values.telemetry.mode "sidecar") }}
           - name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT
-            value: "{{ .Values.telemetry.metrics.port | default 8080 }}"
+            value: "{{ .Values.telemetry.sidecar.metrics.port | default 8080 }}"
+          - name: OTEL_EXPORTER_OTLP_ENDPOINT
+            value: "https://localhost:4317"
+          - name: OTEL_EXPORTER_OTLP_INSECURE
+            value: "true"
+        {{- end }}
+        {{- if or .Values.telemetry.metrics .Values.telemetry.tracing }}
+        {{- if eq .Values.telemetry.mode "custom" }}
+          - name: OTEL_EXPORTER_OTLP_ENDPOINT
+            value: {{.Values.telemetry.custom.endpoint}}
+          - name: OTEL_EXPORTER_OTLP_INSECURE
+            value: {{ .Values.telemetry.custom.insecure | default false | quote }}
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorCertificateSecret }}
+          - name: OTEL_EXPORTER_OTLP_CERTIFICATE
+            value: /kubewarden/otel-collector-certs/ca.crt 
+        {{- end }}
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+          - name: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE
+            value: /kubewarden/otel-collector-client-certs/tls.crt
+          - name: OTEL_EXPORTER_OTLP_CLIENT_KEY
+            value: /kubewarden/otel-collector-client-certs/tls.key
+        {{- end }}
         {{- end }}
+        {{- end }}
+
         image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}'
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         livenessProbe:
@@ -87,6 +123,16 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorCertificateSecret }}
+        - mountPath: /kubewarden/otel-collector-certs
+          name: otel-collector-certificate
+          readOnly: true
+        {{- end }}
+        {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+        - mountPath: /kubewarden/otel-collector-client-certs
+          name: otel-collector-client-certificate
+          readOnly: true
+        {{- end }}
         ports:
         - containerPort: 9443
           name: webhook-server
@@ -96,6 +142,26 @@ spec:
         secret:
           defaultMode: 420
           secretName: kubewarden-webhook-server-cert
+      {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorCertificateSecret }}
+      - name: otel-collector-certificate
+        secret:
+          defaultMode: 420
+          secretName: {{ .Values.telemetry.custom.otelCollectorCertificateSecret }}
+          items:
+            - key: ca.crt
+              path: ca.crt
+      {{- end }}
+      {{- if and (not .Values.telemetry.custom.insecure) .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+      - name: otel-collector-client-certificate
+        secret:
+          defaultMode: 420
+          secretName: {{ .Values.telemetry.custom.otelCollectorClientCertificateSecret }}
+          items:
+            - key: tls.crt
+              path: tls.crt
+            - key: tls.key
+              path: tls.key
+      {{- end }}
       {{- if .Values.podSecurityContext }}
       securityContext:
 {{ toYaml .Values.podSecurityContext | indent 8 }}

diff --git a/charts/kubewarden-controller/templates/opentelemetry-collector.yaml b/charts/kubewarden-controller/templates/opentelemetry-collector.yaml
@@ -1,4 +1,5 @@
-{{ if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled }}
+{{ if or .Values.telemetry.metrics .Values.telemetry.tracing }}
+{{ if eq .Values.telemetry.mode "sidecar" }}
 apiVersion: opentelemetry.io/v1beta1
 kind: OpenTelemetryCollector
 metadata:
@@ -18,32 +19,33 @@ spec:
     processors:
       batch: {}
     exporters:
-      {{- if and .Values.telemetry.tracing.enabled .Values.telemetry.tracing.jaeger.endpoint }}
+      {{- if and .Values.telemetry.tracing .Values.telemetry.sidecar.tracing.jaeger.endpoint }}
       otlp/jaeger:
-        endpoint: {{ .Values.telemetry.tracing.jaeger.endpoint }}
-        {{- if hasKey .Values.telemetry.tracing.jaeger "tls" }}
-        {{- if .Values.telemetry.tracing.jaeger.tls.insecure }}
+        endpoint: {{ .Values.telemetry.sidecar.tracing.jaeger.endpoint }}
+        {{- if hasKey .Values.telemetry.sidecar.tracing.jaeger "tls" }}
+        {{- if .Values.telemetry.sidecar.tracing.jaeger.tls.insecure }}
         tls:
-          insecure: {{ .Values.telemetry.tracing.jaeger.tls.insecure }}
+          insecure: {{ .Values.telemetry.sidecar.tracing.jaeger.tls.insecure }}
         {{- end }}
         {{- end }}
       {{- end }}
-      {{- if and .Values.telemetry.metrics.enabled .Values.telemetry.metrics.port }}
+      {{- if and .Values.telemetry.metrics .Values.telemetry.sidecar.metrics.port }}
       prometheus:
-        endpoint: ":{{ .Values.telemetry.metrics.port }}"
+        endpoint: ":{{ .Values.telemetry.sidecar.metrics.port }}"
       {{- end }}
     service:
       pipelines:
-        {{- if and .Values.telemetry.metrics.enabled .Values.telemetry.metrics.port }}
+        {{- if and .Values.telemetry.metrics .Values.telemetry.sidecar.metrics.port }}
         metrics:
           receivers: [otlp]
           processors: []
           exporters: [prometheus]
         {{- end }}
-        {{- if and .Values.telemetry.tracing.enabled .Values.telemetry.tracing.jaeger.endpoint }}
+        {{- if and .Values.telemetry.tracing .Values.telemetry.sidecar.tracing.jaeger.endpoint }}
         traces:
           receivers: [otlp]
           processors: [batch]
           exporters: [otlp/jaeger]
         {{- end }}
 {{ end }}
+{{ end }}
diff --git a/charts/kubewarden-controller/templates/post-install-hook.yaml b/charts/kubewarden-controller/templates/post-install-hook.yaml
@@ -3,7 +3,7 @@
 # pod, it cannot find a valid collector configuration. Therefore, it is necessary
 # to recreate the controller pod after the installation. This ensures that the
 # controller pod will have the OTEL collector container.
-{{ if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled }}
+{{ if or .Values.telemetry.metrics .Values.telemetry.tracing }}
 apiVersion: batch/v1
 kind: Job
 metadata:

diff --git a/charts/kubewarden-controller/templates/service.yaml b/charts/kubewarden-controller/templates/service.yaml
@@ -10,7 +10,7 @@ metadata:
     {{- include "kubewarden-controller.annotations" . | nindent 4 }}
 spec:
   ports:
-  {{- if .Values.telemetry.metrics.enabled }}
+  {{- if .Values.telemetry.metrics }}
   - name: metrics
     port: 8080
     targetPort: 8080