fixup! fixup! fixup! add spark-operator-v2 (in-progress)

spotinst · Nov 21, 2024 · 93af4f7 · 93af4f7
1 parent 55660d5
commit 93af4f7
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 106 deletions.
diff --git a/charts/spark-operator-static/Chart.yaml b/charts/spark-operator-static/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Spark Operator (static part)
 name: spark-operator-static
-version: 0.1.6
-appVersion: 0.1.6
+version: 0.1.7
+appVersion: 0.1.7
 maintainers:
   - name: thorsteinnth
     email: [email protected]
diff --git a/charts/spark-operator-v2/charts/spark-operator/templates/controller/deployment.yaml b/charts/spark-operator-v2/charts/spark-operator/templates/controller/deployment.yaml
@@ -147,6 +147,7 @@ spec:
       {{- with .Values.controller.sidecars }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
+      {{ include "telemetry.container" . | indent 6 }}
       {{- with .Values.image.pullSecrets }}
       imagePullSecrets:
       {{- toYaml . | nindent 6 }}

diff --git a/charts/spark-operator-v2/charts/spark-operator/templates/webhook/deployment.yaml b/charts/spark-operator-v2/charts/spark-operator/templates/webhook/deployment.yaml
@@ -117,6 +117,7 @@ spec:
       {{- with .Values.webhook.sidecars }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
+      {{ include "telemetry.container" . | indent 6 }}
       {{- with .Values.image.pullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}

diff --git a/charts/spark-operator-v2/templates/_helpers.tpl b/charts/spark-operator-v2/templates/_helpers.tpl
@@ -0,0 +1,55 @@
+{{- define "telemetry.container" -}}
+{{- if .Values.telemetry.enabled }}
+- name: fluentbit
+  image: public.ecr.aws/ocean-spark/fluent-bit:3.0.5
+  securityContext:
+    runAsNonRoot: true
+  ports:
+    - name: http
+      containerPort: 2020
+      protocol: TCP
+  env:
+    - name: AWS_BUCKET_NAME
+      valueFrom:
+        secretKeyRef:
+          name: spot-bigdata-telemetry-creds
+          key: AWS_BUCKET_NAME
+    - name: AWS_REGION
+      valueFrom:
+        secretKeyRef:
+          name: spot-bigdata-telemetry-creds
+          key: AWS_REGION
+    - name: CLUSTER_ID
+      valueFrom:
+        configMapKeyRef:
+          name: spot-ofas-cluster-info
+          key: clusterId
+    - name: ACCOUNT_ID
+      valueFrom:
+        configMapKeyRef:
+          name: spot-ofas-cluster-info
+          key: accountId
+  resources: {}
+  volumeMounts:
+    - name: telementry-global-config
+      mountPath: /opt/bitnami/fluent-bit/conf/fluent-bit.conf
+      subPath: fluent-bit.conf
+    - name: telementry-custom-config
+      mountPath: /opt/bitnami/fluent-bit/conf/custom-filters.conf
+      subPath: custom-filters.conf
+    - name: telementry-global-config
+      mountPath: /opt/bitnami/fluent-bit/conf/parsers.conf
+      subPath: parsers.conf
+    - name: telementry-custom-config
+      mountPath: /opt/bitnami/fluent-bit/conf/metrics-collection.conf
+      subPath: metrics-collection.conf
+    - name: varlog
+      readOnly: true
+      mountPath: /var/log/
+    - name: varlibdockercontainers
+      readOnly: true
+      mountPath: /var/lib/docker/containers
+    - name: telemetry-aws-credentials
+      mountPath: /.aws
+{{- end }}
+{{- end -}}
diff --git a/charts/spark-operator-v2/values.yaml b/charts/spark-operator-v2/values.yaml
@@ -48,58 +48,6 @@ spark-operator:
         operator: "Equal"
         value: "spot"
         effect: "NoSchedule"
-    sidecars:
-      - name: fluentbit
-        image: public.ecr.aws/ocean-spark/fluent-bit:3.0.5
-        securityContext:
-          runAsNonRoot: true
-        ports:
-          - name: http
-            containerPort: 2020
-            protocol: TCP
-        env:
-          - name: AWS_BUCKET_NAME
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_BUCKET_NAME
-          - name: AWS_REGION
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_REGION
-          - name: CLUSTER_ID
-            valueFrom:
-              configMapKeyRef:
-                name: spot-ofas-cluster-info
-                key: clusterId
-          - name: ACCOUNT_ID
-            valueFrom:
-              configMapKeyRef:
-                name: spot-ofas-cluster-info
-                key: accountId
-        resources: {}
-        volumeMounts:
-          - name: telementry-global-config
-            mountPath: /opt/bitnami/fluent-bit/conf/fluent-bit.conf
-            subPath: fluent-bit.conf
-          - name: telementry-custom-config
-            mountPath: /opt/bitnami/fluent-bit/conf/custom-filters.conf
-            subPath: custom-filters.conf
-          - name: telementry-global-config
-            mountPath: /opt/bitnami/fluent-bit/conf/parsers.conf
-            subPath: parsers.conf
-          - name: telementry-custom-config
-            mountPath: /opt/bitnami/fluent-bit/conf/metrics-collection.conf
-            subPath: metrics-collection.conf
-          - name: varlog
-            readOnly: true
-            mountPath: /var/log/
-          - name: varlibdockercontainers
-            readOnly: true
-            mountPath: /var/lib/docker/containers
-          - name: telemetry-aws-credentials
-            mountPath: /.aws
     volumes:
       - name: telementry-global-config
         configMap:
@@ -149,58 +97,6 @@ spark-operator:
         operator: "Equal"
         value: "spot"
         effect: "NoSchedule"
-    sidecars:
-      - name: fluentbit
-        image: public.ecr.aws/ocean-spark/fluent-bit:3.0.5
-        securityContext:
-          runAsNonRoot: true
-        ports:
-          - name: http
-            containerPort: 2020
-            protocol: TCP
-        env:
-          - name: AWS_BUCKET_NAME
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_BUCKET_NAME
-          - name: AWS_REGION
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_REGION
-          - name: CLUSTER_ID
-            valueFrom:
-              configMapKeyRef:
-                name: spot-ofas-cluster-info
-                key: clusterId
-          - name: ACCOUNT_ID
-            valueFrom:
-              configMapKeyRef:
-                name: spot-ofas-cluster-info
-                key: accountId
-        resources: {}
-        volumeMounts:
-          - name: telementry-global-config
-            mountPath: /opt/bitnami/fluent-bit/conf/fluent-bit.conf
-            subPath: fluent-bit.conf
-          - name: telementry-custom-config
-            mountPath: /opt/bitnami/fluent-bit/conf/custom-filters.conf
-            subPath: custom-filters.conf
-          - name: telementry-global-config
-            mountPath: /opt/bitnami/fluent-bit/conf/parsers.conf
-            subPath: parsers.conf
-          - name: telementry-custom-config
-            mountPath: /opt/bitnami/fluent-bit/conf/metrics-collection.conf
-            subPath: metrics-collection.conf
-          - name: varlog
-            readOnly: true
-            mountPath: /var/log/
-          - name: varlibdockercontainers
-            readOnly: true
-            mountPath: /var/lib/docker/containers
-          - name: telemetry-aws-credentials
-            mountPath: /.aws
     volumes:
       - name: telementry-global-config
         configMap:
@@ -217,3 +113,6 @@ spark-operator:
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
+
+  telemetry:
+    enabled: true