From cfb4b08034086705105572a6c4e773b727503a82 Mon Sep 17 00:00:00 2001
From: Manu <21658174+epignot@users.noreply.github.com>
Date: Wed, 3 Apr 2024 09:17:43 +0200
Subject: [PATCH 1/8] bump proxy-body-size to 15M (#195)

Required to enable large file save.

Co-authored-by: Emmanuel Pignot <emmanuel.pignot@netapp.com>
---
 charts/bigdata-proxy/Chart.yaml  | 2 +-
 charts/bigdata-proxy/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/bigdata-proxy/Chart.yaml b/charts/bigdata-proxy/Chart.yaml
index 8c89758f..c12b462d 100644
--- a/charts/bigdata-proxy/Chart.yaml
+++ b/charts/bigdata-proxy/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-proxy
 description: A Helm chart for the Spot Big Data Proxy
 type: application
-version: 0.4.6
+version: 0.4.7
 appVersion: 0.5.3
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-proxy/values.yaml b/charts/bigdata-proxy/values.yaml
index 6c452447..392680c2 100644
--- a/charts/bigdata-proxy/values.yaml
+++ b/charts/bigdata-proxy/values.yaml
@@ -28,7 +28,7 @@ ingress:
   host: ""  # Overridden at deploy time
   secretNamespace: ""  # Overridden at deploy time
   secretName: ""  # Overridden at deploy time
-  bodySize: "2M"
+  bodySize: "15M"
 
 nameOverride: ""
 fullnameOverride: ""

From 0715a4cfd1b7f8d0f66fd66abf7af4384cc49415 Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Mon, 18 Mar 2024 17:58:04 +0100
Subject: [PATCH 2/8] [charts] add fluebtbit custom configs to all bigdata
 components

[charts/bigdata-operator] create common and custom fluentbit configmaps

[charts/bigdata-proxy] create custom fluentbit configmaps

[charts/bigdata-spark-watcher] create custom fluentbit configmaps

[charts/spark-operator] create custom fluentbit configmaps

[charts/notebook-service-storage] create custom fluentbit configmaps

[charts/notebook-service] create custom fluentbit configmaps

[charts] bump versions

[charts/bigdata-notebook-workspace] add telemetry sidecar

[charts/bigdata-telemetry] fix bigdata-telemetry thanos args & object-store mounting

[charts/bigdata-operator] add metrics collection to fluentbit

bump telemetry chart version

fix telemetry thanos receiver config

set thanos upload frequency to 15 minutes

add metrics collection config to all components

fix metrics collection configs in all components

fix telemetry thanos secret mounting

reviews

reviews:templating

reviews:leftovers

bump version
---
 .../Chart.yaml                                |  2 +-
 .../notebook-service-storage-deployment.yaml  | 30 ++++----
 .../templates/telemetry_configmap.yaml        | 17 +++++
 charts/bigdata-notebook-service/Chart.yaml    |  2 +-
 .../templates/deployment.yaml                 | 15 +++-
 .../templates/telemetry_configmap.yaml        | 17 +++++
 charts/bigdata-notebook-workspace/Chart.yaml  |  2 +-
 .../templates/deployment.yaml                 | 68 ++++++++++++++++++-
 .../templates/telemetry_configmap.yaml        | 17 +++++
 charts/bigdata-notebook-workspace/values.yaml | 10 ++-
 charts/bigdata-operator/Chart.yaml            |  2 +-
 .../templates/common_telemetry_configmap.yaml | 60 ++++++++++++++++
 .../templates/deployment.yaml                 | 15 +++-
 .../templates/telemetry_configmap.yaml        | 30 ++++++--
 charts/bigdata-operator/values.yaml           | 53 ---------------
 charts/bigdata-proxy/Chart.yaml               |  2 +-
 .../bigdata-proxy/templates/deployment.yaml   | 15 +++-
 .../templates/telemetry_configmap.yaml        | 18 +++++
 charts/bigdata-spark-watcher/Chart.yaml       |  2 +-
 .../templates/deployment.yaml                 | 15 +++-
 .../templates/telemetry_configmap.yaml        | 33 +++++++++
 charts/bigdata-telemetry/Chart.yaml           |  2 +-
 .../templates/thanos-receiver-configmap.yaml  |  2 +-
 .../templates/thanos-receiver-service.yaml    |  4 +-
 .../thanos-receiver-statefulset.yaml          | 27 ++++----
 charts/bigdata-telemetry/values.yaml          |  2 +-
 charts/spark-operator/Chart.yaml              |  2 +-
 .../spark-operator/templates/deployment.yaml  | 15 +++-
 .../templates/telemetry_configmap.yaml        | 33 +++++++++
 29 files changed, 395 insertions(+), 117 deletions(-)
 create mode 100644 charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml
 create mode 100644 charts/bigdata-notebook-service/templates/telemetry_configmap.yaml
 create mode 100644 charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml
 create mode 100644 charts/bigdata-operator/templates/common_telemetry_configmap.yaml
 create mode 100644 charts/bigdata-proxy/templates/telemetry_configmap.yaml
 create mode 100644 charts/bigdata-spark-watcher/templates/telemetry_configmap.yaml
 create mode 100644 charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml

diff --git a/charts/bigdata-notebook-service-storage-server/Chart.yaml b/charts/bigdata-notebook-service-storage-server/Chart.yaml
index 110b19cf..b7ba2c18 100644
--- a/charts/bigdata-notebook-service-storage-server/Chart.yaml
+++ b/charts/bigdata-notebook-service-storage-server/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-service-storage-server
 description: A Helm chart for the Spot Big Data Notebook Service Storage Server
 type: application
-version: 0.1.11
+version: 0.1.12
 appVersion: "1.2.0-ofas"
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml b/charts/bigdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml
index c9b31334..2f21a3b1 100644
--- a/charts/bigdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml
+++ b/charts/bigdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml
@@ -46,16 +46,6 @@ spec:
             containerPort: 2020
             protocol: TCP
         env:
-          - name: AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_ACCESS_KEY_ID
-          - name: AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_SECRET_ACCESS_KEY
           - name: AWS_BUCKET_NAME
             valueFrom:
               secretKeyRef:
@@ -78,25 +68,39 @@ spec:
                 key: accountId
         resources: {}
         volumeMounts:
-          - name: config
+          - name: telementry-global-config
             mountPath: /fluent-bit/etc/fluent-bit.conf
             subPath: fluent-bit.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/custom-filters.conf
+            subPath: custom-filters.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/metrics-collection.conf
+            subPath: metrics-collection.conf
           - name: varlog
             readOnly: true
             mountPath: /var/log/
           - name: varlibdockercontainers
             readOnly: true
             mountPath: /var/lib/docker/containers
+          - name: telemetry-aws-credentials
+            mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
+          configMap:
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name:  {{ include "bigdata-notebook-service-storage-server.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
         - name: varlibdockercontainers
           hostPath:
             path: /var/lib/docker/containers
+        - name: telemetry-aws-credentials
+          secret:
+            secretName: spot-bigdata-telemetry-creds
     {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
diff --git a/charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..2cb06002
--- /dev/null
+++ b/charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-service-storage-server.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-service-storage-server.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ include "bigdata-notebook-service-storage-server.fullname" . }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook service storage server
+{{- end }}
diff --git a/charts/bigdata-notebook-service/Chart.yaml b/charts/bigdata-notebook-service/Chart.yaml
index 41985aca..fc19df5c 100644
--- a/charts/bigdata-notebook-service/Chart.yaml
+++ b/charts/bigdata-notebook-service/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-service
 description: A Helm chart for the Spot Big Data Notebook Service
 type: application
-version: 0.2.4
+version: 0.2.5
 appVersion: 0.1.16
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-notebook-service/templates/deployment.yaml b/charts/bigdata-notebook-service/templates/deployment.yaml
index b0a4c606..bab46970 100644
--- a/charts/bigdata-notebook-service/templates/deployment.yaml
+++ b/charts/bigdata-notebook-service/templates/deployment.yaml
@@ -168,9 +168,15 @@ spec:
                   optional: true
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -180,9 +186,12 @@ spec:
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
diff --git a/charts/bigdata-notebook-service/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-service/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..8ab50db0
--- /dev/null
+++ b/charts/bigdata-notebook-service/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-service.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook service
+{{- end }}
diff --git a/charts/bigdata-notebook-workspace/Chart.yaml b/charts/bigdata-notebook-workspace/Chart.yaml
index 0697b4ae..0693f0af 100644
--- a/charts/bigdata-notebook-workspace/Chart.yaml
+++ b/charts/bigdata-notebook-workspace/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-workspace
 description: A Helm chart for the Spot Big Data Notebook Workspace
 type: application
-version: 0.0.8
+version: 0.0.9
 appVersion: 4.0.10-ofas-d35201c
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-notebook-workspace/templates/deployment.yaml b/charts/bigdata-notebook-workspace/templates/deployment.yaml
index ea6e7b96..fad7bcca 100644
--- a/charts/bigdata-notebook-workspace/templates/deployment.yaml
+++ b/charts/bigdata-notebook-workspace/templates/deployment.yaml
@@ -88,12 +88,78 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+        {{- if .Values.telemetry.enabled }}
+        - name: fluentbit
+          image: "{{ .Values.telemetry.fluentbit.image.repository }}:{{ .Values.telemetry.fluentbit.image.tag }}"
+          ports:
+            - name: http
+              containerPort: 2020
+              protocol: TCP
+          env:
+            - name: AWS_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: spot-bigdata-telemetry-creds
+                  key: AWS_BUCKET_NAME
+            - name: AWS_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: spot-bigdata-telemetry-creds
+                  key: AWS_REGION
+            - name: CLUSTER_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: spot-ofas-cluster-info
+                  key: clusterId
+            - name: ACCOUNT_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: spot-ofas-cluster-info
+                  key: accountId
+          volumeMounts:
+            - name: telementry-global-config
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
+            - name: varlog
+              readOnly: true
+              mountPath: /var/log/
+            - name: varlibdockercontainers
+              readOnly: true
+              mountPath: /var/lib/docker/containers
+            - name: telemetry-aws-credentials
+              mountPath: /root/.aws
+      {{- end }}
       restartPolicy: {{ .Values.restartPolicy }}
-      {{- if .Values.pvc.create }}
+      {{- if or .Values.pvc.create .Values.telemetry.enabled }}
       volumes:
+        {{- if .Values.pvc.create }}
         - name: {{ .Values.volume.name }}
           persistentVolumeClaim:
             claimName: {{ .Release.Name }}
+        {{- end }}
+        {{- if .Values.telemetry.enabled }}
+        - name: telementry-global-config
+          configMap:
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-notebook-workspace.fullname" . }}-telemetry-cm
+        - name: varlog
+          hostPath:
+            path: /var/log/
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: telemetry-aws-credentials
+          secret:
+            secretName: spot-bigdata-telemetry-creds
+        {{- end }}
       {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
diff --git a/charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..93beecdf
--- /dev/null
+++ b/charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-workspace.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-workspace.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook workspace
+{{- end }}
diff --git a/charts/bigdata-notebook-workspace/values.yaml b/charts/bigdata-notebook-workspace/values.yaml
index 17daedcb..9835822a 100644
--- a/charts/bigdata-notebook-workspace/values.yaml
+++ b/charts/bigdata-notebook-workspace/values.yaml
@@ -30,8 +30,16 @@ gatewayClient:
   headers: '{"Content-Type": "application/json"}'
   secretKeyToken: token
 
+telemetry:
+  enabled: false
+  fluentbit:
+    image:
+      repository: public.ecr.aws/ocean-spark/fluent-bit
+      tag: 2.0.10
+
+podLabels:
+  bigdata.spot.io/component: "bigdata-notebook-workspace"
 podAnnotations: {}
-podLabels: {}
 sharedLabels: {}
 sharedAnnotations: {}
 
diff --git a/charts/bigdata-operator/Chart.yaml b/charts/bigdata-operator/Chart.yaml
index 509f5dfc..5fbc68d7 100644
--- a/charts/bigdata-operator/Chart.yaml
+++ b/charts/bigdata-operator/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-operator
 description: Spot Ocean BigData Operator
 type: application
-version: 0.4.10
+version: 0.4.11
 appVersion: 0.4.9
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-operator/templates/common_telemetry_configmap.yaml b/charts/bigdata-operator/templates/common_telemetry_configmap.yaml
new file mode 100644
index 00000000..9906951c
--- /dev/null
+++ b/charts/bigdata-operator/templates/common_telemetry_configmap.yaml
@@ -0,0 +1,60 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bigdata-common-telemetry-fluentbit-cm
+  labels:
+  {{- include "bigdata-operator.labels" . | nindent 4 }}
+data:
+  parsers.conf: |
+    [PARSER]
+        # http://rubular.com/r/tjUt3Awgg4
+        Name cri
+        Format regex
+        Regex ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>[A-Z]+)\t(?<logger>[^\s]+)\t(?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+        storage.max_chunks_up     10
+
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush        1
+        Log_Level    info
+        Parsers_File parsers.conf
+        HTTP_Server   On
+        HTTP_Listen   0.0.0.0
+        HTTP_Port     2020
+
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        Parser cri
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+
+    @INCLUDE custom-filters.conf
+    @INCLUDE metrics-collection.conf
+
+    [FILTER]
+        Name modify
+        Match kube.*
+        Add ClusterId ${CLUSTER_ID}
+        Add AccountID ${ACCOUNT_ID}
+
+    [OUTPUT]
+        Name s3
+        Match kube.*
+        bucket                       ${AWS_BUCKET_NAME}
+        region                       ${AWS_REGION}
+        use_put_object               On
+        total_file_size              5M
+        upload_timeout               5m
+{{- end }}
\ No newline at end of file
diff --git a/charts/bigdata-operator/templates/deployment.yaml b/charts/bigdata-operator/templates/deployment.yaml
index be6265c8..746ea408 100644
--- a/charts/bigdata-operator/templates/deployment.yaml
+++ b/charts/bigdata-operator/templates/deployment.yaml
@@ -118,9 +118,15 @@ spec:
                   optional: true
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -130,9 +136,12 @@ spec:
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-operator.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
diff --git a/charts/bigdata-operator/templates/telemetry_configmap.yaml b/charts/bigdata-operator/templates/telemetry_configmap.yaml
index 5321efda..8d80ec22 100644
--- a/charts/bigdata-operator/templates/telemetry_configmap.yaml
+++ b/charts/bigdata-operator/templates/telemetry_configmap.yaml
@@ -2,14 +2,32 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: bigdata-telemetry-fluentbit-cm
+  name: {{ include "bigdata-operator.fullname" . }}-telemetry-cm
   labels:
   {{- include "bigdata-operator.labels" . | nindent 4 }}
 data:
-  parsers.conf: |
-{{ .Values.telemetry.fluentbit.configMap.parsers | toString | indent 6 }}
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
 
-  fluent-bit.conf: |
-{{ .Values.telemetry.fluentbit.configMap.config | toString | indent 6 }}
-{{- end }}
+  metrics-collection.conf: |
+    [INPUT]
+      name                    prometheus_scrape
+      host                    0.0.0.0
+      port                    8080
+      tag                     {{ include "bigdata-operator.fullname" . }}
+      metrics_path            /metrics
+      scrape_interval         5s
 
+    [OUTPUT]
+      Name                 prometheus_remote_write
+      Match                {{ include "bigdata-operator.fullname" . }}
+      Host                 bigdata-telemetry-thanos-receiver-svc.{{ .Release.Namespace }}.svc.cluster.local
+      Port                 19291
+      uri                  /api/v1/receive
+      tls                  off
+      tls.verify           off
+      Workers              1
+{{- end }}
diff --git a/charts/bigdata-operator/values.yaml b/charts/bigdata-operator/values.yaml
index 25f4572d..6a9f3799 100644
--- a/charts/bigdata-operator/values.yaml
+++ b/charts/bigdata-operator/values.yaml
@@ -50,59 +50,6 @@ telemetry:
     image:
       repository: public.ecr.aws/ocean-spark/fluent-bit
       tag: 2.0.10
-    configMap:
-      parsers: |
-        [PARSER]
-            # http://rubular.com/r/tjUt3Awgg4
-            Name cri
-            Format regex
-            Regex ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>[A-Z]+)\t(?<logger>[^\s]+)\t(?<message>.*)$
-            Time_Key    time
-            Time_Format %Y-%m-%dT%H:%M:%S.%L%z
-
-      config: |
-        [SERVICE]
-            Flush        1
-            Log_Level    info
-            Parsers_File parsers.conf
-            HTTP_Server   On
-            HTTP_Listen   0.0.0.0
-            HTTP_Port     2020
-
-        [INPUT]
-            Name tail
-            Path /var/log/containers/*.log
-            Parser cri
-            Tag kube.*
-            Mem_Buf_Limit 5MB
-
-        [FILTER]
-            Name kubernetes
-            Match kube.*
-            Merge_Log On
-            Keep_Log Off
-            K8S-Logging.Parser On
-            K8S-Logging.Exclude On
-
-        [FILTER]
-            Name modify
-            Match kube.*
-            Add ClusterId ${CLUSTER_ID}
-            Add AccountID ${ACCOUNT_ID}
-
-        [FILTER]
-            Name    grep
-            Match   kube.*
-            Regex   $kubernetes['labels']['bigdata.spot.io/component'] .
-
-        [OUTPUT]
-            Name s3
-            Match kube.*
-            bucket                       ${AWS_BUCKET_NAME}
-            region                       ${AWS_REGION}
-            use_put_object               On
-            total_file_size              5M
-            upload_timeout               5m
 
 resources:
   limits:
diff --git a/charts/bigdata-proxy/Chart.yaml b/charts/bigdata-proxy/Chart.yaml
index c12b462d..8ea5ed3f 100644
--- a/charts/bigdata-proxy/Chart.yaml
+++ b/charts/bigdata-proxy/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-proxy
 description: A Helm chart for the Spot Big Data Proxy
 type: application
-version: 0.4.7
+version: 0.4.8
 appVersion: 0.5.3
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-proxy/templates/deployment.yaml b/charts/bigdata-proxy/templates/deployment.yaml
index e37e37f7..2460475d 100644
--- a/charts/bigdata-proxy/templates/deployment.yaml
+++ b/charts/bigdata-proxy/templates/deployment.yaml
@@ -83,9 +83,15 @@ spec:
                   key: accountId
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -95,9 +101,12 @@ spec:
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-proxy.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
diff --git a/charts/bigdata-proxy/templates/telemetry_configmap.yaml b/charts/bigdata-proxy/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..abd5c4cc
--- /dev/null
+++ b/charts/bigdata-proxy/templates/telemetry_configmap.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-proxy.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-proxy.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the bigdata proxy
+
+{{- end }}
diff --git a/charts/bigdata-spark-watcher/Chart.yaml b/charts/bigdata-spark-watcher/Chart.yaml
index 349fb10d..afbec017 100644
--- a/charts/bigdata-spark-watcher/Chart.yaml
+++ b/charts/bigdata-spark-watcher/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-spark-watcher
 description: A Helm chart for the Spot Big Data Spark Watcher
 type: application
-version: 0.5.10
+version: 0.5.11
 appVersion: 0.4.10
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-spark-watcher/templates/deployment.yaml b/charts/bigdata-spark-watcher/templates/deployment.yaml
index 258b456c..996b1762 100644
--- a/charts/bigdata-spark-watcher/templates/deployment.yaml
+++ b/charts/bigdata-spark-watcher/templates/deployment.yaml
@@ -126,9 +126,15 @@ spec:
                   optional: true
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -146,9 +152,12 @@ spec:
             secretName: spot-bigdata-log-collector-creds
       {{- end }}
       {{- if .Values.telemetry.enabled }}
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-spark-watcher.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
diff --git a/charts/bigdata-spark-watcher/templates/telemetry_configmap.yaml b/charts/bigdata-spark-watcher/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..e8dd6cc8
--- /dev/null
+++ b/charts/bigdata-spark-watcher/templates/telemetry_configmap.yaml
@@ -0,0 +1,33 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-spark-watcher.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-spark-watcher.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    [INPUT]
+      name                    prometheus_scrape
+      host                    0.0.0.0
+      port                    8080
+      tag                     {{ include "bigdata-spark-watcher.fullname" . }}
+      metrics_path            /metrics
+      scrape_interval         5s
+
+    [OUTPUT]
+      Name                 prometheus_remote_write
+      Match                {{ include "bigdata-spark-watcher.fullname" . }}
+      Host                 bigdata-telemetry-thanos-receiver-svc.spot-system.svc.cluster.local
+      Port                 19291
+      uri                  /api/v1/receive
+      tls                  off
+      tls.verify           off
+      Workers              1
+{{- end }}
diff --git a/charts/bigdata-telemetry/Chart.yaml b/charts/bigdata-telemetry/Chart.yaml
index b8fb7914..129ee308 100644
--- a/charts/bigdata-telemetry/Chart.yaml
+++ b/charts/bigdata-telemetry/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-telemetry
 description: A Helm chart for the Spot Big Data Telemetry components
 type: application
-version: 0.2.1
+version: 0.2.2
 appVersion: "1.16.0"
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-telemetry/templates/thanos-receiver-configmap.yaml b/charts/bigdata-telemetry/templates/thanos-receiver-configmap.yaml
index 4c4caa1e..1aa8d8bd 100644
--- a/charts/bigdata-telemetry/templates/thanos-receiver-configmap.yaml
+++ b/charts/bigdata-telemetry/templates/thanos-receiver-configmap.yaml
@@ -10,7 +10,7 @@ data:
     [
       {
         "endpoints": [
-          "{{ include "bigdata-telemetry.fullname" . }}-thanos-receiver-svc.{{ .Release.Namespace }}.svc.cluster.local:10901"
+          "127.0.0.1:10901"
         ]
       }
     ]
\ No newline at end of file
diff --git a/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml b/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
index d63f95c4..83d7f2e3 100644
--- a/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
+++ b/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
@@ -17,6 +17,4 @@ spec:
     port: 19291
     targetPort: 19291
   selector:
-    {{- with .Values.thanos.podLabels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
\ No newline at end of file
+    {{- include "bigdata-telemetry.selectorLabels" . | nindent 4 }}
\ No newline at end of file
diff --git a/charts/bigdata-telemetry/templates/thanos-receiver-statefulset.yaml b/charts/bigdata-telemetry/templates/thanos-receiver-statefulset.yaml
index d3771dea..ef386752 100644
--- a/charts/bigdata-telemetry/templates/thanos-receiver-statefulset.yaml
+++ b/charts/bigdata-telemetry/templates/thanos-receiver-statefulset.yaml
@@ -10,10 +10,7 @@ spec:
   selector:
     matchLabels:
       {{- include "bigdata-telemetry.selectorLabels" . | nindent 6 }}
-      {{- with .Values.thanos.podLabels }}
-      {{- toYaml . | nindent 6 }}
-      {{- end }}
-  serviceName: {{ include "bigdata-telemetry.fullname" . }}-thanos-receiver
+  serviceName: {{ include "bigdata-telemetry.fullname" . }}-thanos-receiver-svc
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -22,21 +19,21 @@ spec:
       {{- end }}
       labels:
         {{- include "bigdata-telemetry.selectorLabels" . | nindent 8 }}
-        {{- with .Values.thanos.podLabels }}
-        {{- toYaml . | nindent 6 }}
-        {{- end }}
     spec:
       containers:
         - args:
             - receive
-            - --objstore.config=$(OBJSTORE_CONFIG)
+            - --objstore.config-file=$(OBJSTORE_CONFIG_FILE)
             - --tsdb.path=/var/thanos/receive
             - --label=cluster_id="$(SPOTINST_CLUSTER_ID)"
             - --grpc-address=0.0.0.0:10901
             - --http-address=0.0.0.0:10902
             - --remote-write.address=0.0.0.0:19291
             - --receive.local-endpoint=127.0.0.1:10901
+            - --receive.grpc-compression=none
             - --tsdb.retention=1d
+            - --tsdb.min-block-duration=15m
+            - --tsdb.max-block-duration=15m
             - --receive.hashrings-file=/var/lib/thanos-receive/hashrings.json
           env:
             - name: SPOTINST_CLUSTER_ID
@@ -48,13 +45,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            - name: OBJSTORE_CONFIG
-              valueFrom:
-                secretKeyRef:
-                  key: thanos_object_store
-                  name: spot-bigdata-telemetry-creds
-                  key: object-store.yaml
-                  name: bigdata-telemetry-object-store
+            - name: OBJSTORE_CONFIG_FILE
+              value: /var/lib/creds/thanos_object_store.yaml
             - name: HTTP_PROXY
               valueFrom:
                 configMapKeyRef:
@@ -99,10 +91,15 @@ spec:
               readOnly: false
             - mountPath: /var/lib/thanos-receive
               name: hashring-config
+            - name: telemetry-aws-credentials
+              mountPath: /var/lib/creds
       terminationGracePeriodSeconds: 900
       volumes:
         - emptyDir: {}
           name: data
+        - name: telemetry-aws-credentials
+          secret:
+            secretName: spot-bigdata-telemetry-creds
         - configMap:
             name: {{ include "bigdata-telemetry.fullname" . }}-thanos-receiver-cm
           name: hashring-config
diff --git a/charts/bigdata-telemetry/values.yaml b/charts/bigdata-telemetry/values.yaml
index f30f9992..88ce7fb7 100644
--- a/charts/bigdata-telemetry/values.yaml
+++ b/charts/bigdata-telemetry/values.yaml
@@ -10,7 +10,7 @@ podLabels:
 thanos:
   image:
     repository: quay.io/thanos/thanos
-    tag: v0.15.0
+    tag: v0.34.1
 
 
 nodeSelector: {}
diff --git a/charts/spark-operator/Chart.yaml b/charts/spark-operator/Chart.yaml
index 7cc9f027..85086279 100644
--- a/charts/spark-operator/Chart.yaml
+++ b/charts/spark-operator/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 description: Spark Operator (b/g part)
 name: spark-operator
-version: 0.1.26
+version: 0.1.27
 appVersion: v1beta2-1.3.4-3.1.1
 dependencies:
   - name: spark-operator
diff --git a/charts/spark-operator/charts/spark-operator/templates/deployment.yaml b/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
index bf14c19e..dc1adad2 100644
--- a/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
+++ b/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
@@ -164,9 +164,15 @@ spec:
                   key: accountId
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -188,9 +194,12 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
         {{- if .Values.telemetry.enabled }}
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "spark-operator.fullname" . }}-telemetry-cm
         - name: telemetry-aws-credentials
           secret:
             secretName: spot-bigdata-telemetry-creds
diff --git a/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
new file mode 100644
index 00000000..6c6865d5
--- /dev/null
+++ b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
@@ -0,0 +1,33 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "spark-operator.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "spark-operator.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    [INPUT]
+      name                    prometheus_scrape
+      host                    0.0.0.0
+      port                    {{ .Values.metrics.portName }}
+      tag                     {{ include "spark-operator.fullname" . }}
+      metrics_path            {{ .Values.metrics.endpoint }}
+      scrape_interval         5s
+
+    [OUTPUT]
+      Name                 prometheus_remote_write
+      Match                {{ include "spark-operator.fullname" . }}
+      Host                 bigdata-telemetry-thanos-receiver-svc.{{ .Release.Namespace }}.svc.cluster.local
+      Port                 19291
+      uri                  /api/v1/receive
+      tls                  off
+      tls.verify           off
+      Workers              1
+{{- end }}

From de6474e8e3eeeec8f267fd9c18b8f33a77197dc0 Mon Sep 17 00:00:00 2001
From: Alex Tarasov <54230036+alextarasov-spot@users.noreply.github.com>
Date: Mon, 8 Apr 2024 13:21:37 +0300
Subject: [PATCH 3/8] BGD-4774 - bigdata-watcher new version (#194)

* BGD-4774 - bigdata-watcher new version
---
 charts/bigdata-spark-watcher/Chart.yaml  | 4 ++--
 charts/bigdata-spark-watcher/values.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/charts/bigdata-spark-watcher/Chart.yaml b/charts/bigdata-spark-watcher/Chart.yaml
index afbec017..8728da4b 100644
--- a/charts/bigdata-spark-watcher/Chart.yaml
+++ b/charts/bigdata-spark-watcher/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: bigdata-spark-watcher
 description: A Helm chart for the Spot Big Data Spark Watcher
 type: application
-version: 0.5.11
-appVersion: 0.4.10
+version: 0.5.12
+appVersion: 0.5.0
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
 sources:
diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml
index c890f746..03be8fa0 100644
--- a/charts/bigdata-spark-watcher/values.yaml
+++ b/charts/bigdata-spark-watcher/values.yaml
@@ -8,7 +8,7 @@ image:
   repository: 066597193667.dkr.ecr.us-east-1.amazonaws.com/private/bigdata-spark-watcher
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: 0.4.10-89694e75
+  tag: 0.5.0-19090a2b
 
 imagePullSecrets:
   - name: spot-bigdata-image-pull

From 0640a2e67a02e89e97fa0faa742ce6f93bc06fbc Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Tue, 9 Apr 2024 23:55:07 +0200
Subject: [PATCH 4/8] [charts/spark-operator] fix indentation in the telemetry
 sidecar section

---
 charts/spark-operator/Chart.yaml              |  2 +-
 .../spark-operator/templates/deployment.yaml  | 92 +++++++++----------
 2 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/charts/spark-operator/Chart.yaml b/charts/spark-operator/Chart.yaml
index 85086279..e1a25b09 100644
--- a/charts/spark-operator/Chart.yaml
+++ b/charts/spark-operator/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 description: Spark Operator (b/g part)
 name: spark-operator
-version: 0.1.27
+version: 0.1.28
 appVersion: v1beta2-1.3.4-3.1.1
 dependencies:
   - name: spark-operator
diff --git a/charts/spark-operator/charts/spark-operator/templates/deployment.yaml b/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
index dc1adad2..a2fe79d9 100644
--- a/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
+++ b/charts/spark-operator/charts/spark-operator/templates/deployment.yaml
@@ -135,52 +135,52 @@ spec:
         {{- toYaml . | nindent 10 }}
         {{- end }}
       {{- if .Values.telemetry.enabled }}
-        - name: fluentbit
-          image: "{{ .Values.telemetry.fluentbit.image.repository }}:{{ .Values.telemetry.fluentbit.image.tag }}"
-          ports:
-            - name: http
-              containerPort: 2020
-              protocol: TCP
-          env:
-            - name: AWS_BUCKET_NAME
-              valueFrom:
-                secretKeyRef:
-                  name: spot-bigdata-telemetry-creds
-                  key: AWS_BUCKET_NAME
-            - name: AWS_REGION
-              valueFrom:
-                secretKeyRef:
-                  name: spot-bigdata-telemetry-creds
-                  key: AWS_REGION
-            - name: CLUSTER_ID
-              valueFrom:
-                configMapKeyRef:
-                  name: spot-ofas-cluster-info
-                  key: clusterId
-            - name: ACCOUNT_ID
-              valueFrom:
-                configMapKeyRef:
-                  name: spot-ofas-cluster-info
-                  key: accountId
-          resources: {}
-          volumeMounts:
-            - name: telementry-global-config
-              mountPath: /fluent-bit/etc/fluent-bit.conf
-              subPath: fluent-bit.conf
-            - name: telementry-custom-config
-              mountPath: /fluent-bit/etc/custom-filters.conf
-              subPath: custom-filters.conf
-            - name: telementry-custom-config
-              mountPath: /fluent-bit/etc/metrics-collection.conf
-              subPath: metrics-collection.conf
-            - name: varlog
-              readOnly: true
-              mountPath: /var/log/
-            - name: varlibdockercontainers
-              readOnly: true
-              mountPath: /var/lib/docker/containers
-            - name: telemetry-aws-credentials
-              mountPath: /root/.aws
+      - name: fluentbit
+        image: "{{ .Values.telemetry.fluentbit.image.repository }}:{{ .Values.telemetry.fluentbit.image.tag }}"
+        ports:
+          - name: http
+            containerPort: 2020
+            protocol: TCP
+        env:
+          - name: AWS_BUCKET_NAME
+            valueFrom:
+              secretKeyRef:
+                name: spot-bigdata-telemetry-creds
+                key: AWS_BUCKET_NAME
+          - name: AWS_REGION
+            valueFrom:
+              secretKeyRef:
+                name: spot-bigdata-telemetry-creds
+                key: AWS_REGION
+          - name: CLUSTER_ID
+            valueFrom:
+              configMapKeyRef:
+                name: spot-ofas-cluster-info
+                key: clusterId
+          - name: ACCOUNT_ID
+            valueFrom:
+              configMapKeyRef:
+                name: spot-ofas-cluster-info
+                key: accountId
+        resources: {}
+        volumeMounts:
+          - name: telementry-global-config
+            mountPath: /fluent-bit/etc/fluent-bit.conf
+            subPath: fluent-bit.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/custom-filters.conf
+            subPath: custom-filters.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/metrics-collection.conf
+            subPath: metrics-collection.conf
+          - name: varlog
+            readOnly: true
+            mountPath: /var/log/
+          - name: varlibdockercontainers
+            readOnly: true
+            mountPath: /var/lib/docker/containers
+          - name: telemetry-aws-credentials
+            mountPath: /root/.aws
       {{- end }}
       {{- if or .Values.webhook.enable (ne (len .Values.volumes) 0) .Values.telemetry.enabled }}
       volumes:

From a4ea6a963806d97622a31d8adc89cbcbd13693c1 Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Wed, 10 Apr 2024 00:06:08 +0200
Subject: [PATCH 5/8] [charts/spark-operator] fix telemetry input port number

---
 .../charts/spark-operator/templates/telemetry_configmap.yaml    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
index 6c6865d5..4cab8280 100644
--- a/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
+++ b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
@@ -16,7 +16,7 @@ data:
     [INPUT]
       name                    prometheus_scrape
       host                    0.0.0.0
-      port                    {{ .Values.metrics.portName }}
+      port                    {{ .Values.metrics.port }}
       tag                     {{ include "spark-operator.fullname" . }}
       metrics_path            {{ .Values.metrics.endpoint }}
       scrape_interval         5s

From 7d19f864a3faa621efd1a27dff781ae690c0cc5e Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Wed, 10 Apr 2024 00:26:04 +0200
Subject: [PATCH 6/8] [charts/bigdata-telemetry] add a service with a fixed
 name

---
 charts/bigdata-telemetry/Chart.yaml           |  2 +-
 .../templates/thanos-receiver-service.yaml    | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/charts/bigdata-telemetry/Chart.yaml b/charts/bigdata-telemetry/Chart.yaml
index 129ee308..0a3b998b 100644
--- a/charts/bigdata-telemetry/Chart.yaml
+++ b/charts/bigdata-telemetry/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-telemetry
 description: A Helm chart for the Spot Big Data Telemetry components
 type: application
-version: 0.2.2
+version: 0.2.3
 appVersion: "1.16.0"
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml b/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
index 83d7f2e3..0de6a698 100644
--- a/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
+++ b/charts/bigdata-telemetry/templates/thanos-receiver-service.yaml
@@ -5,6 +5,26 @@ metadata:
   name: {{ include "bigdata-telemetry.fullname" . }}-thanos-receiver-svc
   labels:
     {{- include "bigdata-telemetry.labels" . | nindent 4 }}
+spec:
+  ports:
+  - name: grpc
+    port: 10901
+    targetPort: 10901
+  - name: http
+    port: 10902
+    targetPort: 10902
+  - name: remote-write
+    port: 19291
+    targetPort: 19291
+  selector:
+    {{- include "bigdata-telemetry.selectorLabels" . | nindent 4 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "bigdata-telemetry.name" . }}-thanos-receiver-svc
+  labels:
+    {{- include "bigdata-telemetry.labels" . | nindent 4 }}
 spec:
   ports:
   - name: grpc

From d1ef66e4a0f712b3d04f002d92b56745d4862289 Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Thu, 11 Apr 2024 11:45:39 +0200
Subject: [PATCH 7/8] [charts/spark-operator] add metrics collection only when
 metrics are enabled

---
 .../charts/spark-operator/templates/telemetry_configmap.yaml   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
index 4cab8280..a6b2a327 100644
--- a/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
+++ b/charts/spark-operator/charts/spark-operator/templates/telemetry_configmap.yaml
@@ -13,6 +13,8 @@ data:
         Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
 
   metrics-collection.conf: |
+    ## Configuration for collecting metrics from the spark operator
+    {{- if .Values.metrics.enable }}
     [INPUT]
       name                    prometheus_scrape
       host                    0.0.0.0
@@ -30,4 +32,5 @@ data:
       tls                  off
       tls.verify           off
       Workers              1
+    {{- end }}
 {{- end }}

From 3800b69b9377480bfe7c36a2e8e204f460fa3583 Mon Sep 17 00:00:00 2001
From: Z4ck404 <zakaria.elbazi@netapp.com>
Date: Thu, 11 Apr 2024 15:52:09 +0200
Subject: [PATCH 8/8] [charts/bigdata-notebook-service] Fix volumes in
 notebook-service when telemetry is enabled

---
 charts/bigdata-notebook-service/Chart.yaml    |  2 +-
 .../templates/deployment.yaml                 | 33 ++++++++++---------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/charts/bigdata-notebook-service/Chart.yaml b/charts/bigdata-notebook-service/Chart.yaml
index fc19df5c..c6e21e98 100644
--- a/charts/bigdata-notebook-service/Chart.yaml
+++ b/charts/bigdata-notebook-service/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-service
 description: A Helm chart for the Spot Big Data Notebook Service
 type: application
-version: 0.2.5
+version: 0.2.6
 appVersion: 0.1.16
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png
diff --git a/charts/bigdata-notebook-service/templates/deployment.yaml b/charts/bigdata-notebook-service/templates/deployment.yaml
index bab46970..c06e086c 100644
--- a/charts/bigdata-notebook-service/templates/deployment.yaml
+++ b/charts/bigdata-notebook-service/templates/deployment.yaml
@@ -32,6 +32,23 @@ spec:
       - name: session-storage
         persistentVolumeClaim:
           claimName: {{ .Values.sessionStoragePvcName }}
+      {{- if .Values.telemetry.enabled }}
+      - name: telementry-global-config
+        configMap:
+          name: bigdata-common-telemetry-fluentbit-cm
+      - name: telementry-custom-config
+        configMap:
+          name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
+      - name: varlog
+        hostPath:
+          path: /var/log/
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: telemetry-aws-credentials
+        secret:
+          secretName: spot-bigdata-telemetry-creds
+      {{- end }}
       # This initContainers section should not be needed if the
       # securityContext.fsGroup entry above wasn't ignored, but unfortunately
       # it is ignored for NFS volumes:
@@ -185,22 +202,6 @@ spec:
               mountPath: /var/lib/docker/containers
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
-      volumes:
-        - name: telementry-global-config
-          configMap:
-            name: bigdata-common-telemetry-fluentbit-cm
-        - name: telementry-custom-config
-          configMap:
-            name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
-        - name: varlog
-          hostPath:
-            path: /var/log/
-        - name: varlibdockercontainers
-          hostPath:
-            path: /var/lib/docker/containers
-        - name: telemetry-aws-credentials
-          secret:
-            secretName: spot-bigdata-telemetry-creds
     {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector: