Merge branch 'main' into BGD-4846-support-new-ocean-controller

spotinst · Apr 11, 2024 · 5420b69 · 5420b69
2 parents 72875a3 + 3800b69
commit 5420b69
Show file tree

Hide file tree

Showing 30 changed files with 471 additions and 169 deletions.
diff --git a/charts/bigdata-notebook-service-storage-server/Chart.yaml b/charts/bigdata-notebook-service-storage-server/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-service-storage-server
 description: A Helm chart for the Spot Big Data Notebook Service Storage Server
 type: application
-version: 0.1.11
+version: 0.1.12
 appVersion: "1.2.0-ofas"
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png

diff --git a/...igdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml b/...igdata-notebook-service-storage-server/templates/notebook-service-storage-deployment.yaml
@@ -46,16 +46,6 @@ spec:
             containerPort: 2020
             protocol: TCP
         env:
-          - name: AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_ACCESS_KEY_ID
-          - name: AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: spot-bigdata-telemetry-creds
-                key: AWS_SECRET_ACCESS_KEY
           - name: AWS_BUCKET_NAME
             valueFrom:
               secretKeyRef:
@@ -78,25 +68,39 @@ spec:
                 key: accountId
         resources: {}
         volumeMounts:
-          - name: config
+          - name: telementry-global-config
             mountPath: /fluent-bit/etc/fluent-bit.conf
             subPath: fluent-bit.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/custom-filters.conf
+            subPath: custom-filters.conf
+          - name: telementry-custom-config
+            mountPath: /fluent-bit/etc/metrics-collection.conf
+            subPath: metrics-collection.conf
           - name: varlog
             readOnly: true
             mountPath: /var/log/
           - name: varlibdockercontainers
             readOnly: true
             mountPath: /var/lib/docker/containers
+          - name: telemetry-aws-credentials
+            mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
+          configMap:
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name:  {{ include "bigdata-notebook-service-storage-server.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/
         - name: varlibdockercontainers
           hostPath:
             path: /var/lib/docker/containers
+        - name: telemetry-aws-credentials
+          secret:
+            secretName: spot-bigdata-telemetry-creds
     {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

diff --git a/charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-service-storage-server/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-service-storage-server.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-service-storage-server.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ include "bigdata-notebook-service-storage-server.fullname" . }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook service storage server
+{{- end }}
diff --git a/charts/bigdata-notebook-service/Chart.yaml b/charts/bigdata-notebook-service/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-service
 description: A Helm chart for the Spot Big Data Notebook Service
 type: application
-version: 0.2.4
+version: 0.2.6
 appVersion: 0.1.16
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png

diff --git a/charts/bigdata-notebook-service/templates/deployment.yaml b/charts/bigdata-notebook-service/templates/deployment.yaml
@@ -32,6 +32,23 @@ spec:
       - name: session-storage
         persistentVolumeClaim:
           claimName: {{ .Values.sessionStoragePvcName }}
+      {{- if .Values.telemetry.enabled }}
+      - name: telementry-global-config
+        configMap:
+          name: bigdata-common-telemetry-fluentbit-cm
+      - name: telementry-custom-config
+        configMap:
+          name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
+      - name: varlog
+        hostPath:
+          path: /var/log/
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: telemetry-aws-credentials
+        secret:
+          secretName: spot-bigdata-telemetry-creds
+      {{- end }}
       # This initContainers section should not be needed if the
       # securityContext.fsGroup entry above wasn't ignored, but unfortunately
       # it is ignored for NFS volumes:
@@ -168,9 +185,15 @@ spec:
                   optional: true
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -179,19 +202,6 @@ spec:
               mountPath: /var/lib/docker/containers
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
-      volumes:
-        - name: config
-          configMap:
-            name: bigdata-telemetry-fluentbit-cm
-        - name: varlog
-          hostPath:
-            path: /var/log/
-        - name: varlibdockercontainers
-          hostPath:
-            path: /var/lib/docker/containers
-        - name: telemetry-aws-credentials
-          secret:
-            secretName: spot-bigdata-telemetry-creds
     {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

diff --git a/charts/bigdata-notebook-service/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-service/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-service.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-service.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook service
+{{- end }}
diff --git a/charts/bigdata-notebook-workspace/Chart.yaml b/charts/bigdata-notebook-workspace/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bigdata-notebook-workspace
 description: A Helm chart for the Spot Big Data Notebook Workspace
 type: application
-version: 0.0.8
+version: 0.0.9
 appVersion: 4.0.10-ofas-d35201c
 home: https://github.com/spotinst/charts
 icon: https://docs.spot.io/_media/images/spot_mark.png

diff --git a/charts/bigdata-notebook-workspace/templates/deployment.yaml b/charts/bigdata-notebook-workspace/templates/deployment.yaml
@@ -88,12 +88,78 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+        {{- if .Values.telemetry.enabled }}
+        - name: fluentbit
+          image: "{{ .Values.telemetry.fluentbit.image.repository }}:{{ .Values.telemetry.fluentbit.image.tag }}"
+          ports:
+            - name: http
+              containerPort: 2020
+              protocol: TCP
+          env:
+            - name: AWS_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: spot-bigdata-telemetry-creds
+                  key: AWS_BUCKET_NAME
+            - name: AWS_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: spot-bigdata-telemetry-creds
+                  key: AWS_REGION
+            - name: CLUSTER_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: spot-ofas-cluster-info
+                  key: clusterId
+            - name: ACCOUNT_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: spot-ofas-cluster-info
+                  key: accountId
+          volumeMounts:
+            - name: telementry-global-config
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
+            - name: varlog
+              readOnly: true
+              mountPath: /var/log/
+            - name: varlibdockercontainers
+              readOnly: true
+              mountPath: /var/lib/docker/containers
+            - name: telemetry-aws-credentials
+              mountPath: /root/.aws
+      {{- end }}
       restartPolicy: {{ .Values.restartPolicy }}
-      {{- if .Values.pvc.create }}
+      {{- if or .Values.pvc.create .Values.telemetry.enabled }}
       volumes:
+        {{- if .Values.pvc.create }}
         - name: {{ .Values.volume.name }}
           persistentVolumeClaim:
             claimName: {{ .Release.Name }}
+        {{- end }}
+        {{- if .Values.telemetry.enabled }}
+        - name: telementry-global-config
+          configMap:
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-notebook-workspace.fullname" . }}-telemetry-cm
+        - name: varlog
+          hostPath:
+            path: /var/log/
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: telemetry-aws-credentials
+          secret:
+            secretName: spot-bigdata-telemetry-creds
+        {{- end }}
       {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

diff --git a/charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml b/charts/bigdata-notebook-workspace/templates/telemetry_configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bigdata-notebook-workspace.fullname" . }}-telemetry-cm
+  labels:
+  {{- include "bigdata-notebook-workspace.labels" . | nindent 4 }}
+data:
+  custom-filters.conf: |
+    [FILTER]
+        Name    grep
+        Match   kube.*
+        Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ index .Values.podLabels "bigdata.spot.io/component" | quote }}
+
+  metrics-collection.conf: |
+    # Configuration for collecting metrics from the notebook workspace
+{{- end }}
diff --git a/charts/bigdata-notebook-workspace/values.yaml b/charts/bigdata-notebook-workspace/values.yaml
@@ -30,8 +30,16 @@ gatewayClient:
   headers: '{"Content-Type": "application/json"}'
   secretKeyToken: token
 
+telemetry:
+  enabled: false
+  fluentbit:
+    image:
+      repository: public.ecr.aws/ocean-spark/fluent-bit
+      tag: 2.0.10
+
+podLabels:
+  bigdata.spot.io/component: "bigdata-notebook-workspace"
 podAnnotations: {}
-podLabels: {}
 sharedLabels: {}
 sharedAnnotations: {}
 

diff --git a/charts/bigdata-operator/templates/common_telemetry_configmap.yaml b/charts/bigdata-operator/templates/common_telemetry_configmap.yaml
@@ -0,0 +1,60 @@
+{{- if .Values.telemetry.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bigdata-common-telemetry-fluentbit-cm
+  labels:
+  {{- include "bigdata-operator.labels" . | nindent 4 }}
+data:
+  parsers.conf: |
+    [PARSER]
+        # http://rubular.com/r/tjUt3Awgg4
+        Name cri
+        Format regex
+        Regex ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>[A-Z]+)\t(?<logger>[^\s]+)\t(?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+        storage.max_chunks_up     10
+
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush        1
+        Log_Level    info
+        Parsers_File parsers.conf
+        HTTP_Server   On
+        HTTP_Listen   0.0.0.0
+        HTTP_Port     2020
+
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        Parser cri
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+
+    @INCLUDE custom-filters.conf
+    @INCLUDE metrics-collection.conf
+
+    [FILTER]
+        Name modify
+        Match kube.*
+        Add ClusterId ${CLUSTER_ID}
+        Add AccountID ${ACCOUNT_ID}
+
+    [OUTPUT]
+        Name s3
+        Match kube.*
+        bucket                       ${AWS_BUCKET_NAME}
+        region                       ${AWS_REGION}
+        use_put_object               On
+        total_file_size              5M
+        upload_timeout               5m
+{{- end }}
diff --git a/charts/bigdata-operator/templates/deployment.yaml b/charts/bigdata-operator/templates/deployment.yaml
@@ -118,9 +118,15 @@ spec:
                   optional: true
           resources: {}
           volumeMounts:
-            - name: config
+            - name: telementry-global-config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/custom-filters.conf
+              subPath: custom-filters.conf
+            - name: telementry-custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -130,9 +136,12 @@ spec:
             - name: telemetry-aws-credentials
               mountPath: /root/.aws
       volumes:
-        - name: config
+        - name: telementry-global-config
           configMap:
-            name: bigdata-telemetry-fluentbit-cm
+            name: bigdata-common-telemetry-fluentbit-cm
+        - name: telementry-custom-config
+          configMap:
+            name: {{ include "bigdata-operator.fullname" . }}-telemetry-cm
         - name: varlog
           hostPath:
             path: /var/log/