Merge pull request #55 from jfrog/Upgrading_fluentd_image

Upgrading fluentd image
jfrog · Nov 14, 2023 · f401b3e · f401b3e
2 parents f7d4e54 + 38d0210
commit f401b3e
Show file tree

Hide file tree

Showing 6 changed files with 106 additions and 76 deletions.
diff --git a/fluent_metrics.conf.rt b/fluent_metrics.conf.rt
@@ -458,7 +458,7 @@
 
 <match jfrog.rt.**>
   @type loki
-  url LOKI_URL
+  url "#{ENV['LOKI_URL']}"
   flush_interval 1s
   flush_at_shutdown true
   buffer_chunk_limit 1m

diff --git a/fluent_metrics.conf.xray b/fluent_metrics.conf.xray
@@ -333,7 +333,7 @@
 
 <match jfrog.xray.**>
   @type loki
-  url LOKI_URL
+  url "#{ENV['LOKI_URL']}"
   flush_interval 1s
   flush_at_shutdown true
   buffer_chunk_limit 1m

diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml
@@ -3,32 +3,39 @@ global:
     observability:
       branch: master
   prometheus:
-    loki_url: http:\/\/loki.jfrog-plg:3100
+    loki_url: http://loki.jfrog-plg:3100
     release_name: prometheus
 installerInfo: '{ "productId": "Helm_prometheus_artifactory/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" },  { "featureId": "Channel/Helm_prometheus_artifactory" } ] }'
 artifactory:
   openMetrics:
     enabled: true
+  customInitContainersBegin: |
+    - name: "prepare-fluentd-conf-on-persistent-volume"
+      image: "{{ .Values.initContainerImage }}"
+      imagePullPolicy: "{{ .Values.artifactory.image.pullPolicy }}"
+      command:
+        - 'sh'
+        - '-c'
+        - >
+          mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/;
+          curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
+      volumeMounts:
+        - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
+          name: volume
   customSidecarContainers: |
     - name: "artifactory-fluentd-sidecar"
-      image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.1"
       imagePullPolicy: "IfNotPresent"
-      command:
-          - 'sh'
-          - '-c'
-          - >
-            gem install fluent-plugin-prometheus --no-doc;
-            curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o fluentd_metrics.conf;
-            cp fluentd_metrics.conf fluent.tmp.conf;
-            sed -i -e 's/LOKI_URL/{{ .Values.global.prometheus.loki_url }}/g' fluent.tmp.conf;
-            mv fluent.tmp.conf fluentd.conf;
-            fluentd -v -c fluentd.conf
       volumeMounts:
         - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
           name: volume
       env:
         - name: JF_PRODUCT_DATA_INTERNAL
           value: {{ .Values.artifactory.persistence.mountPath }}
+        - name: LOKI_URL
+          value: {{ .Values.global.prometheus.loki_url }}
+        - name: FLUENTD_CONF
+          value: ../../../../{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
 
 ## Allows to add additional kubernetes resources
 ## Use --- as a separator between multiple resources

diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml
@@ -3,33 +3,39 @@ global:
     observability:
       branch: master
   prometheus: 
-    loki_url: http:\/\/loki.jfrog-plg:3100
+    loki_url: http://loki.jfrog-plg:3100
     release_name: prometheus
 installerInfo: '{ "productId": "Helm_prometheus_artifactory/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" },  { "featureId": "Channel/Helm_prometheus_artifactory" } ] }'
 artifactory:
   openMetrics:
     enabled: true
+  customInitContainersBegin: |
+    - name: "prepare-fluentd-conf-on-persistent-volume"
+      image: "{{ .Values.initContainerImage }}"
+      imagePullPolicy: "{{ .Values.artifactory.image.pullPolicy }}"
+      command:
+        - 'sh'
+        - '-c'
+        - >
+          mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/;
+          curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
+      volumeMounts:
+        - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
+          name: artifactory-volume
   customSidecarContainers: |
-      - name: "artifactory-fluentd-sidecar"
-        image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0"
-        imagePullPolicy: "IfNotPresent"
-        command:
-          - 'sh'
-          - '-c'
-          - >
-            gem install fluent-plugin-prometheus --no-doc;
-            curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o fluentd_metrics.conf;
-            cp fluentd_metrics.conf fluent.tmp.conf;
-            sed -i -e 's/LOKI_URL/{{ .Values.global.prometheus.loki_url }}/g' fluent.tmp.conf;
-            mv fluent.tmp.conf fluentd.conf;
-            fluentd -v -c fluentd.conf 
-        volumeMounts:
-          - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
-            name: artifactory-volume
-        env:
-          - name: JF_PRODUCT_DATA_INTERNAL
-            value: {{ .Values.artifactory.persistence.mountPath }}
-
+    - name: "artifactory-fluentd-sidecar"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.1"
+      imagePullPolicy: "IfNotPresent"
+      volumeMounts:
+        - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
+          name: artifactory-volume
+      env:
+        - name: JF_PRODUCT_DATA_INTERNAL
+          value: {{ .Values.artifactory.persistence.mountPath }}
+        - name: LOKI_URL
+          value: {{ .Values.global.prometheus.loki_url }}
+        - name: FLUENTD_CONF
+          value: ../../../../{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
 ## Allows to add additional kubernetes resources
 ## Use --- as a separator between multiple resources
 ## For an example, refer - https://github.com/jfrog/log-analytics-prometheus

diff --git a/helm/jfrog-platform-values.yaml b/helm/jfrog-platform-values.yaml
@@ -3,34 +3,41 @@ global:
     observability:
       branch: master
   prometheus:
-    loki_url: http:\/\/loki.jfrog-plg:3100
+    loki_url: http://loki.jfrog-plg:3100
     release_name: prometheus
 installerInfo: '{ "productId": "Helm_prometheus_jfrog_platform/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" },  { "featureId": "Channel/Helm_prometheus_jfrog_platform" } ] }'
 artifactory:
   artifactory:
     openMetrics:
       enabled: true
     replicaCount: 1
+    customInitContainersBegin: |
+      - name: "prepare-fluentd-conf-on-persistent-volume"
+        image: "{{ .Values.initContainerImage }}"
+        imagePullPolicy: "{{ .Values.artifactory.image.pullPolicy }}"
+        command:
+          - 'sh'
+          - '-c'
+          - >
+            mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/;
+            curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
+        volumeMounts:
+          - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
+            name: artifactory-volume
     customSidecarContainers: |
-        - name: "artifactory-fluentd-sidecar"
-          image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0"
-          imagePullPolicy: "IfNotPresent"
-          command:
-            - 'sh'
-            - '-c'
-            - >
-              gem install fluent-plugin-prometheus --no-doc;
-              curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.rt -o fluentd_metrics.conf;
-              cp fluentd_metrics.conf fluent.tmp.conf;
-              sed -i -e 's/LOKI_URL/{{ .Values.global.prometheus.loki_url }}/g' fluent.tmp.conf;
-              mv fluent.tmp.conf fluentd.conf;
-              fluentd -v -c fluentd.conf
-          volumeMounts:
-            - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
-              name: artifactory-volume
-          env:
-            - name: JF_PRODUCT_DATA_INTERNAL
-              value: {{ .Values.artifactory.persistence.mountPath }}
+      - name: "artifactory-fluentd-sidecar"
+        image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.1"
+        imagePullPolicy: "IfNotPresent"
+        volumeMounts:
+          - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
+            name: artifactory-volume
+        env:
+          - name: JF_PRODUCT_DATA_INTERNAL
+            value: {{ .Values.artifactory.persistence.mountPath }}
+          - name: LOKI_URL
+            value: {{ .Values.global.prometheus.loki_url }}
+          - name: FLUENTD_CONF
+            value: ../../../../{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
   additionalResources: |
     apiVersion: v1
     kind: Service
@@ -85,20 +92,23 @@ xray:
   replicaCount: 1
   common:
     preStartCommand: "export JF_OBSERVABILITY_METRICS_ENABLED=true"
-    customSidecarContainers: |
-      - name: "xray-fluentd-sidecar"
-        image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0"
-        imagePullPolicy: "IfNotPresent"
+    customInitContainersBegin: |
+      - name: "download-fluentd-conf-on-persistent-volume"
+        image: "{{ .Values.initContainerImage }}"
+        imagePullPolicy: "{{ .Values.imagePullPolicy }}"
         command:
           - 'sh'
           - '-c'
           - >
-            gem install fluent-plugin-prometheus --no-doc;
-            curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.xray -o fluent.conf.xray;
-            cp fluent.conf.xray fluent.tmp.conf;
-            sed -i -e 's/LOKI_URL/{{ .Values.global.prometheus.loki_url }}/g' fluent.tmp.conf;
-            mv fluent.tmp.conf fluentd.conf;
-            fluentd -v -c fluentd.conf;
+            mkdir -p {{ .Values.xray.persistence.mountPath }}/etc/fluentd/;
+            curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
+        volumeMounts:
+          - mountPath: "{{ .Values.xray.persistence.mountPath }}"
+            name: data-volume
+    customSidecarContainers: |
+      - name: "xray-platform-fluentd-sidecar"
+        image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.1"
+        imagePullPolicy: "IfNotPresent"
         volumeMounts:
           - mountPath: "{{ .Values.xray.persistence.mountPath }}"
             name: data-volume
@@ -107,6 +117,8 @@ xray:
             value: {{ .Values.xray.persistence.mountPath }}
           - name: FLUENTD_CONF
             value: ../../../../{{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf
+          - name: LOKI_URL
+            value: {{ .Values.global.prometheus.loki_url }}
   additionalResources: |
     apiVersion: v1
     kind: Service

diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml
@@ -3,7 +3,7 @@ global:
     observability:
       branch: master
   prometheus: 
-    loki_url: http:\/\/loki.jfrog-plg:3100
+    loki_url: http://loki.jfrog-plg:3100
     release_name: prometheus
 unifiedUpgradeAllowed: true
 databaseUpgradeReady: true
@@ -21,20 +21,23 @@ rabbitmq:
     password: guest
 common:
   preStartCommand: "export JF_OBSERVABILITY_METRICS_ENABLED=true"
-  customSidecarContainers: |
-    - name: "xray-fluentd-sidecar"
-      image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0"
-      imagePullPolicy: "IfNotPresent"
+  customInitContainersBegin: |
+    - name: "download-fluentd-conf-on-persistent-volume"
+      image: "{{ .Values.initContainerImage }}"
+      imagePullPolicy: "{{ .Values.imagePullPolicy }}"
       command:
         - 'sh'
         - '-c'
         - >
-          gem install fluent-plugin-prometheus --no-doc;
-          curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent.conf.xray -o fluent.conf.xray;
-          cp fluent.conf.xray fluent.tmp.conf;
-          sed -i -e 's/LOKI_URL/{{ .Values.global.prometheus.loki_url }}/g' fluent.tmp.conf;
-          mv fluent.tmp.conf fluentd.conf;
-          fluentd -v -c fluentd.conf;
+          mkdir -p {{ .Values.xray.persistence.mountPath }}/etc/fluentd/;
+          curl https://raw.githubusercontent.com/jfrog/log-analytics-prometheus/{{ .Values.global.jfrog.observability.branch }}/fluent_metrics.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
+      volumeMounts:
+        - mountPath: "{{ .Values.xray.persistence.mountPath }}"
+          name: data-volume
+  customSidecarContainers: |
+    - name: "xray-platform-fluentd-sidecar"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.1"
+      imagePullPolicy: "IfNotPresent"
       volumeMounts:
         - mountPath: "{{ .Values.xray.persistence.mountPath }}"
           name: data-volume
@@ -43,6 +46,8 @@ common:
           value: {{ .Values.xray.persistence.mountPath }}
         - name: FLUENTD_CONF
           value: ../../../../{{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf
+        - name: LOKI_URL
+          value: {{ .Values.global.prometheus.loki_url }}
 ## Allows to add additional kubernetes resources
 ## Use --- as a separator between multiple resources
 ## For an example, refer - https://github.com/jfrog/log-analytics-prometheus