Merge pull request #7 from signalfx/main

merge main from signalfx

.github/workflows/lint-test.yaml

@@ -12,7 +12,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.4
+        uses: azure/setup-helm@v3.5
         with:
           version: v3.6.3
 
@@ -33,7 +33,7 @@ jobs:
         run: ct lint --config=ct.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.4.0
+        uses: helm/kind-action@v1.5.0
         # Only build a kind cluster if there are chart changes to test.
         if: steps.list-changed.outputs.changed == 'true'
 

.github/workflows/pre-commit.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Install Helm
-      uses: azure/setup-helm@v3.4
+      uses: azure/setup-helm@v3.5
       with:
         version: v3.5.4
     - name: Checkout code

.github/workflows/release.yaml

@@ -23,7 +23,7 @@ jobs:
           git config user.email "[email protected]"
 
       - name: Install Helm
-        uses: azure/setup-helm@v3.4
+        uses: azure/setup-helm@v3.5
         with:
           version: v3.5.4
 

CHANGELOG.md

@@ -4,6 +4,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+## [0.67.0] - 2022-12-19
+
+### Changed
+
+- Upgrade splunk-otel-collector image to 0.67.0 (#612)
+
+### Fixed
+
+- Make sure the daemonset can start in GKE Autopiot (#608)
+- Make containerd engine default in for fluentd logs and use always use it in GKE Autopiot (#609)
+- Temporary disable compression in Splunk Observability logs exporter until
+  0.68.0 to workaround a compression bug (#610)
+
+## [0.66.1] - 2022-12-08
+
+### Fixed
+
+- Fixed network explorer image pull secrets
+
 ## [0.66.0] - 2022-12-06
 
 ### Changed

README.md

@@ -12,7 +12,7 @@
 
 <p align="center">
   <a href="https://circleci.com/gh/signalfx/splunk-otel-collector-chart">
-    <img alt="Build Status" src="https://img.shields.io/github/workflow/status/signalfx/splunk-otel-collector-chart/Lint%20and%20Test%20Charts?style=for-the-badge">
+    <img alt="Build Status" src="https://img.shields.io/github/actions/workflow/status/signalfx/splunk-otel-collector-chart/lint-test.yaml?branch=main&style=for-the-badge">
   </a>
   <a href="https://github.com/signalfx/splunk-otel-collector/releases">
     <img alt="GitHub release (latest by date including pre-releases)" src="https://img.shields.io/github/v/release/signalfx/splunk-otel-collector-chart?include_prereleases&style=for-the-badge">

docs/migration-from-sck.md

@@ -335,3 +335,31 @@ To delete the SCK deployment, find the name of the deployment using the `helm ls
   * ```index="Your logs index"```
 * Check the metrics index to see if you are receiving metrics from your Kubernetes cluster
   * ```| mcatalog values(metric_name) WHERE index="Your metrics index"```
+
+## Differences between Splunk Connect for Kubernetes and Splunk OpenTelemetry Collector for Kubernetes
+
+### Read logs location
+
+Splunk Connect for Kubernetes by default read containers logs from `/var/log/containers/*`
+Splunk OpenTelemetry Collector for Kubernetes by default read containers logs from `/var/log/pods/*`
+Change is reflected in `source` filed for extracted logs.
+
+### Default `sourcetype` for containers logs
+
+Both Splunk Connect for Kubernetes and Splunk OpenTelemetry Collector for Kubernetes define `sourcetype` for container logs as `kube:container:<container_name>` by default. But, Splunk Connect for Kubernetes explicitly defines the sourcetype of Kubernetes core components as `kube:<container_name>`. They are defined [here](https://github.com/splunk/splunk-connect-for-kubernetes/blob/2cae9b12bbd6545c9ef09b23e619b9783d9ceb38/helm-chart/splunk-connect-for-kubernetes/values.yaml#L330-L408)
+`sourcetype` configuration can be changed by adding `logsCollection.containers.extraOperators` configuration.
+
+### Extracted fields for logs
+
+Splunk OpenTelemetry Collector for Kubernetes follows naming convention for OpenTelemetry for extracted fields. Table below present differences in filed names extracted by Splunk OpenTelemetry Collector for Kubernetes and Splunk Connect for Kubernetes
+
+| Splunk Connect for Kubernetes | Splunk OpenTelemetry Collector for Kubernetes |
+|-------------------------------|-----------------------------------------------|
+| container_id                  | container.id                                  |
+| container_image               | container.image.name and container.image.tag  |
+| container_name                | k8s.container.name                            |
+| cluster_name                  | k8s.cluster.name                              |
+| namespace                     | k8s.namespace.name                            |
+| pod                           | k8s.pod.name                                  |
+| pod_uid                       | k8s.pod_uid                                   |
+| label_app                     | k8s.pod.labels.app                            |

helm-charts/splunk-otel-collector/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: splunk-otel-collector
-version: 0.66.0
-appVersion: 0.66.0
+version: 0.67.0
+appVersion: 0.67.0
 description: Splunk OpenTelemetry Collector for Kubernetes
 icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png
 type: application

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -579,6 +579,8 @@ exporters:
     token: "${SPLUNK_OBSERVABILITY_ACCESS_TOKEN}"
     log_data_enabled: {{ .Values.splunkObservability.logsEnabled }}
     profiling_data_enabled: {{ .Values.splunkObservability.profilingEnabled }}
+    # Temporary disable compression until 0.68.0 to workaround a compression bug
+    disable_compression: true
   {{- end }}
   {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
   {{- include "splunk-otel-collector.splunkPlatformLogsExporter" . | nindent 2 }}

helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl

@@ -152,6 +152,8 @@ exporters:
     profiling_data_enabled: {{ .Values.splunkObservability.profilingEnabled }}
     sending_queue:
       num_consumers: 32
+    # Temporary disable compression until 0.68.0 to workaround a compression bug
+    disable_compression: true
   {{- end }}
 
   {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl

@@ -164,6 +164,8 @@ exporters:
     token: "${SPLUNK_OBSERVABILITY_ACCESS_TOKEN}"
     log_data_enabled: true
     profiling_data_enabled: false
+    # Temporary disable compression until 0.68.0 to workaround a compression bug
+    disable_compression: true
   {{- end }}
 
   {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml

@@ -282,4 +282,40 @@ data:
         </format>
       </match>
     </label>
+
+  {{- /*
+    Default configuration files for CRI container engine.
+    Can be overridden for docker engine by prepare-fluentd-config container.
+  */}}
+  source.containers.parse.conf: |-
+    @type regexp
+    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
+    time_format {{ .Values.fluentd.config.containers.criTimeFormat }}
+  output.filter.conf: |-
+    # = handle cri/containerd multiline format =
+    <filter tail.containers.var.log.containers.**>
+      @type concat
+      key log
+      partial_key partial_flag
+      partial_value P
+      separator ''
+      timeout_label @SPLUNK
+    </filter>
+  output.transform.conf: |-
+    # extract pod_uid and container_name for CRIO runtime
+    # currently CRI does not produce log paths with all the necessary
+    # metadata to parse out pod, namespace, container_name, container_id.
+    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+    <filter tail.containers.var.log.pods.**>
+      @type jq_transformer
+      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
+    </filter>
+    # rename pod_uid and container_name to otel semantics.
+    <filter tail.containers.var.log.pods.**>
+      @type record_transformer
+      <record>
+        k8s.pod.uid ${record["pod_uid"]}
+        k8s.container.name ${record["container_name"]}
+      </record>
+    </filter>
 {{- end }}

helm-charts/splunk-otel-collector/templates/daemonset.yaml

@@ -70,22 +70,19 @@ spec:
       {{- end }}
       {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (not .Values.isWindows) }}
       initContainers:
-        {{- if (eq .Values.logsEngine "fluentd") }}
+        {{- if and (eq .Values.logsEngine "fluentd") (not (eq .Values.distribution "gke/autopilot")) }}
         - name: prepare-fluentd-config
           image: {{ template "splunk-otel-collector.image.fluentd" . }}
           imagePullPolicy: {{ .Values.image.fluentd.pullPolicy }}
           command: [ "sh", "-c"]
           securityContext:
             runAsUser: 0
           args:
-            - if [ -z "${LOG_FORMAT_TYPE}" ]; then
-                if [ "$(ls {{ .Values.fluentd.config.containers.pathDest }}/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
-                  export LOG_FORMAT_TYPE=json;
-                else
-                  export LOG_FORMAT_TYPE=cri;
-                fi;
+            - cp /fluentd/etc/common/* /fluentd/etc/;
+              {{- /* Update fluentd configs if docker engine is configured or discovered from existing log files. */}}
+              if [ "${LOG_FORMAT_TYPE}" == "json" ] || [ "$(ls {{ .Values.fluentd.config.containers.pathDest }}/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
+                  cp /fluentd/etc/json/* /fluentd/etc/;
               fi;
-              cp /fluentd/etc/common/* /fluentd/etc/${LOG_FORMAT_TYPE}/* /fluentd/etc/
           env:
             - name: LOG_FORMAT_TYPE
               value: "{{ .Values.fluentd.config.containers.logFormatType }}"
@@ -99,8 +96,6 @@ spec:
               mountPath: /fluentd/etc/common
             - name: fluentd-config-json
               mountPath: /fluentd/etc/json
-            - name: fluentd-config-cri
-              mountPath: /fluentd/etc/cri
         {{- else }}
         {{- if not (eq .Values.distribution "gke/autopilot") }}
         - name: migrate-checkpoint
@@ -221,6 +216,10 @@ spec:
         {{- else }}
         - /otelcol
         - --config=/conf/relay.yaml
+        {{- if eq .Values.distribution "gke/autopilot" }}
+        {{- /* Temporary no-op argument required to run collector on GKE Autopilot */}}
+        - --metrics-addr=0.0.0.0:8889
+        {{- end }}
         {{- end }}
         {{- if .Values.agent.featureGates }}
         - --feature-gates={{ .Values.agent.featureGates }}
@@ -408,8 +407,10 @@ spec:
         hostPath:
           path: {{ .Values.fluentd.config.journalLogPath | quote }}
       - name: fluentd-config
+      {{- if (not (eq .Values.distribution "gke/autopilot")) }}
         emptyDir: {}
       - name: fluentd-config-common
+      {{- end }}
         configMap:
           name: {{ template "splunk-otel-collector.fullname" . }}-fluentd
       - name: fluentd-config-cri

helm-charts/splunk-otel-collector/templates/network-explorer/cloud-collector-deployment.yaml

@@ -28,10 +28,12 @@ spec:
         app.kubernetes.io/name: {{ include "splunk-otel-collector-cloud-collector.fullname" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
-      {{- with .Values.networkExplorer.imagePullSecrets }}
+{{- if .Values.networkExplorer.imagePullSecrets }}
       imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+{{- range .Values.networkExplorer.imagePullSecrets }}
+        - name: {{ . }}
+{{- end }}
+{{- end }}
       containers:
       - image: "{{ default .Values.networkExplorer.images.repository .Values.networkExplorer.cloudCollector.image.repository }}/{{ .Values.networkExplorer.cloudCollector.image.name }}:{{ default .Values.networkExplorer.images.tag .Values.networkExplorer.cloudCollector.image.tag }}"
         imagePullPolicy: {{ .Values.networkExplorer.images.pullPolicy }}

helm-charts/splunk-otel-collector/templates/network-explorer/k8s-collector-deployment.yaml

@@ -34,10 +34,12 @@ spec:
         app.kubernetes.io/name: {{ include "splunk-otel-collector-k8s-collector.fullname" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
-      {{- with .Values.networkExplorer.imagePullSecrets }}
+{{- if .Values.networkExplorer.imagePullSecrets }}
       imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+{{- range .Values.networkExplorer.imagePullSecrets }}
+        - name: {{ . }}
+{{- end }}
+{{- end }}
       containers:
       - image: "{{ default .Values.networkExplorer.images.repository .Values.networkExplorer.k8sCollector.watcher.image.repository }}/{{ .Values.networkExplorer.k8sCollector.watcher.image.name }}:{{ default .Values.networkExplorer.images.tag .Values.networkExplorer.k8sCollector.watcher.image.tag }}"
         imagePullPolicy: {{ .Values.networkExplorer.images.pullPolicy }}

helm-charts/splunk-otel-collector/templates/network-explorer/kernel-collector-daemonset.yaml

@@ -28,10 +28,12 @@ spec:
         app.kubernetes.io/name: {{ include "splunk-otel-collector-kernel-collector.fullname" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
-      {{- with .Values.networkExplorer.imagePullSecrets }}
+{{- if .Values.networkExplorer.imagePullSecrets }}
       imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+{{- range .Values.networkExplorer.imagePullSecrets }}
+        - name: {{ . }}
+{{- end }}
+{{- end }}
       containers:
         - name: kernel-collector
           image: "{{ default .Values.networkExplorer.images.repository .Values.networkExplorer.kernelCollector.image.repository }}/{{ .Values.networkExplorer.kernelCollector.image.name }}:{{ default .Values.networkExplorer.images.tag .Values.networkExplorer.kernelCollector.image.tag }}"

helm-charts/splunk-otel-collector/templates/network-explorer/reducer-deployment.yaml

@@ -26,10 +26,12 @@ spec:
         app.kubernetes.io/name: {{ include "splunk-otel-collector-reducer.fullname" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
-      {{- with .Values.networkExplorer.imagePullSecrets }}
+{{- if .Values.networkExplorer.imagePullSecrets }}
       imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+{{- range .Values.networkExplorer.imagePullSecrets }}
+        - name: {{ . }}
+{{- end }}
+{{- end }}
       containers:
         - name: reducer
           image: "{{ default .Values.networkExplorer.images.repository .Values.networkExplorer.reducer.image.repository }}/{{ .Values.networkExplorer.reducer.image.name }}:{{ default .Values.networkExplorer.images.tag .Values.networkExplorer.reducer.image.tag }}"

rendered/manifests/agent-only/clusterRole.yaml

@@ -6,12 +6,12 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app.kubernetes.io/name: splunk-otel-collector
-    helm.sh/chart: splunk-otel-collector-0.66.0
+    helm.sh/chart: splunk-otel-collector-0.67.0
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: default
-    app.kubernetes.io/version: "0.66.0"
+    app.kubernetes.io/version: "0.67.0"
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.66.0
+    chart: splunk-otel-collector-0.67.0
     release: default
     heritage: Helm
 rules:

rendered/manifests/agent-only/clusterRoleBinding.yaml

@@ -6,12 +6,12 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app.kubernetes.io/name: splunk-otel-collector
-    helm.sh/chart: splunk-otel-collector-0.66.0
+    helm.sh/chart: splunk-otel-collector-0.67.0
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: default
-    app.kubernetes.io/version: "0.66.0"
+    app.kubernetes.io/version: "0.67.0"
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.66.0
+    chart: splunk-otel-collector-0.67.0
     release: default
     heritage: Helm
 roleRef: