Update fluent-operator from v2.0.1 to v2.1.0 (#327)

* Update fluent-operator from v2.0.1 to v2.1.0 Signed-off-by: Tomáš Novák <[email protected]> * Revert chart.yaml and bump up only versions Signed-off-by: Tomáš Novák <[email protected]> * Fix fluent-operator lint errors Signed-off-by: Tomáš Novák <[email protected]> --------- Signed-off-by: Tomáš Novák <[email protected]>
fluent · Mar 24, 2023 · 569f43e · 569f43e
1 parent 0d55545
commit 569f43e
Show file tree

Hide file tree

Showing 17 changed files with 950 additions and 157 deletions.
diff --git a/charts/fluent-operator/Chart.yaml b/charts/fluent-operator/Chart.yaml
@@ -6,8 +6,8 @@ keywords:
   - fluent-bit
   - fluentd
   - operator
-version: 2.0.2
-appVersion: 2.0.1
+version: 2.1.0
+appVersion: 2.1.0
 icon: https://raw.githubusercontent.com/fluent/fluent-operator/master/docs/images/fluent-operator-icon.svg
 home: https://www.fluentd.org/
 sources:

diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml
@@ -235,6 +235,36 @@ spec:
                     description: Wait time on exit
                     format: int64
                     type: integer
+                  hcErrorsCount:
+                    description: 'the error count to meet the unhealthy requirement,
+                      this is a sum for all output plugins in a defined HC_Period,
+                      example for output error: [2022/02/16 10:44:10] [ warn] [engine]
+                      failed to flush chunk ''1-1645008245.491540684.flb'', retry
+                      in 7 seconds: task_id=0, input=forward.1 > output=cloudwatch_logs.3
+                      (out_id=3)'
+                    format: int64
+                    minimum: 1
+                    type: integer
+                  hcPeriod:
+                    description: The time period by second to count the error and
+                      retry failure data point
+                    format: int64
+                    minimum: 1
+                    type: integer
+                  hcRetryFailureCount:
+                    description: 'the retry failure count to meet the unhealthy requirement,
+                      this is a sum for all output plugins in a defined HC_Period,
+                      example for retry failure: [2022/02/16 20:11:36] [ warn] [engine]
+                      chunk ''1-1645042288.260516436.flb'' cannot be retried: task_id=0,
+                      input=tcp.3 > output=cloudwatch_logs.1'
+                    format: int64
+                    minimum: 1
+                    type: integer
+                  healthCheck:
+                    description: 'enable Health check feature at http://127.0.0.1:2020/api/v1/health
+                      Note: Enabling this will not automatically configure kubernetes
+                      to use fluentbit''s healthcheck endpoint'
+                    type: boolean
                   httpListen:
                     description: Address to listen
                     pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$

diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml
diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_collectors.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_collectors.yaml
diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_fluentbits.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_fluentbits.yaml
diff --git a/charts/fluent-operator/crds/fluentd.fluent.io_clusterfilters.yaml b/charts/fluent-operator/crds/fluentd.fluent.io_clusterfilters.yaml
@@ -43,6 +43,14 @@ spec:
                   description: Filter defines all available filter plugins and their
                     parameters.
                   properties:
+                    customPlugin:
+                      description: Custom plugin type
+                      properties:
+                        config:
+                          type: string
+                      required:
+                      - config
+                      type: object
                     grep:
                       description: The filter_grep filter plugin
                       properties:

diff --git a/charts/fluent-operator/crds/fluentd.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/crds/fluentd.fluent.io_clusteroutputs.yaml
@@ -110,8 +110,8 @@ spec:
                           - immediate
                           type: string
                         flushThreadCount:
-                          description: The sleep interval (seconds) for threads to
-                            wait for the next flush try(when no chunks are waiting)
+                          description: The number of threads to flush/write chunks
+                            in parallel
                           pattern: ^\d+$
                           type: string
                         id:
@@ -246,9 +246,16 @@ spec:
                           description: If true, uses UTC.
                           type: boolean
                       required:
-                      - path
                       - type
                       type: object
+                    customPlugin:
+                      description: Custom plugin type
+                      properties:
+                        config:
+                          type: string
+                      required:
+                      - config
+                      type: object
                     elasticsearch:
                       description: out_es plugin
                       properties:

diff --git a/charts/fluent-operator/crds/fluentd.fluent.io_filters.yaml b/charts/fluent-operator/crds/fluentd.fluent.io_filters.yaml
@@ -43,6 +43,14 @@ spec:
                   description: Filter defines all available filter plugins and their
                     parameters.
                   properties:
+                    customPlugin:
+                      description: Custom plugin type
+                      properties:
+                        config:
+                          type: string
+                      required:
+                      - config
+                      type: object
                     grep:
                       description: The filter_grep filter plugin
                       properties:

diff --git a/charts/fluent-operator/crds/fluentd.fluent.io_fluentds.yaml b/charts/fluent-operator/crds/fluentd.fluent.io_fluentds.yaml
@@ -939,9 +939,12 @@ spec:
                               * An existing PVC (PersistentVolumeClaim) If the provisioner
                               or an external controller can support the specified
                               data source, it will create a new volume based on the
-                              contents of the specified data source. If the AnyVolumeDataSource
-                              feature gate is enabled, this field will always have
-                              the same contents as the DataSourceRef field.'
+                              contents of the specified data source. When the AnyVolumeDataSource
+                              feature gate is enabled, dataSource contents will be
+                              copied to dataSourceRef, and dataSourceRef contents
+                              will be copied to dataSource when dataSourceRef.namespace
+                              is not specified. If the namespace is specified, then
+                              dataSourceRef will not be copied to dataSource.'
                             properties:
                               apiGroup:
                                 description: APIGroup is the group for the resource
@@ -962,25 +965,32 @@ spec:
                           dataSourceRef:
                             description: 'dataSourceRef specifies the object from
                               which to populate the volume with data, if a non-empty
-                              volume is desired. This may be any local object from
-                              a non-empty API group (non core object) or a PersistentVolumeClaim
+                              volume is desired. This may be any object from a non-empty
+                              API group (non core object) or a PersistentVolumeClaim
                               object. When this field is specified, volume binding
                               will only succeed if the type of the specified object
                               matches some installed volume populator or dynamic provisioner.
-                              This field will replace the functionality of the DataSource
+                              This field will replace the functionality of the dataSource
                               field and as such if both fields are non-empty, they
                               must have the same value. For backwards compatibility,
-                              both fields (DataSource and DataSourceRef) will be set
-                              to the same value automatically if one of them is empty
-                              and the other is non-empty. There are two important
-                              differences between DataSource and DataSourceRef: *
-                              While DataSource only allows two specific types of objects,
-                              DataSourceRef   allows any non-core object, as well
-                              as PersistentVolumeClaim objects. * While DataSource
-                              ignores disallowed values (dropping them), DataSourceRef   preserves
-                              all values, and generates an error if a disallowed value
-                              is   specified. (Beta) Using this field requires the
-                              AnyVolumeDataSource feature gate to be enabled.'
+                              when namespace isn''t specified in dataSourceRef, both
+                              fields (dataSource and dataSourceRef) will be set to
+                              the same value automatically if one of them is empty
+                              and the other is non-empty. When namespace is specified
+                              in dataSourceRef, dataSource isn''t set to the same
+                              value and must be empty. There are three important differences
+                              between dataSource and dataSourceRef: * While dataSource
+                              only allows two specific types of objects, dataSourceRef   allows
+                              any non-core object, as well as PersistentVolumeClaim
+                              objects. * While dataSource ignores disallowed values
+                              (dropping them), dataSourceRef   preserves all values,
+                              and generates an error if a disallowed value is   specified.
+                              * While dataSource only allows local objects, dataSourceRef
+                              allows objects   in any namespaces. (Beta) Using this
+                              field requires the AnyVolumeDataSource feature gate
+                              to be enabled. (Alpha) Using the namespace field of
+                              dataSourceRef requires the CrossNamespaceVolumeDataSource
+                              feature gate to be enabled.'
                             properties:
                               apiGroup:
                                 description: APIGroup is the group for the resource
@@ -994,6 +1004,16 @@ spec:
                               name:
                                 description: Name is the name of resource being referenced
                                 type: string
+                              namespace:
+                                description: Namespace is the namespace of resource
+                                  being referenced Note that when a namespace is specified,
+                                  a gateway.networking.k8s.io/ReferenceGrant object
+                                  is required in the referent namespace to allow that
+                                  namespace's owner to accept the reference. See the
+                                  ReferenceGrant documentation for details. (Alpha)
+                                  This field requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.
+                                type: string
                             required:
                             - kind
                             - name
@@ -1006,6 +1026,29 @@ spec:
                               must still be higher than capacity recorded in the status
                               field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                             properties:
+                              claims:
+                                description: "Claims lists the names of resources,
+                                  defined in spec.resourceClaims, that are used by
+                                  this container. \n This is an alpha field and requires
+                                  enabling the DynamicResourceAllocation feature gate.
+                                  \n This field is immutable."
+                                items:
+                                  description: ResourceClaim references one entry
+                                    in PodSpec.ResourceClaims.
+                                  properties:
+                                    name:
+                                      description: Name must match the name of one
+                                        entry in pod.spec.resourceClaims of the Pod
+                                        where this field is used. It makes that resource
+                                        available inside a container.
+                                      type: string
+                                  required:
+                                  - name
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - name
+                                x-kubernetes-list-type: map
                               limits:
                                 additionalProperties:
                                   anyOf:
@@ -1752,6 +1795,26 @@ spec:
               resources:
                 description: Compute Resources required by container.
                 properties:
+                  claims:
+                    description: "Claims lists the names of resources, defined in
+                      spec.resourceClaims, that are used by this container. \n This
+                      is an alpha field and requires enabling the DynamicResourceAllocation
+                      feature gate. \n This field is immutable."
+                    items:
+                      description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                      properties:
+                        name:
+                          description: Name must match the name of one entry in pod.spec.resourceClaims
+                            of the Pod where this field is used. It makes that resource
+                            available inside a container.
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                    x-kubernetes-list-map-keys:
+                    - name
+                    x-kubernetes-list-type: map
                   limits:
                     additionalProperties:
                       anyOf:

diff --git a/charts/fluent-operator/crds/fluentd.fluent.io_outputs.yaml b/charts/fluent-operator/crds/fluentd.fluent.io_outputs.yaml
@@ -110,8 +110,8 @@ spec:
                           - immediate
                           type: string
                         flushThreadCount:
-                          description: The sleep interval (seconds) for threads to
-                            wait for the next flush try(when no chunks are waiting)
+                          description: The number of threads to flush/write chunks
+                            in parallel
                           pattern: ^\d+$
                           type: string
                         id:
@@ -246,9 +246,16 @@ spec:
                           description: If true, uses UTC.
                           type: boolean
                       required:
-                      - path
                       - type
                       type: object
+                    customPlugin:
+                      description: Custom plugin type
+                      properties:
+                        config:
+                          type: string
+                      required:
+                      - config
+                      type: object
                     elasticsearch:
                       description: out_es plugin
                       properties:

diff --git a/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml b/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml
@@ -47,11 +47,17 @@ rules:
     resources:
       - collectors
       - fluentbits
+      - fluentbits/finalizers
       - clusterfluentbitconfigs
+      - clusterfluentbitconfigs/finalizers
       - clusterfilters
+      - clusterfilters/finalizers
       - clusterinputs
+      - clusterinputs/finalizers
       - clusteroutputs
+      - clusteroutputs/finalizers
       - clusterparsers
+      - clusterparsers/finalizers
     verbs:
       - create
       - delete

diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml
@@ -1,4 +1,5 @@
 {{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.input.systemd.enable -}}
 apiVersion: fluentbit.fluent.io/v1alpha2
 kind: ClusterInput
 metadata:
@@ -9,10 +10,13 @@ metadata:
 spec:
   systemd:
     tag: service.*
-    path: /var/log/journal
+    path: {{ .Values.fluentbit.input.systemd.path }}
     db: /fluent-bit/tail/systemd.db
     dbSync: Normal
     systemdFilter:
       - _SYSTEMD_UNIT={{ .Values.containerRuntime }}.service
+      {{- if .Values.fluentbit.input.systemd.includeKubelet }}
       - _SYSTEMD_UNIT=kubelet.service
+      {{- end }}
+{{- end }}
 {{- end }}
diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml
@@ -1,4 +1,5 @@
 {{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.input.tail.enable -}}
 apiVersion: fluentbit.fluent.io/v1alpha2
 kind: ClusterInput
 metadata:
@@ -9,17 +10,18 @@ metadata:
 spec:
   tail:
     tag: kube.*
-    path: /var/log/containers/*.log
+    path: {{ .Values.fluentbit.input.tail.path }}
     {{- if eq .Values.containerRuntime "docker" }}
     parser: docker
     {{- else if eq .Values.containerRuntime "containerd" }}
     parser: cri
     {{- else if eq .Values.containerRuntime "crio" }}
     parser: cri
     {{- end }}
-    refreshIntervalSeconds: 10
+    refreshIntervalSeconds: {{ .Values.fluentbit.input.tail.refreshIntervalSeconds }}
     memBufLimit: {{ .Values.fluentbit.input.tail.memBufLimit }}
-    skipLongLines: true
+    skipLongLines: {{ .Values.fluentbit.input.tail.skipLongLines }}
     db: /fluent-bit/tail/pos.db
     dbSync: Normal
 {{- end }}
+{{- end }}
diff --git a/charts/fluent-operator/templates/fluentbit-output-opensearch.yaml b/charts/fluent-operator/templates/fluentbit-output-opensearch.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.output.opensearch -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterOutput
+metadata:
+  name: fluentd-output-opensearch
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  matchRegex: (?:kube|service)\.(.*)
+  opensearch:
+{{ toYaml .Values.fluentbit.output.opensearch | indent 4}}
+{{- end }}
+{{- end }}
diff --git a/charts/fluent-operator/templates/fluentbit-output-opentelemetry.yaml b/charts/fluent-operator/templates/fluentbit-output-opentelemetry.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.output.opentelemetry -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterOutput
+metadata:
+  name: fluentd-output-opentelemetry
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  matchRegex: (?:kube|service)\.(.*)
+  opentelemetry:
+{{ toYaml .Values.fluentbit.output.opentelemetry | indent 4}}
+{{- end }}
+{{- end }}
diff --git a/charts/fluent-operator/templates/serviceaccount.yaml b/charts/fluent-operator/templates/serviceaccount.yaml
@@ -4,4 +4,4 @@ metadata:
   name: fluent-operator
   labels:
     app.kubernetes.io/component: operator
-    app.kubernetes.io/name: fluent-operator
+    app.kubernetes.io/name: fluent-operator