From 3a87ec9498ae01c04a817b9721f8d7ab14e8bedf Mon Sep 17 00:00:00 2001 From: Ashish Nepal Date: Thu, 23 May 2024 12:45:12 +0100 Subject: [PATCH] [fluent-operator] upgrade fluent-operator from v2.7.0 to v2.8.0 Signed-off-by: Ashish Nepal --- charts/fluent-operator/Chart.yaml | 8 +- .../charts/fluent-bit-crds/Chart.yaml | 4 +- .../fluentbit.fluent.io_clusterfilters.yaml | 43 +++- ...bit.fluent.io_clusterfluentbitconfigs.yaml | 57 +++++ ...bit.fluent.io_clustermultilineparsers.yaml | 83 +++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 73 ++++++ .../crds/fluentbit.fluent.io_filters.yaml | 43 +++- .../fluentbit.fluent.io_fluentbitconfigs.yaml | 229 ++++++++++++++++++ .../fluentbit.fluent.io_multilineparsers.yaml | 83 +++++++ .../crds/fluentbit.fluent.io_outputs.yaml | 73 ++++++ .../charts/fluentd-crds/Chart.yaml | 4 +- .../fluentd.fluent.io_clusteroutputs.yaml | 15 ++ .../crds/fluentd.fluent.io_outputs.yaml | 15 ++ .../fluent-operator-clusterRole.yaml | 7 +- .../templates/fluent-operator-deployment.yaml | 8 +- .../fluentbit-clusterfilter-multiline.yaml | 20 ++ .../fluentbit-clusterinput-tail.yaml | 6 + .../templates/fluentbit-fluentBit.yaml | 7 +- ...uentbit-multilineParser-javaMultiline.yaml | 30 +++ .../fluentbit-output-elasticsearch.yaml | 9 +- .../templates/fluentbit-output-loki.yaml | 3 +- ...t-output-prometheus-remote-write-edge.yaml | 4 +- .../fluentbit-output-stackdriver.yaml | 18 ++ .../templates/fluentbit-servicemonitor.yaml | 20 ++ .../fluentbitconfig-fluentBitConfig.yaml | 10 +- charts/fluent-operator/values.yaml | 65 +++-- 26 files changed, 900 insertions(+), 37 deletions(-) create mode 100644 charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml create mode 100644 charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml create mode 100644 charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml create mode 100644 charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml create mode 100644 charts/fluent-operator/templates/fluentbit-output-stackdriver.yaml create mode 100644 charts/fluent-operator/templates/fluentbit-servicemonitor.yaml diff --git a/charts/fluent-operator/Chart.yaml b/charts/fluent-operator/Chart.yaml index 7b98f3e3..8fcd82dd 100644 --- a/charts/fluent-operator/Chart.yaml +++ b/charts/fluent-operator/Chart.yaml @@ -6,8 +6,8 @@ keywords: - fluent-bit - fluentd - operator -version: 2.7.0 -appVersion: 2.7.0 +version: 2.8.0 +appVersion: 2.8.0 icon: https://raw.githubusercontent.com/fluent/fluent-operator/master/docs/images/fluent-operator-icon.svg home: https://www.fluentd.org/ sources: @@ -18,9 +18,9 @@ maintainers: dependencies: - name: fluent-bit-crds repository: "file://charts/fluent-bit-crds" - version: 2.7.0 + version: 2.8.0 condition: fluentbit.crdsEnable - name: fluentd-crds repository: "file://charts/fluentd-crds" - version: 2.7.0 + version: 2.8.0 condition: fluentd.crdsEnable diff --git a/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml b/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml index 56932d10..dccd4b66 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml @@ -5,8 +5,8 @@ keywords: - logging - fluent-bit - operator -version: 2.7.0 -appVersion: "2.7.0" +version: 2.8.0 +appVersion: "2.8.0" sources: - https://github.com/fluent/fluent-operator maintainers: diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index 38064b4a..e327e322 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -289,6 +289,10 @@ spec: do filtering. It's assumed that the function is declared inside the Script defined above. type: string + code: + description: Inline LUA code instead of loading from a path + via script. + type: string protectedMode: description: If enabled, Lua script will be executed in protected mode. It prevents to crash when invalid Lua @@ -341,7 +345,6 @@ spec: type: array required: - call - - script type: object modify: description: Modify defines Modify Filter configuration. @@ -486,12 +489,46 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory in MB the + emitter can consume if the outputs provide backpressure. + The default for this limit is 10M. The pipeline will pause + once the buffer exceeds the value of this setting. For + example, if the value is set to 10MB then the pipeline + will pause if the buffer exceeds 10M. The pipeline will + remain paused until the output drains the buffer below + the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -640,6 +677,8 @@ spec: alias: description: Alias for the plugin type: string + emitterMemBufLimit: + type: string emitterName: description: When the filter emits a record under the new Tag, there is an internal emitter plugin that takes care @@ -647,6 +686,8 @@ spec: component of the pipeline, you can use this property to configure an optional name for it. type: string + emitterStorageType: + type: string retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 31d611b9..ebfbf48e 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -128,6 +128,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic namespace: description: If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is @@ -230,6 +275,13 @@ spec: daemon: description: If true go to background on start type: boolean + emitterMemBufLimit: + type: string + emitterName: + description: Per-namespace re-emitter configuration + type: string + emitterStorageType: + type: string flushSeconds: description: Interval to flush output format: int64 @@ -297,6 +349,11 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array storage: description: Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml new file mode 100644 index 00000000..71eefcd5 --- /dev/null +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: clustermultilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterMultilineParser + listKind: ClusterMultilineParserList + plural: clustermultilineparsers + shortNames: + - cfbmp + singular: clustermultilineparser + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterMultilineParser is the Schema for the cluster-level multiline + parser API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index a05a5305..d32f7cc5 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -661,6 +661,9 @@ spec: type: description: Type name type: string + writeOperation: + description: Operation to use to write in bulk requests. + type: string type: object file: description: File defines File Output configuration. @@ -2480,6 +2483,9 @@ spec: a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean + Profile: + description: Option to specify an AWS Profile for credentials. + type: string Region: description: The AWS region of your S3 bucket type: string @@ -2542,6 +2548,73 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object required: - Bucket - Region diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index f84c8b4d..1a0145ba 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -289,6 +289,10 @@ spec: do filtering. It's assumed that the function is declared inside the Script defined above. type: string + code: + description: Inline LUA code instead of loading from a path + via script. + type: string protectedMode: description: If enabled, Lua script will be executed in protected mode. It prevents to crash when invalid Lua @@ -341,7 +345,6 @@ spec: type: array required: - call - - script type: object modify: description: Modify defines Modify Filter configuration. @@ -486,12 +489,46 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory in MB the + emitter can consume if the outputs provide backpressure. + The default for this limit is 10M. The pipeline will pause + once the buffer exceeds the value of this setting. For + example, if the value is set to 10MB then the pipeline + will pause if the buffer exceeds 10M. The pipeline will + remain paused until the output drains the buffer below + the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -640,6 +677,8 @@ spec: alias: description: Alias for the plugin type: string + emitterMemBufLimit: + type: string emitterName: description: When the filter emits a record under the new Tag, there is an internal emitter plugin that takes care @@ -647,6 +686,8 @@ spec: component of the pipeline, you can use this property to configure an optional name for it. type: string + emitterStorageType: + type: string retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml index 73a7e7ed..a81a2c4f 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml @@ -37,6 +37,51 @@ spec: spec: description: NamespacedFluentBitCfgSpec defines the desired state of FluentBit properties: + clusterMultilineParserSelector: + description: Select cluster level multiline parser config + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic clusterParserSelector: description: Select cluster level parser config properties: @@ -127,6 +172,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic outputSelector: description: Select output plugins properties: @@ -217,6 +307,145 @@ spec: type: object type: object x-kubernetes-map-type: atomic + service: + description: Service defines the global behaviour of the Fluent Bit + engine. + properties: + daemon: + description: If true go to background on start + type: boolean + emitterMemBufLimit: + type: string + emitterName: + description: Per-namespace re-emitter configuration + type: string + emitterStorageType: + type: string + flushSeconds: + description: Interval to flush output + format: int64 + type: integer + graceSeconds: + description: Wait time on exit + format: int64 + type: integer + hcErrorsCount: + description: 'the error count to meet the unhealthy requirement, + this is a sum for all output plugins in a defined HC_Period, + example for output error: [2022/02/16 10:44:10] [ warn] [engine] + failed to flush chunk ''1-1645008245.491540684.flb'', retry + in 7 seconds: task_id=0, input=forward.1 > output=cloudwatch_logs.3 + (out_id=3)' + format: int64 + minimum: 1 + type: integer + hcPeriod: + description: The time period by second to count the error and + retry failure data point + format: int64 + minimum: 1 + type: integer + hcRetryFailureCount: + description: 'the retry failure count to meet the unhealthy requirement, + this is a sum for all output plugins in a defined HC_Period, + example for retry failure: [2022/02/16 20:11:36] [ warn] [engine] + chunk ''1-1645042288.260516436.flb'' cannot be retried: task_id=0, + input=tcp.3 > output=cloudwatch_logs.1' + format: int64 + minimum: 1 + type: integer + healthCheck: + description: 'enable Health check feature at http://127.0.0.1:2020/api/v1/health + Note: Enabling this will not automatically configure kubernetes + to use fluentbit''s healthcheck endpoint' + type: boolean + httpListen: + description: Address to listen + pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + type: string + httpPort: + description: Port to listen + format: int32 + maximum: 65535 + minimum: 1 + type: integer + httpServer: + description: If true enable statistics HTTP server + type: boolean + logFile: + description: File to log diagnostic output + type: string + logLevel: + description: Diagnostic level (error/warning/info/debug/trace) + enum: + - "off" + - error + - warning + - info + - debug + - trace + type: string + parsersFile: + description: Optional 'parsers' config file (can be multiple) + type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array + storage: + description: Configure a global environment for the storage layer + in Service. It is recommended to configure the volume and volumeMount + separately for this storage. The hostPath type should be used + for that Volume in Fluentbit daemon set. + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + format: int64 + type: integer + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object + type: object type: object type: object served: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml new file mode 100644 index 00000000..d3081e84 --- /dev/null +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: multilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: MultilineParser + listKind: MultilineParserList + plural: multilineparsers + shortNames: + - fbmp + singular: multilineparser + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: MultilineParser is the Schema of namespace-level multiline parser + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 383ce708..3949e9bc 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -661,6 +661,9 @@ spec: type: description: Type name type: string + writeOperation: + description: Operation to use to write in bulk requests. + type: string type: object file: description: File defines File Output configuration. @@ -2480,6 +2483,9 @@ spec: a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean + Profile: + description: Option to specify an AWS Profile for credentials. + type: string Region: description: The AWS region of your S3 bucket type: string @@ -2542,6 +2548,73 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object required: - Bucket - Region diff --git a/charts/fluent-operator/charts/fluentd-crds/Chart.yaml b/charts/fluent-operator/charts/fluentd-crds/Chart.yaml index 36082fd4..cc76b7a1 100644 --- a/charts/fluent-operator/charts/fluentd-crds/Chart.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/Chart.yaml @@ -5,8 +5,8 @@ keywords: - logging - fluentd - operator -version: 2.7.0 -appVersion: "2.7.0" +version: 2.8.0 +appVersion: "2.8.0" sources: - https://github.com/fluent/fluent-operator maintainers: diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml index 54e9157e..cc69e61f 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml @@ -1793,6 +1793,18 @@ spec: s3Region: description: The Amazon S3 region name type: string + sseCustomerAlgorithm: + description: The AWS KMS enctyption algorithm. + type: string + sseCustomerKey: + description: The AWS KMS key. + type: string + sseCustomerKeyMd5: + description: The AWS KMS key MD5. + type: string + ssekmsKeyId: + description: The AWS KMS key ID. + type: string sslVerifyPeer: description: Verify the SSL certificate of the endpoint. type: boolean @@ -1807,6 +1819,9 @@ spec: timeSliceFormat: description: This timestamp is added to each file name type: string + useServerSideEncryption: + description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + type: string type: object stdout: description: out_stdout plugin diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml index 4f582558..bacfb15f 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml @@ -1793,6 +1793,18 @@ spec: s3Region: description: The Amazon S3 region name type: string + sseCustomerAlgorithm: + description: The AWS KMS enctyption algorithm. + type: string + sseCustomerKey: + description: The AWS KMS key. + type: string + sseCustomerKeyMd5: + description: The AWS KMS key MD5. + type: string + ssekmsKeyId: + description: The AWS KMS key ID. + type: string sslVerifyPeer: description: Verify the SSL certificate of the endpoint. type: boolean @@ -1807,6 +1819,9 @@ spec: timeSliceFormat: description: This timestamp is added to each file name type: string + useServerSideEncryption: + description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + type: string type: object stdout: description: out_stdout plugin diff --git a/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml b/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml index 7fc48bc4..7418404a 100644 --- a/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml +++ b/charts/fluent-operator/templates/fluent-operator-clusterRole.yaml @@ -57,11 +57,16 @@ rules: - clusteroutputs - clusteroutputs/finalizers - clusterparsers + - clusterparsers/finalizers - fluentbitconfigs + - fluentbitconfigs/finalizers + - multilineparsers + - multilineparsers/finalizers + - clustermultilineparsers + - clustermultilineparsers/finalizers - filters - outputs - parsers - - clusterparsers/finalizers verbs: - create - delete diff --git a/charts/fluent-operator/templates/fluent-operator-deployment.yaml b/charts/fluent-operator/templates/fluent-operator-deployment.yaml index 68eaafbf..0cb8a687 100644 --- a/charts/fluent-operator/templates/fluent-operator-deployment.yaml +++ b/charts/fluent-operator/templates/fluent-operator-deployment.yaml @@ -98,9 +98,11 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - {{- with .Values.operator.disableComponentControllers }} - args: ["--disable-component-controllers","{{ . }}"] - {{- end }} + args: + {{- with .Values.operator.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + - --disable-component-controllers={{ .Values.operator.disableComponentControllers | quote }} volumeMounts: - name: env mountPath: /fluent-operator diff --git a/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml b/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml new file mode 100644 index 00000000..58144969 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml @@ -0,0 +1,20 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.filter.multiline.enable -}} +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterFilter +metadata: + name: multiline + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + match: kube.* + filters: + - multiline: + keyContent: {{ .Values.fluentbit.filter.multiline.keyContent | quote }} + emitterMemBufLimit: {{ .Values.fluentbit.filter.multiline.emitterMemBufLimit }} + parser: "{{- join "," .Values.fluentbit.filter.multiline.parsers -}}" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml index 76fd0738..8f703224 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml @@ -20,8 +20,14 @@ spec: {{- else if eq .Values.containerRuntime "crio" }} parser: cri {{- end }} + {{- if .Values.fluentbit.input.tail.multilineParser }} + multilineParser: {{ .Values.fluentbit.input.tail.multilineParser | quote }} + {{- end }} refreshIntervalSeconds: {{ .Values.fluentbit.input.tail.refreshIntervalSeconds }} memBufLimit: {{ .Values.fluentbit.input.tail.memBufLimit }} + {{- if .Values.fluentbit.input.tail.bufferMaxSize }} + bufferMaxSize: {{ .Values.fluentbit.input.tail.bufferMaxSize }} + {{- end }} skipLongLines: {{ .Values.fluentbit.input.tail.skipLongLines }} db: /fluent-bit/tail/pos.db dbSync: Normal diff --git a/charts/fluent-operator/templates/fluentbit-fluentBit.yaml b/charts/fluent-operator/templates/fluentbit-fluentBit.yaml index 06377220..ec610d00 100644 --- a/charts/fluent-operator/templates/fluentbit-fluentBit.yaml +++ b/charts/fluent-operator/templates/fluentbit-fluentBit.yaml @@ -12,6 +12,8 @@ spec: {{- end }} {{- if .Values.fluentbit.dnsPolicy }} dnsPolicy: {{ .Values.fluentbit.dnsPolicy }} +{{- else if .Values.fluentbit.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet {{- end }} image: {{ .Values.fluentbit.image.repository }}:{{ .Values.fluentbit.image.tag }} {{- if .Values.fluentbit.imagePullSecrets }} @@ -87,5 +89,8 @@ spec: serviceAccountAnnotations: {{ toYaml .Values.fluentbit.serviceAccountAnnotations | indent 4 }} {{- end }} +{{- if .Values.fluentbit.disableLogVolumes }} + disableLogVolumes: {{ .Values.fluentbit.disableLogVolumes }} +{{- end }} +{{- end }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml b/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml new file mode 100644 index 00000000..e8703992 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml @@ -0,0 +1,30 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.parsers.javaMultiline.enable -}} +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterMultilineParser +metadata: + name: java-multiline + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + type: "regex" + flushTimeout: 1000 + keyContent: "log" + rules: + - start: "start_state" + regex: '/^\[?(\d+\-\d+\-\d+ \d+\:\d+\:\d+(\.\d+)?)\]? /' + next: "cont" + - start: "cont" + regex: '/^com\..*/' + next: "cont" + - start: "cont" + regex: '/^\s+.*/' + next: "cont" + - start: "cont" + regex: '/^Caused.*$/' + next: "cont" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/fluent-operator/templates/fluentbit-output-elasticsearch.yaml b/charts/fluent-operator/templates/fluentbit-output-elasticsearch.yaml index 256a7982..340253d2 100644 --- a/charts/fluent-operator/templates/fluentbit-output-elasticsearch.yaml +++ b/charts/fluent-operator/templates/fluentbit-output-elasticsearch.yaml @@ -34,8 +34,15 @@ spec: {{ toYaml .Values.fluentbit.output.es.httpPassword | indent 6 }} {{- end }} logstashFormat: {{ .Values.fluentbit.output.es.logstashFormat | default true }} - logstashPrefix: {{ .Values.fluentbit.output.es.logstashPrefix | default "ks-logstash-log" | quote }} +{{- if .Values.fluentbit.output.es.logstashPrefix }} + logstashPrefix: {{ .Values.fluentbit.output.es.logstashPrefix | quote }} +{{- end }} +{{- if .Values.fluentbit.output.es.logstashPrefixKey }} + logstashPrefixKey: {{ .Values.fluentbit.output.es.logstashPrefixKey | quote }} +{{- end }} replaceDots: {{ .Values.fluentbit.output.es.replaceDots | default false }} + writeOperation: {{ .Values.fluentbit.output.es.writeOperation | default "create" | quote }} + traceError: {{ .Values.fluentbit.output.es.traceError | default false }} generateID: true timeKey: "@timestamp" {{- if .Values.fluentbit.output.es.enableTLS }} diff --git a/charts/fluent-operator/templates/fluentbit-output-loki.yaml b/charts/fluent-operator/templates/fluentbit-output-loki.yaml index 2622a3b3..4d23d6c9 100644 --- a/charts/fluent-operator/templates/fluentbit-output-loki.yaml +++ b/charts/fluent-operator/templates/fluentbit-output-loki.yaml @@ -108,7 +108,8 @@ spec: {{ .removeKeys | toYaml | indent 6 }} {{ end -}} - {{ if .tls -}} + {{ if .tls }} + tls: {{ .tls | toYaml | indent 6 }} {{ end -}} diff --git a/charts/fluent-operator/templates/fluentbit-output-prometheus-remote-write-edge.yaml b/charts/fluent-operator/templates/fluentbit-output-prometheus-remote-write-edge.yaml index 9c98a1a9..86a3dda7 100644 --- a/charts/fluent-operator/templates/fluentbit-output-prometheus-remote-write-edge.yaml +++ b/charts/fluent-operator/templates/fluentbit-output-prometheus-remote-write-edge.yaml @@ -9,8 +9,8 @@ metadata: spec: match: kubeedge.metrics.* prometheusRemoteWrite: - host: {{ .Values.fluentbit.edge.prometheusRemoteWrite.host }} - port: {{ .Values.fluentbit.edge.prometheusRemoteWrite.port }} + host: {{ .Values.fluentbit.kubeedge.prometheusRemoteWrite.host }} + port: {{ .Values.fluentbit.kubeedge.prometheusRemoteWrite.port }} uri: /api/v1/write addLabels : app : fluentbit diff --git a/charts/fluent-operator/templates/fluentbit-output-stackdriver.yaml b/charts/fluent-operator/templates/fluentbit-output-stackdriver.yaml new file mode 100644 index 00000000..6c271d77 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-output-stackdriver.yaml @@ -0,0 +1,18 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.output.stackdriver -}} +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterOutput +metadata: + name: stackdriver + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + matchRegex: (?:kube|service)\.(.*) + stackdriver: +{{ toYaml .Values.fluentbit.output.stackdriver | indent 4}} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/fluent-operator/templates/fluentbit-servicemonitor.yaml b/charts/fluent-operator/templates/fluentbit-servicemonitor.yaml new file mode 100644 index 00000000..3ea50ef5 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.serviceMonitor -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: fluent-bit + labels: + app.kubernetes.io/name: fluent-bit +spec: + endpoints: + - port: metrics + path: /api/v2/metrics/prometheus + interval: 30s + selector: + matchLabels: + app.kubernetes.io/name: fluent-bit +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml index b6e57cf2..2bf58c70 100644 --- a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml +++ b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml @@ -8,8 +8,13 @@ metadata: app.kubernetes.io/name: fluent-bit spec: service: - parsersFile: parsers.conf + parsersFiles: + - /fluent-bit/config/parsers.conf + - /fluent-bit/config/parsers_multiline.conf httpServer: true + {{- if .Values.fluentbit.logLevel }} + logLevel: {{ .Values.fluentbit.logLevel }} + {{- end }} {{- if .Values.fluentbit.service.storage }} storage: {{ toYaml .Values.fluentbit.service.storage | indent 6 }} @@ -26,5 +31,8 @@ spec: outputSelector: matchLabels: fluentbit.fluent.io/enabled: "true" + multilineParserSelector: + matchLabels: + fluentbit.fluent.io/enabled: "true" {{- end }} {{- end }} diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index a7de14e6..588fcc44 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -24,7 +24,7 @@ operator: memory: 64Mi container: repository: "kubesphere/fluent-operator" - tag: "v2.7.0" + tag: "v2.8.0" # nodeSelector configuration for Fluent Operator. Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} # Node tolerations applied to Fluent Operator. Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ @@ -64,14 +64,18 @@ operator: # setting fluentbit.crdsEnable or fluentd.crdsEnable values to false. # By default all CRDs are deployed. disableComponentControllers: "" + # Extra arguments given to the controller flags + extraArgs: [] + # - --watch-namespaces=logging fluentbit: # Installs a sub chart carrying the CRDs for the fluent-bit controller. The sub chart is enabled by default. crdsEnable: true enable: true + serviceMonitor: false image: repository: "kubesphere/fluent-bit" - tag: "v2.2.0" + tag: "v2.2.2" # fluentbit resources. If you do want to specify resources, adjust them as necessary # You can adjust it based on the log volume. resources: @@ -99,6 +103,7 @@ fluentbit: ## imagePullSecrets: [] # - name: "image-pull-secret" + logLevel: "" secrets: [] # fluent-bit daemonset use host network hostNetwork: false @@ -171,12 +176,15 @@ fluentbit: enable: true refreshIntervalSeconds: 10 memBufLimit: 100MB + bufferMaxSize: "" path: "/var/log/containers/*.log" skipLongLines: true readFromHead: false # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for tail input. storageType: memory pauseOnChunksOverlimit: "off" + # multiline.parser + # multilineParser: "docker, cri" systemd: enable: true systemdFilter: @@ -214,6 +222,7 @@ fluentbit: logstashPrefix: ks-logstash-log bufferSize: 20MB traceError: true + # logstashPrefixKey: ks-logstash-log # suppressTypeName: "On" # path: "" # bufferSize: "4KB" @@ -223,6 +232,7 @@ fluentbit: # logstashFormat: true # replaceDots: false # enableTLS: false + # writeOperation: upsert # tls: # verify: On # debug: 1 @@ -257,42 +267,44 @@ fluentbit: # See https://docs.fluentbit.io/manual/pipeline/outputs/loki loki: # Switch for generation of fluentbit loki ClusterOutput (and loki basic auth http user and pass secrets if required) - enable: false # Bool - host: 127.0.0.1 # String - port: 3100 # Int + enable: false # Bool + host: 127.0.0.1 # String + port: 3100 # Int # Either, give http{User,Password},tenantID string values specifying them directly httpUser: myuser httpPassword: mypass tenantID: '' # Or give {http{User,Password},tenantID} as reference to secrets that you have manually installed into your kubernetes cluster - # httpUser: + #httpUser: # valueFrom: # secretKeyRef: # key: value # name: husersecret # optional: true - # httpPassword: + #httpPassword: # valueFrom: # secretKeyRef: # key: value # name: hpasssecret # optional: true - # tenantID: + #tenantID: # valueFrom: # secretKeyRef: # key: value # name: tenantsecret # optional: true # - # labels: [] # String list of = - # labelKeys: [] # String list of - # removeKeys: [] # String list of - # labelMapPath: '' # String, path to file, ex /here/it/is - # dropSingleKey: off - # lineFormat: '' # String - # autoKubernetesLabels: on - # tenantIDKey: # String - # tls: {} # *plugins.TLS fluentbit docs + #labels: [] # String list of = + #labelKeys: [] # String list of + #removeKeys: [] # String list of + #labelMapPath: '' # String, path to file, ex /here/it/is + #dropSingleKey: off + #lineFormat: '' # String + #autoKubernetesLabels: on + #tenantIDKey: # String + #tls: {} # *plugins.TLS fluentbit docs + stackdriver: {} + # You can configure the stackdriver configuration here service: storage: {} @@ -309,6 +321,17 @@ fluentbit: # Configure the default filters in FluentBit. # The `filter` will filter and parse the collected log information and output the logs into a uniform format. You can choose whether to turn this on or not. filter: + multiline: + enable: false + keyContent: log + # emitterMemBufLimit 120 (MB) + emitterMemBufLimit: 120 + parsers: + - go + - python + - java + # use custom multiline parser need set .Values.parsers.javaMultiline.enable = true + # - java-multiline kubernetes: enable: true labels: false @@ -329,6 +352,14 @@ fluentbit: # Change the port to the port of a cloud-side Prometheus-compatible server that can receive Prometheus remote write data port: "" + # removes the hostPath mounts for varlibcontainers, varlogs and systemd. + disableLogVolumes: false + + parsers: + javaMultiline: + # use in filter for parser generic springboot multiline log format + enable: false + fluentd: # Installs a sub chart carrying the CRDs for the fluentd controller. The sub chart is enabled by default. crdsEnable: true