From 1c0aba0e6aab58e5fc8aa02d2ee7b7cc994c6d7b Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 3 Oct 2024 11:42:34 -0400 Subject: [PATCH] [Elasticsearch] Fix the missing configuration environment variables propagation Signed-off-by: Aleksandr Maus --- config.go | 32 +++++++++++++++--------- docs/outputs/elasticsearch.md | 46 +++++++++++++++++------------------ 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/config.go b/config.go index d2ba750c6..f323a7d4f 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "text/template" + "time" kingpin "github.com/alecthomas/kingpin/v2" "github.com/spf13/viper" @@ -94,17 +95,23 @@ var httpOutputDefaults = map[string]map[string]any{ "DropEventThresholds": "10000:critical: 1000:critical: 100:critical: 10:warning: 1:warning", }, "Elasticsearch": { - "HostPort": "", - "Index": "falco", - "Type": "_doc", - "MinimumPriority": "", - "Suffix": "daily", - "Username": "", - "Password": "", - "FlattenFields": false, - "CreateIndexTemplate": false, - "NumberOfShards": 3, - "NumberOfReplicas": 3, + "ApiKey": "", + "HostPort": "", + "Index": "falco", + "Type": "_doc", + "MinimumPriority": "", + "Suffix": "daily", + "Username": "", + "Password": "", + "Pipeline": "", + "FlattenFields": false, + "CreateIndexTemplate": false, + "EnableCompression": false, + "NumberOfShards": 3, + "NumberOfReplicas": 3, + "Batching.Enabled": false, + "Batching.BatchSize": 5242880, + "Batching.FlushInterval": time.Second, }, "Quickwit": { "HostPort": "", @@ -589,6 +596,9 @@ func getConfig() *types.Configuration { v.GetStringMapString("AlertManager.CustomSeverityMap") v.GetStringMapString("GCP.PubSub.CustomAttributes") v.GetStringMapString("OTLP.Traces.ExtraEnvVars") + + c.Elasticsearch.CustomHeaders = v.GetStringMapString("Elasticsearch.CustomHeaders") + if err := v.Unmarshal(c); err != nil { log.Printf("[ERROR] : Error unmarshalling config : %s", err) } diff --git a/docs/outputs/elasticsearch.md b/docs/outputs/elasticsearch.md index 731f8c279..142fd894f 100644 --- a/docs/outputs/elasticsearch.md +++ b/docs/outputs/elasticsearch.md @@ -13,29 +13,29 @@ ## Configuration -| Setting | Env var | Default value | Description | -| ------------------------------------- | ------------------------------------ | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `elasticsearch.hostport` | `ELASTICSEARCH_HOSTPORT` | | http://{domain or ip}:{port}, if not empty, Elasticsearch output is **enabled** | -| `elasticsearch.index` | `ELASTICSEARCH_INDEX` | `falco` | Index | -| `elasticsearch.type` | `ELASTICSEARCH_TYPE` | `_doc` | Index | -| `elasticsearch.pipeline` | `ELASTICSEARCH_PIPELINE` | | Optional ingest pipeline name. Documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html | -| `elasticsearch.suffix` | `ELASTICSEARCH_SUFFIX` | `daily` | Date suffix for index rotation : `daily`, `monthly`, `annually`, `none` | -| `elasticsearch.apikey` | `ELASTICSEARCH_APIKEY` | | Use this APIKey to authenticate to Elasticsearch | -| `elasticsearch.username` | `ELASTICSEARCH_USERNAME` | | Use this username to authenticate to Elasticsearch | -| `elasticsearch.password` | `ELASTICSEARCH_PASSWORD` | | Use this password to authenticate to Elasticsearch | -| `elasticsearch.flattenfields` | `ELASTICSEARCH_FLATTENFIELDS` | `false` | Replace . by _ to avoid mapping conflicts, force to true if `createindextemplate=true` | -| `elasticsearch.createindextemplate` | `ELASTICSEARCH_CREATEINDEXTEMPLATE` | `false` | Create an index template | -| `elasticsearch.numberofshards` | `ELASTICSEARCH_NUMBEROFSHARDS` | `3` | Number of shards set by the index template | -| `elasticsearch.numberofreplicas` | `ELASTICSEARCH_REPLICAS` | `3` | Number of replicas set by the index template | -| `elasticsearch.customheaders` | `ELASTICSEARCH_CUSTOMHEADERS` | | Custom headers to add in POST, useful for Authentication | -| `elasticsearch.mutualtls` | `ELASTICSEARCH_MUTUALTLS` | `false` | Authenticate to the output with TLS, if true, checkcert flag will be ignored (server cert will always be checked) | -| `elasticsearch.checkcert` | `ELASTICSEARCH_CHECKCERT` | `true` | Check if ssl certificate of the output is valid | -| `elasticsearch.minimumpriority` | `ELASTICSEARCH_MINIMUMPRIORITY` | `""` (= `debug`) | Minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""` | -| `elasticsearch.maxconcurrentrequests` | `ELASTICSEARCH_MAXCONCURRENTREQUESTS`| `1` | Max number of concurrent requests | -| `elasticsearch.enablecompression` | `ELASTICSEARCH_ENABLECOMPRESSION` | `false` | Enables gzip compression | -| `elasticsearch.batching.enabled` | | `false` | Enables batching (utilizing Elasticsearch bulk API) | -| `elasticsearch.batching.batchsize` | | `5242880` | Batch size in bytes, default 5MB | -| `elasticsearch.batching.flushinterval`| | `1s` | Batch flush interval, use valid Go duration string | +| Setting | Env var | Default value | Description | +| ------------------------------------- | -------------------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `elasticsearch.hostport` | `ELASTICSEARCH_HOSTPORT` | | http://{domain or ip}:{port}, if not empty, Elasticsearch output is **enabled** | +| `elasticsearch.index` | `ELASTICSEARCH_INDEX` | `falco` | Index | +| `elasticsearch.type` | `ELASTICSEARCH_TYPE` | `_doc` | Index | +| `elasticsearch.pipeline` | `ELASTICSEARCH_PIPELINE` | | Optional ingest pipeline name. Documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html | +| `elasticsearch.suffix` | `ELASTICSEARCH_SUFFIX` | `daily` | Date suffix for index rotation : `daily`, `monthly`, `annually`, `none` | +| `elasticsearch.apikey` | `ELASTICSEARCH_APIKEY` | | Use this APIKey to authenticate to Elasticsearch | +| `elasticsearch.username` | `ELASTICSEARCH_USERNAME` | | Use this username to authenticate to Elasticsearch | +| `elasticsearch.password` | `ELASTICSEARCH_PASSWORD` | | Use this password to authenticate to Elasticsearch | +| `elasticsearch.flattenfields` | `ELASTICSEARCH_FLATTENFIELDS` | `false` | Replace . by _ to avoid mapping conflicts, force to true if `createindextemplate=true` | +| `elasticsearch.createindextemplate` | `ELASTICSEARCH_CREATEINDEXTEMPLATE` | `false` | Create an index template | +| `elasticsearch.numberofshards` | `ELASTICSEARCH_NUMBEROFSHARDS` | `3` | Number of shards set by the index template | +| `elasticsearch.numberofreplicas` | `ELASTICSEARCH_REPLICAS` | `3` | Number of replicas set by the index template | +| `elasticsearch.customheaders` | `ELASTICSEARCH_CUSTOMHEADERS` | | Custom headers to add in POST, useful for Authentication | +| `elasticsearch.mutualtls` | `ELASTICSEARCH_MUTUALTLS` | `false` | Authenticate to the output with TLS, if true, checkcert flag will be ignored (server cert will always be checked) | +| `elasticsearch.checkcert` | `ELASTICSEARCH_CHECKCERT` | `true` | Check if ssl certificate of the output is valid | +| `elasticsearch.minimumpriority` | `ELASTICSEARCH_MINIMUMPRIORITY` | `""` (= `debug`) | Minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""` | +| `elasticsearch.maxconcurrentrequests` | `ELASTICSEARCH_MAXCONCURRENTREQUESTS` | `1` | Max number of concurrent requests | +| `elasticsearch.enablecompression` | `ELASTICSEARCH_ENABLECOMPRESSION` | `false` | Enables gzip compression | +| `elasticsearch.batching.enabled` | `ELASTICSEARCH_BATCHING_ENABLED` | `false` | Enables batching (utilizing Elasticsearch bulk API) | +| `elasticsearch.batching.batchsize` | `ELASTICSEARCH_BATCHING_BATCHSIZE` | `5242880` | Batch size in bytes, default 5MB | +| `elasticsearch.batching.flushinterval`| `ELASTICSEARCH_BATCHING_FLUSHINTERVAL` | `1s` | Batch flush interval, use valid Go duration string | > [!NOTE] The Env var values override the settings from yaml file.