diff --git a/.chloggen/elasticsearchexporter_sync-bulk-indexer-flush-bytes.yaml b/.chloggen/elasticsearchexporter_sync-bulk-indexer-flush-bytes.yaml index 74cec1fe274f..979d80bdfad8 100644 --- a/.chloggen/elasticsearchexporter_sync-bulk-indexer-flush-bytes.yaml +++ b/.chloggen/elasticsearchexporter_sync-bulk-indexer-flush-bytes.yaml @@ -1,13 +1,13 @@ # Use this changelog template to create an entry for release notes. # One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' -change_type: enhancement +change_type: breaking # The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Respect `flush::bytes` in sync bulk indexer +note: Respect `flush::bytes` in sync bulk indexer, `flush::bytes` measures uncompressed size, change default `batcher::max_size_items` to `0` # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [36163] @@ -18,6 +18,8 @@ issues: [36163] subtext: Limit the bulk request size to roughly `flush::bytes` for sync bulk indexer. Sync bulk indexer is used when `batcher::enabled` is either true or false. In order words, sync bulk indexer is not used when batcher config is undefined. + Change `flush::bytes` to always measure in uncompressed bytes. + Change default `batcher::max_size_items` to `0` as bulk request size limit is now more effectively enforced by `flush::bytes`. # If your change doesn't affect end users or the exported elements of any package, # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index fd7321ffb111..391debba8a1a 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -91,7 +91,7 @@ The Elasticsearch exporter supports the [common `batcher` settings](https://gith - `batcher`: - `enabled` (default=unset): Enable batching of requests into a single bulk request. - `min_size_items` (default=5000): Minimum number of log records / spans in the buffer to trigger a flush immediately. - - `max_size_items` (default=10000): Maximum number of log records / spans in a request. + - `max_size_items` (default=0): Maximum number of log records / spans in a request. - `flush_timeout` (default=30s): Maximum time of the oldest item spent inside the buffer, aka "max age of buffer". A flush will happen regardless of the size of content in buffer. By default, the exporter will perform its own buffering and batching, as configured through the diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index bc803173440b..f3f848db2c90 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -118,7 +118,7 @@ func TestConfig(t *testing.T) { MinSizeItems: 5000, }, MaxSizeConfig: exporterbatcher.MaxSizeConfig{ - MaxSizeItems: 10000, + MaxSizeItems: 0, }, }, }, @@ -190,7 +190,7 @@ func TestConfig(t *testing.T) { MinSizeItems: 5000, }, MaxSizeConfig: exporterbatcher.MaxSizeConfig{ - MaxSizeItems: 10000, + MaxSizeItems: 0, }, }, }, @@ -262,7 +262,7 @@ func TestConfig(t *testing.T) { MinSizeItems: 5000, }, MaxSizeConfig: exporterbatcher.MaxSizeConfig{ - MaxSizeItems: 10000, + MaxSizeItems: 0, }, }, }, diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index e79883d99962..4783d430196a 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -91,7 +91,7 @@ func createDefaultConfig() component.Config { MinSizeItems: 5000, }, MaxSizeConfig: exporterbatcher.MaxSizeConfig{ - MaxSizeItems: 10000, + MaxSizeItems: 0, }, }, Flush: FlushSettings{