From 1501b90e1cd93e4c39c21fba9ded7987089f2512 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 14 Aug 2024 12:03:56 +0200 Subject: [PATCH] ingester: set active series default to 20m (#8975) * ingester: set active series default to 20m In GL we bill based on active series in the last 20 minutes. This is also the value we want bill GEM for. It would be easier to reason about active series if they have the same default value everywhere. Because of this I'm changing the default value from 10m to 20m. I'll leave this open until the end of the week to gather more feedback. Signed-off-by: Dimitar Dimitrov * Update CHANGELOG.md entry Signed-off-by: Dimitar Dimitrov * Update CHANGELOG.md --------- Signed-off-by: Dimitar Dimitrov --- CHANGELOG.md | 2 +- cmd/mimir/config-descriptor.json | 2 +- cmd/mimir/help-all.txt.tmpl | 2 +- docs/sources/mimir/configure/configuration-parameters/index.md | 2 +- pkg/ingester/activeseries/active_series.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e23cace553b..1d1851e713a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ * [CHANGE] Ingester client: experimental support for client-side circuit breakers, their configuration options (`-ingester.client.circuit-breaker.*`) and metrics (`cortex_ingester_client_circuit_breaker_results_total`, `cortex_ingester_client_circuit_breaker_transitions_total`) were removed. #8802 * [CHANGE] Ingester: circuit breakers do not open in case of per-instance limit errors anymore. Opening can be triggered only in case of push and pull requests exceeding the configured duration. #8854 * [CHANGE] Query-frontend: Return `413 Request Entity Too Large` if a response shard for an `/active_series` request is too large. #8861 -* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #8422 #8430 #8454 #8455 #8360 #8490 #8508 #8577 #8660 #8671 #8677 #8747 #8850 #8872 #8838 #8911 #8909 #8923 #8924 #8925 #8932 #8933 #8934 #8962 #8986 #8993 +* [CHANGE] Ingester: increase the default inactivity timeout of active series (`-ingester.active-series-metrics-idle-timeout`) from `10m` to `20m`. #8975* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #8422 #8430 #8454 #8455 #8360 #8490 #8508 #8577 #8660 #8671 #8677 #8747 #8850 #8872 #8838 #8911 #8909 #8923 #8924 #8925 #8932 #8933 #8934 #8962 #8986 #8993 * [FEATURE] Experimental Kafka-based ingest storage. #6888 #6894 #6929 #6940 #6951 #6974 #6982 #7029 #7030 #7091 #7142 #7147 #7148 #7153 #7160 #7193 #7349 #7376 #7388 #7391 #7393 #7394 #7402 #7404 #7423 #7424 #7437 #7486 #7503 #7508 #7540 #7621 #7682 #7685 #7694 #7695 #7696 #7697 #7701 #7733 #7734 #7741 #7752 #7838 #7851 #7871 #7877 #7880 #7882 #7887 #7891 #7925 #7955 #7967 #8031 #8063 #8077 #8088 #8135 #8176 #8184 #8194 #8216 #8217 #8222 #8233 #8503 #8542 #8579 #8657 #8686 #8688 #8703 #8706 #8708 #8738 #8750 #8778 #8808 #8809 #8841 #8842 #8845 #8853 #8886 * What it is: * When the new ingest storage architecture is enabled, distributors write incoming write requests to a Kafka-compatible backend, and the ingesters asynchronously replay ingested data from Kafka. In this architecture, the write and read path are de-coupled through a Kafka-compatible backend. The write path and Kafka load is a function of the incoming write traffic, the read path load is a function of received queries. Whatever the load on the read path, it doesn't affect the write path. diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index c5c00780836..b6003d8ec54 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -3278,7 +3278,7 @@ "required": false, "desc": "After what time a series is considered to be inactive.", "fieldValue": null, - "fieldDefaultValue": 600000000000, + "fieldDefaultValue": 1200000000000, "fieldFlag": "ingester.active-series-metrics-idle-timeout", "fieldType": "duration", "fieldCategory": "advanced" diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index f3209849174..8633cfbca29 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1368,7 +1368,7 @@ Usage of ./cmd/mimir/mimir: -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration - After what time a series is considered to be inactive. (default 10m0s) + After what time a series is considered to be inactive. (default 20m0s) -ingester.active-series-metrics-update-period duration How often to update active series metrics. (default 1m0s) -ingester.client.backoff-max-period duration diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 0d64394f60d..4cc8b6b76cf 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -1200,7 +1200,7 @@ partition_ring: # (advanced) After what time a series is considered to be inactive. # CLI flag: -ingester.active-series-metrics-idle-timeout -[active_series_metrics_idle_timeout: | default = 10m] +[active_series_metrics_idle_timeout: | default = 20m] # (experimental) Period with which to update the per-tenant TSDB configuration. # CLI flag: -ingester.tsdb-config-update-period diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 7df6d102259..1f957784f49 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -34,7 +34,7 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.Enabled, EnabledFlag, true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.UpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") - f.DurationVar(&cfg.IdleTimeout, IdleTimeoutFlag, 10*time.Minute, "After what time a series is considered to be inactive.") + f.DurationVar(&cfg.IdleTimeout, IdleTimeoutFlag, 20*time.Minute, "After what time a series is considered to be inactive.") } // ActiveSeries is keeping track of recently active series for a single tenant.