From 7a8b4fc260620261be67973486b88de67e21d8be Mon Sep 17 00:00:00 2001 From: Kevin Schoonover Date: Tue, 31 Aug 2021 21:45:15 -0700 Subject: [PATCH 1/2] parameterize recording rules using job_labels + cluster_labels --- cortex-mixin/recording_rules.libsonnet | 48 +++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cortex-mixin/recording_rules.libsonnet b/cortex-mixin/recording_rules.libsonnet index 0383524..b0338e0 100644 --- a/cortex-mixin/recording_rules.libsonnet +++ b/cortex-mixin/recording_rules.libsonnet @@ -13,54 +13,54 @@ local utils = import 'mixin-utils/utils.libsonnet'; { name: 'cortex_api_1', rules: - utils.histogramRules('cortex_request_duration_seconds', ['cluster', 'job']), + utils.histogramRules('cortex_request_duration_seconds', $._config.cluster_labels), }, { name: 'cortex_api_2', rules: - utils.histogramRules('cortex_request_duration_seconds', ['cluster', 'job', 'route']), + utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels), }, { name: 'cortex_api_3', rules: - utils.histogramRules('cortex_request_duration_seconds', ['cluster', 'namespace', 'job', 'route']), + utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels + ['route']), }, { name: 'cortex_querier_api', rules: - utils.histogramRules('cortex_querier_request_duration_seconds', ['cluster', 'job']) + - utils.histogramRules('cortex_querier_request_duration_seconds', ['cluster', 'job', 'route']) + - utils.histogramRules('cortex_querier_request_duration_seconds', ['cluster', 'namespace', 'job', 'route']), + utils.histogramRules('cortex_querier_request_duration_seconds', $._config.job_labels) + + utils.histogramRules('cortex_querier_request_duration_seconds', $._config.job_labels + ['route']) + + utils.histogramRules('cortex_querier_request_duration_seconds', $._config.cluster_labels + ['route']), }, { name: 'cortex_cache', rules: - utils.histogramRules('cortex_memcache_request_duration_seconds', ['cluster', 'job', 'method']) + - utils.histogramRules('cortex_cache_request_duration_seconds', ['cluster', 'job']) + - utils.histogramRules('cortex_cache_request_duration_seconds', ['cluster', 'job', 'method']), + utils.histogramRules('cortex_memcache_request_duration_seconds', $._config.job_labels + ['method']) + + utils.histogramRules('cortex_cache_request_duration_seconds', $._config.job_labels) + + utils.histogramRules('cortex_cache_request_duration_seconds', $._config.job_labels + ['method']), }, { name: 'cortex_storage', rules: - utils.histogramRules('cortex_bigtable_request_duration_seconds', ['cluster', 'job', 'operation']) + - utils.histogramRules('cortex_cassandra_request_duration_seconds', ['cluster', 'job', 'operation']) + - utils.histogramRules('cortex_dynamo_request_duration_seconds', ['cluster', 'job', 'operation']) + - utils.histogramRules('cortex_chunk_store_index_lookups_per_query', ['cluster', 'job']) + - utils.histogramRules('cortex_chunk_store_series_pre_intersection_per_query', ['cluster', 'job']) + - utils.histogramRules('cortex_chunk_store_series_post_intersection_per_query', ['cluster', 'job']) + - utils.histogramRules('cortex_chunk_store_chunks_per_query', ['cluster', 'job']) + - utils.histogramRules('cortex_database_request_duration_seconds', ['cluster', 'job', 'method']) + - utils.histogramRules('cortex_gcs_request_duration_seconds', ['cluster', 'job', 'operation']) + - utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job']), + utils.histogramRules('cortex_bigtable_request_duration_seconds', $._config.job_labels + ['operation']) + + utils.histogramRules('cortex_cassandra_request_duration_seconds', $._config.job_labels + ['operation']) + + utils.histogramRules('cortex_dynamo_request_duration_seconds', $._config.job_labels + ['operation']) + + utils.histogramRules('cortex_chunk_store_index_lookups_per_query', $._config.job_labels) + + utils.histogramRules('cortex_chunk_store_series_pre_intersection_per_query', $._config.job_labels) + + utils.histogramRules('cortex_chunk_store_series_post_intersection_per_query', $._config.job_labels) + + utils.histogramRules('cortex_chunk_store_chunks_per_query', $._config.job_labels) + + utils.histogramRules('cortex_database_request_duration_seconds', $._config.job_labels + ['method']) + + utils.histogramRules('cortex_gcs_request_duration_seconds', $._config.job_labels + ['operation']) + + utils.histogramRules('cortex_kv_request_duration_seconds', $._config.job_labels), }, { name: 'cortex_queries', rules: - utils.histogramRules('cortex_query_frontend_retries', ['cluster', 'job']) + - utils.histogramRules('cortex_query_frontend_queue_duration_seconds', ['cluster', 'job']) + - utils.histogramRules('cortex_ingester_queried_series', ['cluster', 'job']) + - utils.histogramRules('cortex_ingester_queried_chunks', ['cluster', 'job']) + - utils.histogramRules('cortex_ingester_queried_samples', ['cluster', 'job']), + utils.histogramRules('cortex_query_frontend_retries', $._config.job_labels) + + utils.histogramRules('cortex_query_frontend_queue_duration_seconds', $._config.job_labels) + + utils.histogramRules('cortex_ingester_queried_series', $._config.job_labels) + + utils.histogramRules('cortex_ingester_queried_chunks', $._config.job_labels) + + utils.histogramRules('cortex_ingester_queried_samples', $._config.job_labels), }, { name: 'cortex_received_samples', From 85c57b5c195cd1485c65f42454277b0bfaadcf7d Mon Sep 17 00:00:00 2001 From: Kevin Schoonover Date: Thu, 2 Sep 2021 20:24:42 -0700 Subject: [PATCH 2/2] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3032d94..84508dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ * [CHANGE] Changed default `job_names` for query-frontend, query-scheduler and querier to match custom deployments too. #376 * [CHANGE] Increase the rules per group and rule groups limits on different tiers. #396 * [CHANGE] Removed `max_samples_per_query` limit, since it only works with chunks and only when using `-distributor.shard-by-all-labels=false`. #397 +* [CHANGE] Parameterize the 'cortex_api', 'cortex_querier_api', 'cortex_cache', 'cortex_storage', and 'cortex_queries' recording results with 'job_labels' and 'cluster_labels' * [CHANGE] Removed chunks storage query sharding config support. The following config options have been removed: #398 * `_config` > `queryFrontend` > `shard_factor` * `_config` > `queryFrontend` > `sharded_queries_enabled`