From cfa050fd8c7d00c28b1e526974b460a417e893c6 Mon Sep 17 00:00:00 2001 From: Marcin Kasznia Date: Thu, 19 Oct 2023 12:13:56 +0200 Subject: [PATCH 1/3] Anonymize password in subscription query (#1752) --- .../management/domain/subscription/SubscriptionService.java | 1 + .../hermes/integration/management/QueryEndpointTest.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java index 6afe91eeae..fe313b9ff6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java @@ -289,6 +289,7 @@ public List getAllSubscriptions() { .map(Topic::getName) .map(this::listSubscriptions) .flatMap(List::stream) + .map(Subscription::anonymize) .collect(toList()); } diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/QueryEndpointTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/QueryEndpointTest.java index 4cfd56932d..cb1333e29b 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/QueryEndpointTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/QueryEndpointTest.java @@ -132,6 +132,7 @@ public static Object[][] subscriptionData() { {"{\"query\": {\"or\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", asList(1, 3)}, {"{\"query\": {\"owner.id\": \"Team Alpha\"}}", asList(4)}, {"{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}", asList(4)}, + {"{\"query\": {\"endpoint\": \".*password.*\"}}", asList()}, }; } @@ -147,13 +148,13 @@ topic, enrichSubscription(subscription(topic.getName(), "subscription1"), "http: topic, enrichSubscription(subscription(topic.getName(), "subscription2"), "http://endpoint2") ); Subscription subscription3 = operations.createSubscription( - topic, enrichSubscription(subscription(topic.getName(), "subTestScription3"), "http://endpoint1") + topic, enrichSubscription(subscription(topic.getName(), "subTestScription3"), "http://login:password@endpoint1") ); Subscription subscription4 = operations.createSubscription(topic, enrichSubscription(subscription(topic.getName(), "subscription4") .withOwner(new OwnerId("Plaintext", "Team Alpha")), "http://endpoint2") ); - List subscriptions = asList(subscription1, subscription2, subscription3, subscription4); + List subscriptions = asList(subscription1, subscription2, subscription3.anonymize(), subscription4); // when List found = management.query().querySubscriptions(query); From cffefa7bfc82e175a5b5dd1061756ef883cf1f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20R=C5=BCysko?= Date: Thu, 19 Oct 2023 12:40:04 +0200 Subject: [PATCH 2/3] Add possibility of embedding cost report in hermes-console (#1734) --- hermes-console/static/js/console/Home.js | 5 +- .../subscription/SubscriptionController.js | 25 +++++-- .../js/console/topic/TopicController.js | 16 +++++ hermes-console/static/partials/home.html | 7 +- .../static/partials/subscription.html | 13 ++++ hermes-console/static/partials/topic.html | 11 ++++ .../config/console/ConsoleProperties.java | 66 +++++++++++++++++++ 7 files changed, 136 insertions(+), 7 deletions(-) diff --git a/hermes-console/static/js/console/Home.js b/hermes-console/static/js/console/Home.js index ce07848943..9dbe5be48e 100644 --- a/hermes-console/static/js/console/Home.js +++ b/hermes-console/static/js/console/Home.js @@ -4,5 +4,8 @@ home.controller('HomeController', ['$scope', 'DASHBOARD_CONFIG', function($scope $scope.statsDashboard = config.metrics; $scope.docs = config.docs; - + $scope.costs = { + detailsUrl: window.config.costs.globalDetailsUrl, + enabled: window.config.costs.enabled + }; }]); diff --git a/hermes-console/static/js/console/subscription/SubscriptionController.js b/hermes-console/static/js/console/subscription/SubscriptionController.js index 6575056d8f..3fc2dc3237 100644 --- a/hermes-console/static/js/console/subscription/SubscriptionController.js +++ b/hermes-console/static/js/console/subscription/SubscriptionController.js @@ -12,14 +12,26 @@ var subscriptions = angular.module('hermes.subscription', [ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'SubscriptionHealth', 'SubscriptionMetrics', 'TopicRepository', 'TopicMetrics', '$scope', '$location', '$stateParams', '$uibModal', '$q', 'ConfirmationModal', - 'toaster', 'PasswordService', 'MessagePreviewModal', 'FiltersDebuggerModalFactory', 'SUBSCRIPTION_CONFIG', + 'toaster', 'PasswordService', 'MessagePreviewModal', 'FiltersDebuggerModalFactory', 'SUBSCRIPTION_CONFIG', '$sce', function (subscriptionRepository, subscriptionHealth, subscriptionMetrics, topicRepository, topicMetrics, $scope, $location, $stateParams, $modal, $q, confirmationModal, toaster, passwordService, - messagePreviewModal, filtersDebuggerModal, config) { + messagePreviewModal, filtersDebuggerModal, config, $sce) { var groupName = $scope.groupName = $stateParams.groupName; var topicName = $scope.topicName = $stateParams.topicName; var subscriptionName = $scope.subscriptionName = $stateParams.subscriptionName; $scope.config = config; + $scope.costs = { + enabled: window.config.costs.enabled, + iframeUrl: $sce.trustAsResourceUrl(resolveCostsUrl(window.config.costs.subscriptionIframeUrl)), + detailsUrl: resolveCostsUrl(window.config.costs.subscriptionDetailsUrl) + }; + + function resolveCostsUrl(template) { + if (template) { + return template.replace('{{topic_name}}', topicName).replace('{{subscription_name}}', subscriptionName); + } + return template; + } function getUndelivered() { subscriptionRepository.undelivered(topicName, subscriptionName).$promise @@ -276,8 +288,8 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S }); }; $scope.skipMessages = function (){ - tommorowDate = new Date() - tommorowDate.setDate(tommorowDate.getDate() + 1) + tommorowDate = new Date(); + tommorowDate.setDate(tommorowDate.getDate() + 1); confirmationModal.open({ action: 'Skip messages', @@ -299,7 +311,7 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S }); }); - } + }; $scope.debugFilters = function () { filtersDebuggerModal.open(topicName, $scope.subscription.filters, $scope.topicContentType) .then(function (result) { @@ -311,6 +323,9 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S $scope.trackingModeName = {"trackingOff": "No tracking", "discardedOnly": "Track message discarding only", "trackingAll": "Track everything"}; + $scope.goToCostDetails = function () { + window.open($scope.costs.detailsUrl, '_blank'); + }; }]); subscriptions.controller('SubscriptionEditController', ['SubscriptionRepository', '$scope', '$uibModalInstance', 'subscription', diff --git a/hermes-console/static/js/console/topic/TopicController.js b/hermes-console/static/js/console/topic/TopicController.js index b42015a023..1c0bcd1907 100644 --- a/hermes-console/static/js/console/topic/TopicController.js +++ b/hermes-console/static/js/console/topic/TopicController.js @@ -27,6 +27,18 @@ topics.controller('TopicController', ['TOPIC_CONFIG', 'TopicRepository', 'TopicM $scope.showHeadersFilter = subscriptionConfig.showHeadersFilter; $scope.offlineRetransmissionEnabled = topicConfig.offlineRetransmissionEnabled; $scope.iframeSource = ""; + $scope.costs = { + enabled: window.config.costs.enabled, + iframeUrl: $sce.trustAsResourceUrl(resolveCostsUrl(window.config.costs.topicIframeUrl)), + detailsUrl: resolveCostsUrl(window.config.costs.topicDetailsUrl) + }; + + function resolveCostsUrl(template) { + if (template) { + return template.replace('{{topic_name}}', topicName); + } + return template; + } topicRepository.get(topicName).then(function(topicWithSchema) { $scope.topic = topicWithSchema; @@ -288,6 +300,10 @@ topics.controller('TopicController', ['TOPIC_CONFIG', 'TopicRepository', 'TopicM } }); }; + + $scope.goToCostDetails = function () { + window.open($scope.costs.detailsUrl, '_blank'); + }; }]); topics.controller('TopicEditController', ['TOPIC_CONFIG', 'TopicRepository', '$scope', '$uibModalInstance', 'PasswordService', diff --git a/hermes-console/static/partials/home.html b/hermes-console/static/partials/home.html index b8766357b4..7d5a34fa2f 100644 --- a/hermes-console/static/partials/home.html +++ b/hermes-console/static/partials/home.html @@ -5,11 +5,16 @@
-
+
Console
+
+ + Costs + +
diff --git a/hermes-console/static/partials/subscription.html b/hermes-console/static/partials/subscription.html index ea70059f81..0f20569782 100644 --- a/hermes-console/static/partials/subscription.html +++ b/hermes-console/static/partials/subscription.html @@ -99,6 +99,19 @@

Service response metrics

Other network errors

+ +
+
+
+ +
+

Costs

+
+
+ +
+
+

Manage subscription messages

diff --git a/hermes-console/static/partials/topic.html b/hermes-console/static/partials/topic.html index 17851a08ca..0145b0242c 100644 --- a/hermes-console/static/partials/topic.html +++ b/hermes-console/static/partials/topic.html @@ -58,6 +58,17 @@

Metrics

Message size

+
+
+
+ +
+

Costs

+
+
+ +
+
diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java index 122da5d7a2..bc34213cda 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java @@ -25,6 +25,7 @@ public class ConsoleProperties { private SubscriptionView subscription = new SubscriptionView(); private ConsistencyView consistency = new ConsistencyView(); private GroupView group = new GroupView(); + private Costs costs = new Costs(); public Dashboard getDashboard() { return dashboard; @@ -82,6 +83,14 @@ public void setGroup(GroupView group) { this.group = group; } + public Costs getCosts() { + return costs; + } + + public void setCosts(Costs costs) { + this.costs = costs; + } + public static final class Console { private String title = "hermes console"; private String footer = ""; @@ -834,4 +843,61 @@ public void setMaxGroupBatchSize(int maxGroupBatchSize) { this.maxGroupBatchSize = maxGroupBatchSize; } } + + public static final class Costs { + private boolean enabled = false; + private String globalDetailsUrl = ""; + private String topicIframeUrl = ""; + private String topicDetailsUrl = ""; + private String subscriptionIframeUrl = ""; + private String subscriptionDetailsUrl = ""; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getTopicIframeUrl() { + return topicIframeUrl; + } + + public void setTopicIframeUrl(String topicIframeUrl) { + this.topicIframeUrl = topicIframeUrl; + } + + public String getGlobalDetailsUrl() { + return globalDetailsUrl; + } + + public void setGlobalDetailsUrl(String globalDetailsUrl) { + this.globalDetailsUrl = globalDetailsUrl; + } + + public String getTopicDetailsUrl() { + return topicDetailsUrl; + } + + public void setTopicDetailsUrl(String topicDetailsUrl) { + this.topicDetailsUrl = topicDetailsUrl; + } + + public String getSubscriptionIframeUrl() { + return subscriptionIframeUrl; + } + + public void setSubscriptionIframeUrl(String subscriptionIframeUrl) { + this.subscriptionIframeUrl = subscriptionIframeUrl; + } + + public String getSubscriptionDetailsUrl() { + return subscriptionDetailsUrl; + } + + public void setSubscriptionDetailsUrl(String subscriptionDetailsUrl) { + this.subscriptionDetailsUrl = subscriptionDetailsUrl; + } + } } From 97a54ab8274810461c5fac589730de95d5f9b91d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20F=C4=85derski?= Date: Thu, 19 Oct 2023 16:20:29 +0200 Subject: [PATCH 3/3] Allow additional queries for prometheus external metrics (#1747) --- .../config/ExternalMonitoringConfiguration.java | 3 ++- .../PrometheusMonitoringClientProperties.java | 9 +++++++++ .../prometheus/VictoriaMetricsMetricsProvider.java | 13 ++++++++----- ...eusBasedSubscriptionMetricsRepositoryTest.groovy | 5 +++-- ...PrometheusBasedTopicMetricsRepositoryTest.groovy | 4 ++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java index 44d9816ded..e3a828a463 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java @@ -55,7 +55,8 @@ public GraphiteClient graphiteClient(@Qualifier("monitoringRestTemplate") RestTe public VictoriaMetricsMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient, PrometheusMonitoringClientProperties properties) { return new VictoriaMetricsMetricsProvider(prometheusClient, - properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix()); + properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix(), + properties.getAdditionalFilters()); } @Bean diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java index 4c0977f192..cdf4ce2cdb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java @@ -3,6 +3,7 @@ public class PrometheusMonitoringClientProperties extends ExternalMonitoringClientProperties { private String consumersMetricsPrefix = "hermes_consumers"; private String frontendMetricsPrefix = "hermes_frontend"; + private String additionalFilters = ""; public String getConsumersMetricsPrefix() { return consumersMetricsPrefix; @@ -19,4 +20,12 @@ public String getFrontendMetricsPrefix() { public void setFrontendMetricsPrefix(String frontendMetricsPrefix) { this.frontendMetricsPrefix = frontendMetricsPrefix; } + + public String getAdditionalFilters() { + return additionalFilters; + } + + public void setAdditionalFilters(String additionalFilters) { + this.additionalFilters = additionalFilters; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java index 29209b9dde..0b44d97cd4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java @@ -28,15 +28,17 @@ public class VictoriaMetricsMetricsProvider implements MonitoringSubscriptionMet private final String consumersMetricsPrefix; private final String frontendMetricsPrefix; + private final String additionalFilters; private final String subscriptionMetricsToQuery; private final String topicMetricsToQuery; private final PrometheusClient prometheusClient; public VictoriaMetricsMetricsProvider(PrometheusClient prometheusClient, String consumersMetricsPrefix, - String frontendMetricsPrefix) { + String frontendMetricsPrefix, String additionalFilters) { this.prometheusClient = prometheusClient; this.consumersMetricsPrefix = consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_"; this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_"; + this.additionalFilters = additionalFilters; this.subscriptionMetricsToQuery = Stream.of(SUBSCRIPTION_DELIVERED, SUBSCRIPTION_TIMEOUTS, SUBSCRIPTION_THROUGHPUT, SUBSCRIPTION_OTHER_ERRORS, SUBSCRIPTION_BATCHES, SUBSCRIPTION_STATUS_CODES) @@ -56,9 +58,9 @@ public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscr https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param. */ String queryFormat = "sum by (__name__,group,topic,subscription,status_code)" - + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s'}[1m]) keep_metric_names)"; + + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s', %s}[1m]) keep_metric_names)"; String query = String.format(queryFormat, subscriptionMetricsToQuery, subscriptionName.getTopicName().getGroupName(), - subscriptionName.getTopicName().getName(), subscriptionName.getName()); + subscriptionName.getTopicName().getName(), subscriptionName.getName(), additionalFilters); MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query); return MonitoringSubscriptionMetricsProvider .metricsBuilder() @@ -80,8 +82,9 @@ public MonitoringTopicMetrics topicMetrics(TopicName topicName) { https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param. */ String queryFormat = "sum by (__name__, group, topic) (irate({__name__=~'%s', group='%s', " - + "topic='%s'}[1m]) keep_metric_names)"; - String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName()); + + "topic='%s', %s}[1m]) keep_metric_names)"; + String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName(), + additionalFilters); MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query); return MonitoringTopicMetricsProvider .metricsBuilder() diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy index e5581587f0..7c7ef12876 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy @@ -22,7 +22,8 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati private SubscriptionLagSource lagSource = new NoOpSubscriptionLagSource() - private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, "hermes_consumers", "hermes_frontend"); + private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider( + client, "hermes_consumers", "hermes_frontend", "service=~'hermes'"); private HybridSubscriptionMetricsRepository repository = new HybridSubscriptionMetricsRepository(prometheusMetricsProvider, summedSharedCounter, zookeeperPaths, lagSource) @@ -34,7 +35,7 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati "|hermes_consumers_subscription_other_errors_total" + "|hermes_consumers_subscription_batches_total" + "|hermes_consumers_subscription_http_status_codes_total'," + - "group='group',topic='topic',subscription='subscription'}[1m]) keep_metric_names)" + "group='group',topic='topic',subscription='subscription', service=~'hermes'}[1m]) keep_metric_names)" def "should read subscription metrics from multiple places"() { given: diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy index 0833b55a15..c3dcad72ad 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy @@ -21,7 +21,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification { private SubscriptionRepository subscriptionRepository = Mock(SubscriptionRepository) private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, - "hermes_consumers", "hermes_frontend") + "hermes_consumers", "hermes_frontend", "service='hermes'") private HybridTopicMetricsRepository repository = new HybridTopicMetricsRepository(prometheusMetricsProvider, summedSharedCounter, zookeeperPaths, subscriptionRepository) @@ -31,7 +31,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification { String query = "sum by (__name__, group, topic) (irate({__name__=~'hermes_frontend_topic_requests_total" + "|hermes_consumers_subscription_delivered_total" + "|hermes_frontend_topic_throughput_bytes_total', group='group', " + - "topic='topic'}[1m]) keep_metric_names)" + "topic='topic', service='hermes'}[1m]) keep_metric_names)" TopicName topic = new TopicName('group', 'topic') client.readMetrics(query) >> MonitoringMetricsContainer.createEmpty()