Merge branch 'master' into Batch-subscription-should-be-stoppable-des…

…pite-of-pending-message-retries

hermes-console/static/js/console/Home.js

@@ -4,5 +4,8 @@ home.controller('HomeController', ['$scope', 'DASHBOARD_CONFIG', function($scope
 
     $scope.statsDashboard = config.metrics;
     $scope.docs = config.docs;
-
+    $scope.costs = {
+        detailsUrl: window.config.costs.globalDetailsUrl,
+        enabled:  window.config.costs.enabled
+    };
 }]);

hermes-console/static/js/console/subscription/SubscriptionController.js

@@ -12,14 +12,26 @@ var subscriptions = angular.module('hermes.subscription', [
 
 subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'SubscriptionHealth', 'SubscriptionMetrics',
     'TopicRepository', 'TopicMetrics', '$scope', '$location', '$stateParams', '$uibModal', '$q', 'ConfirmationModal',
-    'toaster', 'PasswordService', 'MessagePreviewModal', 'FiltersDebuggerModalFactory', 'SUBSCRIPTION_CONFIG',
+    'toaster', 'PasswordService', 'MessagePreviewModal', 'FiltersDebuggerModalFactory', 'SUBSCRIPTION_CONFIG', '$sce',
     function (subscriptionRepository, subscriptionHealth, subscriptionMetrics, topicRepository, topicMetrics,
               $scope, $location, $stateParams, $modal, $q, confirmationModal, toaster, passwordService,
-              messagePreviewModal, filtersDebuggerModal, config) {
+              messagePreviewModal, filtersDebuggerModal, config, $sce) {
         var groupName = $scope.groupName = $stateParams.groupName;
         var topicName = $scope.topicName = $stateParams.topicName;
         var subscriptionName = $scope.subscriptionName = $stateParams.subscriptionName;
         $scope.config = config;
+        $scope.costs = {
+            enabled: window.config.costs.enabled,
+            iframeUrl: $sce.trustAsResourceUrl(resolveCostsUrl(window.config.costs.subscriptionIframeUrl)),
+            detailsUrl: resolveCostsUrl(window.config.costs.subscriptionDetailsUrl)
+        };
+
+        function resolveCostsUrl(template) {
+            if (template) {
+                return template.replace('{{topic_name}}', topicName).replace('{{subscription_name}}', subscriptionName);
+            }
+            return template;
+        }
 
         function getUndelivered() {
             subscriptionRepository.undelivered(topicName, subscriptionName).$promise
@@ -276,8 +288,8 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S
                 });
         };
         $scope.skipMessages = function (){
-            tommorowDate = new Date()
-            tommorowDate.setDate(tommorowDate.getDate() + 1)
+            tommorowDate = new Date();
+            tommorowDate.setDate(tommorowDate.getDate() + 1);
 
             confirmationModal.open({
                 action: 'Skip messages',
@@ -299,7 +311,7 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S
                     });
             });
 
-        }
+        };
         $scope.debugFilters = function () {
             filtersDebuggerModal.open(topicName, $scope.subscription.filters, $scope.topicContentType)
                 .then(function (result) {
@@ -311,6 +323,9 @@ subscriptions.controller('SubscriptionController', ['SubscriptionRepository', 'S
 
         $scope.trackingModeName = {"trackingOff": "No tracking", "discardedOnly": "Track message discarding only", "trackingAll": "Track everything"};
 
+        $scope.goToCostDetails = function () {
+            window.open($scope.costs.detailsUrl, '_blank');
+        };
     }]);
 
 subscriptions.controller('SubscriptionEditController', ['SubscriptionRepository', '$scope', '$uibModalInstance', 'subscription',

hermes-console/static/js/console/topic/TopicController.js

@@ -27,6 +27,18 @@ topics.controller('TopicController', ['TOPIC_CONFIG', 'TopicRepository', 'TopicM
         $scope.showHeadersFilter = subscriptionConfig.showHeadersFilter;
         $scope.offlineRetransmissionEnabled = topicConfig.offlineRetransmissionEnabled;
         $scope.iframeSource = "";
+        $scope.costs = {
+            enabled: window.config.costs.enabled,
+            iframeUrl: $sce.trustAsResourceUrl(resolveCostsUrl(window.config.costs.topicIframeUrl)),
+            detailsUrl: resolveCostsUrl(window.config.costs.topicDetailsUrl)
+        };
+
+        function resolveCostsUrl(template) {
+            if (template) {
+                return template.replace('{{topic_name}}', topicName);
+            }
+            return template;
+        }
 
         topicRepository.get(topicName).then(function(topicWithSchema) {
             $scope.topic = topicWithSchema;
@@ -288,6 +300,10 @@ topics.controller('TopicController', ['TOPIC_CONFIG', 'TopicRepository', 'TopicM
                 }
             });
         };
+
+        $scope.goToCostDetails = function () {
+            window.open($scope.costs.detailsUrl, '_blank');
+        };
     }]);
 
 topics.controller('TopicEditController', ['TOPIC_CONFIG', 'TopicRepository', '$scope', '$uibModalInstance', 'PasswordService',

hermes-console/static/partials/home.html

@@ -5,11 +5,16 @@
     </div>
 
     <div class="row" style="margin-bottom: 2em;">
-        <div class="col-md-6 col-md-offset-3">
+        <div class="col-md-{{costs.enabled ? 3 : 6}} col-md-offset-3">
             <a class="btn btn-main btn-block" ng-href="#/groups">
                 <i class="fa fa-cogs"></i> Console
             </a>
         </div>
+        <div class="col-md-3" ng-if="costs.enabled">
+            <a class="btn btn-main btn-block" ng-href="{{costs.detailsUrl}}" target="_blank">
+                <i class="fa fa-money"></i> Costs
+            </a>
+        </div>
     </div>
 
     <div class="row" style="margin-bottom: 2em;">

hermes-console/static/partials/subscription.html

@@ -99,6 +99,19 @@ <h3 class="panel-title">Service response metrics</h3>
                     <p><strong><a ng-href="{{metricsUrls.networkErrors}}">Other network errors</a></strong></p>
                 </div>
             </div>
+
+            <div class="panel panel-info" ng-if="costs.enabled">
+                <div class="panel-heading">
+                    <div class="pull-right">
+                        <button class="btn btn-primary btn-xs" ng-click="goToCostDetails()">Go to details</button>
+                    </div>
+                    <h3 class="panel-title">Costs</h3>
+                </div>
+                <div class="panel-body">
+                    <iframe height="100%" width="100%" style="border: none;" ng-src="{{costs.iframeUrl}}"></iframe>
+                </div>
+            </div>
+
             <div class="panel panel-default">
                 <div class="panel-heading">
                     <h3 class="panel-title">Manage subscription messages</h3>

hermes-console/static/partials/topic.html

@@ -58,6 +58,17 @@ <h3 class="panel-title">Metrics</h3>
                     <p><strong><a ng-href="{{metricsUrls.messageSize}}">Message size</a></strong></p>
                 </div>
             </div>
+            <div ng-if="costs.enabled" class="panel panel-info">
+                <div class="panel-heading">
+                    <div class="pull-right">
+                        <button class="btn btn-primary btn-xs" ng-click="goToCostDetails()">Go to details</button>
+                    </div>
+                    <h3 class="panel-title">Costs</h3>
+                </div>
+                <div class="panel-body">
+                    <iframe height="100%" width="100%" style="border: none;" ng-src="{{costs.iframeUrl}}"></iframe>
+                </div>
+            </div>
         </div>
         <div class="col-md-6">
             <div class="panel panel-default">

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java

@@ -55,7 +55,8 @@ public GraphiteClient graphiteClient(@Qualifier("monitoringRestTemplate") RestTe
     public VictoriaMetricsMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient,
                                                                     PrometheusMonitoringClientProperties properties) {
         return new VictoriaMetricsMetricsProvider(prometheusClient,
-                properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix());
+                properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix(),
+                properties.getAdditionalFilters());
     }
 
     @Bean

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java

@@ -3,6 +3,7 @@
 public class PrometheusMonitoringClientProperties extends ExternalMonitoringClientProperties {
     private String consumersMetricsPrefix = "hermes_consumers";
     private String frontendMetricsPrefix = "hermes_frontend";
+    private String additionalFilters = "";
 
     public String getConsumersMetricsPrefix() {
         return consumersMetricsPrefix;
@@ -19,4 +20,12 @@ public String getFrontendMetricsPrefix() {
     public void setFrontendMetricsPrefix(String frontendMetricsPrefix) {
         this.frontendMetricsPrefix = frontendMetricsPrefix;
     }
+
+    public String getAdditionalFilters() {
+        return additionalFilters;
+    }
+
+    public void setAdditionalFilters(String additionalFilters) {
+        this.additionalFilters = additionalFilters;
+    }
 }

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java

@@ -25,6 +25,7 @@ public class ConsoleProperties {
     private SubscriptionView subscription = new SubscriptionView();
     private ConsistencyView consistency = new ConsistencyView();
     private GroupView group = new GroupView();
+    private Costs costs = new Costs();
 
     public Dashboard getDashboard() {
         return dashboard;
@@ -82,6 +83,14 @@ public void setGroup(GroupView group) {
         this.group = group;
     }
 
+    public Costs getCosts() {
+        return costs;
+    }
+
+    public void setCosts(Costs costs) {
+        this.costs = costs;
+    }
+
     public static final class Console {
         private String title = "hermes console";
         private String footer = "";
@@ -834,4 +843,61 @@ public void setMaxGroupBatchSize(int maxGroupBatchSize) {
             this.maxGroupBatchSize = maxGroupBatchSize;
         }
     }
+
+    public static final class Costs {
+        private boolean enabled = false;
+        private String globalDetailsUrl = "";
+        private String topicIframeUrl = "";
+        private String topicDetailsUrl = "";
+        private String subscriptionIframeUrl = "";
+        private String subscriptionDetailsUrl = "";
+
+        public boolean isEnabled() {
+            return enabled;
+        }
+
+        public void setEnabled(boolean enabled) {
+            this.enabled = enabled;
+        }
+
+        public String getTopicIframeUrl() {
+            return topicIframeUrl;
+        }
+
+        public void setTopicIframeUrl(String topicIframeUrl) {
+            this.topicIframeUrl = topicIframeUrl;
+        }
+
+        public String getGlobalDetailsUrl() {
+            return globalDetailsUrl;
+        }
+
+        public void setGlobalDetailsUrl(String globalDetailsUrl) {
+            this.globalDetailsUrl = globalDetailsUrl;
+        }
+
+        public String getTopicDetailsUrl() {
+            return topicDetailsUrl;
+        }
+
+        public void setTopicDetailsUrl(String topicDetailsUrl) {
+            this.topicDetailsUrl = topicDetailsUrl;
+        }
+
+        public String getSubscriptionIframeUrl() {
+            return subscriptionIframeUrl;
+        }
+
+        public void setSubscriptionIframeUrl(String subscriptionIframeUrl) {
+            this.subscriptionIframeUrl = subscriptionIframeUrl;
+        }
+
+        public String getSubscriptionDetailsUrl() {
+            return subscriptionDetailsUrl;
+        }
+
+        public void setSubscriptionDetailsUrl(String subscriptionDetailsUrl) {
+            this.subscriptionDetailsUrl = subscriptionDetailsUrl;
+        }
+    }
 }

hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java

@@ -289,6 +289,7 @@ public List<Subscription> getAllSubscriptions() {
                 .map(Topic::getName)
                 .map(this::listSubscriptions)
                 .flatMap(List::stream)
+                .map(Subscription::anonymize)
                 .collect(toList());
     }
 

hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java

@@ -28,15 +28,17 @@ public class VictoriaMetricsMetricsProvider implements MonitoringSubscriptionMet
 
     private final String consumersMetricsPrefix;
     private final String frontendMetricsPrefix;
+    private final String additionalFilters;
     private final String subscriptionMetricsToQuery;
     private final String topicMetricsToQuery;
     private final PrometheusClient prometheusClient;
 
     public VictoriaMetricsMetricsProvider(PrometheusClient prometheusClient, String consumersMetricsPrefix,
-                                          String frontendMetricsPrefix) {
+                                          String frontendMetricsPrefix, String additionalFilters) {
         this.prometheusClient = prometheusClient;
         this.consumersMetricsPrefix = consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_";
         this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_";
+        this.additionalFilters = additionalFilters;
         this.subscriptionMetricsToQuery = Stream.of(SUBSCRIPTION_DELIVERED, SUBSCRIPTION_TIMEOUTS,
                         SUBSCRIPTION_THROUGHPUT, SUBSCRIPTION_OTHER_ERRORS, SUBSCRIPTION_BATCHES,
                         SUBSCRIPTION_STATUS_CODES)
@@ -56,9 +58,9 @@ public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscr
         https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param.
          */
         String queryFormat = "sum by (__name__,group,topic,subscription,status_code)"
-                + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s'}[1m]) keep_metric_names)";
+                + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s', %s}[1m]) keep_metric_names)";
         String query = String.format(queryFormat, subscriptionMetricsToQuery, subscriptionName.getTopicName().getGroupName(),
-                subscriptionName.getTopicName().getName(), subscriptionName.getName());
+                subscriptionName.getTopicName().getName(), subscriptionName.getName(), additionalFilters);
         MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query);
         return MonitoringSubscriptionMetricsProvider
                 .metricsBuilder()
@@ -80,8 +82,9 @@ public MonitoringTopicMetrics topicMetrics(TopicName topicName) {
         https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param.
          */
         String queryFormat = "sum by (__name__, group, topic) (irate({__name__=~'%s', group='%s', "
-                + "topic='%s'}[1m]) keep_metric_names)";
-        String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName());
+                + "topic='%s', %s}[1m]) keep_metric_names)";
+        String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName(),
+                additionalFilters);
         MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query);
         return MonitoringTopicMetricsProvider
                 .metricsBuilder()

hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy

@@ -22,7 +22,8 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati
 
     private SubscriptionLagSource lagSource = new NoOpSubscriptionLagSource()
 
-    private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, "hermes_consumers", "hermes_frontend");
+    private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(
+            client, "hermes_consumers", "hermes_frontend", "service=~'hermes'");
 
     private HybridSubscriptionMetricsRepository repository = new HybridSubscriptionMetricsRepository(prometheusMetricsProvider,
             summedSharedCounter, zookeeperPaths, lagSource)
@@ -34,7 +35,7 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati
             "|hermes_consumers_subscription_other_errors_total" +
             "|hermes_consumers_subscription_batches_total" +
             "|hermes_consumers_subscription_http_status_codes_total'," +
-            "group='group',topic='topic',subscription='subscription'}[1m]) keep_metric_names)"
+            "group='group',topic='topic',subscription='subscription', service=~'hermes'}[1m]) keep_metric_names)"
 
     def "should read subscription metrics from multiple places"() {
         given:

hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy

@@ -21,7 +21,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification {
     private SubscriptionRepository subscriptionRepository = Mock(SubscriptionRepository)
 
     private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client,
-            "hermes_consumers", "hermes_frontend")
+            "hermes_consumers", "hermes_frontend", "service='hermes'")
 
     private HybridTopicMetricsRepository repository = new HybridTopicMetricsRepository(prometheusMetricsProvider,
             summedSharedCounter, zookeeperPaths, subscriptionRepository)
@@ -31,7 +31,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification {
         String query = "sum by (__name__, group, topic) (irate({__name__=~'hermes_frontend_topic_requests_total" +
                 "|hermes_consumers_subscription_delivered_total" +
                 "|hermes_frontend_topic_throughput_bytes_total', group='group', " +
-                "topic='topic'}[1m]) keep_metric_names)"
+                "topic='topic', service='hermes'}[1m]) keep_metric_names)"
         TopicName topic = new TopicName('group', 'topic')
 
         client.readMetrics(query) >> MonitoringMetricsContainer.createEmpty()

integration/src/integration/java/pl/allegro/tech/hermes/integration/management/QueryEndpointTest.java

@@ -132,6 +132,7 @@ public static Object[][] subscriptionData() {
                 {"{\"query\": {\"or\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", asList(1, 3)},
                 {"{\"query\": {\"owner.id\": \"Team Alpha\"}}", asList(4)},
                 {"{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}", asList(4)},
+                {"{\"query\": {\"endpoint\": \".*password.*\"}}", asList()},
         };
     }
 
@@ -147,13 +148,13 @@ topic, enrichSubscription(subscription(topic.getName(), "subscription1"), "http:
                 topic, enrichSubscription(subscription(topic.getName(), "subscription2"), "http://endpoint2")
         );
         Subscription subscription3 = operations.createSubscription(
-                topic, enrichSubscription(subscription(topic.getName(), "subTestScription3"), "http://endpoint1")
+                topic, enrichSubscription(subscription(topic.getName(), "subTestScription3"), "http://login:password@endpoint1")
         );
         Subscription subscription4 = operations.createSubscription(topic, enrichSubscription(subscription(topic.getName(), "subscription4")
                 .withOwner(new OwnerId("Plaintext", "Team Alpha")), "http://endpoint2")
         );
 
-        List<Subscription> subscriptions = asList(subscription1, subscription2, subscription3, subscription4);
+        List<Subscription> subscriptions = asList(subscription1, subscription2, subscription3.anonymize(), subscription4);
 
         // when
         List<Subscription> found = management.query().querySubscriptions(query);