From 60a526e9e02cddbbfb829dd34a7c7b7f710ef77c Mon Sep 17 00:00:00 2001 From: Khushboo Rajput Date: Wed, 12 May 2021 16:09:39 -0700 Subject: [PATCH] Squashed commit of the following: cr https://code.amazon.com/reviews/CR-50314382 commit 2eb3872b0d3bfde09719497440019eeb7061dd7b Merge: fa1ac709 a3877bad Author: Khushboo Rajput Date: Tue May 4 21:08:59 2021 -0700 Merge branch 'main' into aes_7.10_nightly commit fa1ac70933ae5c1939c4a0fa30f50f81925a07ab Merge: fac4f8b8 c70b19dc Author: Khushboo Rajput Date: Tue May 4 21:04:35 2021 -0700 Merge branch 'aes_7.10_gh' into aes_7.10_nightly commit a3877bad46b52220a664ce07606446e03da4525e Author: Sruti Parthiban Date: Thu Apr 15 12:21:34 2021 -0700 Fix format error (#309) commit 7e0d2f8577461b8a788b4bf1e8eb147e93776d1e Author: Sruti Parthiban Date: Thu Apr 15 11:30:34 2021 -0700 Fix failing election term tests (#305) commit ea643fe1ed8f381a63790985bea34e570c563bf2 Author: meetshah777 <48720201+meetshah777@users.noreply.github.com> Date: Thu Apr 1 02:29:45 2021 +0530 Changed master total PendingQueueSize to PendingQueueSize per task type (#296) * Changed master total PendingQueueSize to PendingQueueSize per task type * Addressed comments Co-authored-by: Meet Shah commit bf6a4cc70a33068855d9c53dde95320143736d76 Author: meetshah777 <48720201+meetshah777@users.noreply.github.com> Date: Thu Apr 1 02:27:24 2021 +0530 Added Election Term Metric (#297) * Changed master total PendingQueueSize to PendingQueueSize per task type * Added Election Term Metric * Formmated files * Merge conflict resolved * Addressed comments Co-authored-by: Meet Shah commit 130a494cb56e533d954404f9c805c22ddd9d5212 Author: Mital Awachat Date: Thu Mar 25 23:35:49 2021 +0530 AdmissionControl: Clear metricQueue before running test (#300) Co-authored-by: Mital Awachat commit 3e6b8eca84a79672454db86c224ecfa93e2c0c89 Author: Joshua Tokle Date: Wed Mar 24 19:07:06 2021 -0700 Configure spotless plugin and enable code formatting (#276) * Configure spotless plugin and enable code formatting * Remove checkstyle and @Ignore flaky tests commit fac4f8b835e793e3db5240f1e4d1448fd9d16baa Merge: 253229a2 f7b430b5 Author: Khushboo Rajput Date: Tue Mar 23 17:14:37 2021 -0700 Merge branch 'main' into aes_7.10_nightly commit f7b430b5d50b0f78e4299cd0ca5ce17ec71bfa23 Author: Dharmesh Date: Wed Mar 24 05:25:20 2021 +0530 [Build Fix]Removing the typecasting to IndexingPressure class to fix build failure while backporting (#293) commit d49a701707336e71aa97a58e6104de1682d7c4b9 Author: amathur1893 Date: Thu Mar 18 02:51:11 2021 +0530 Cluster state applier stats (#282) * Publish latency and failure metrics for cluster state applier thread * Publish latency and failure metrics for cluster state applier thread * Extract MetricsTracker * Add Javadoc for Metrics Tracker * Add ClusterStateApplierStats to collect latency and failure metrics * Add ClusterStateApplierStats to collect latency and failure metrics * Added UTs for ClusterApplierServiceStatsCollector * Added UTs for ClusterApplierServiceStatsCollector * Java doc changes Co-authored-by: Arpita Mathur commit 253229a24f49b5479333d8ffda7167441e39162a Merge: 7c5cb8e6 52e5cc4c Author: Khushboo Rajput Date: Mon Mar 8 22:06:07 2021 -0800 Merge branch 'main' into aes_7.10_nightly commit 52e5cc4c1a1189245b48c4f4aa83a0eead0276df Author: Mital Awachat Date: Tue Mar 9 10:49:11 2021 +0530 Track admission control metrics (#290) * Track admission control metrics Co-authored-by: Mital Awachat commit c2809155208b5f94905251bd4e2efde506a91c84 Author: Dharmesh Date: Thu Mar 4 23:12:37 2021 +0530 [Bug Fix] Incorrect metric value was returned from the getter of current bytes (#287) commit 7c5cb8e63cde9023084ab57075bbdf554df9748c Merge: 28ee8302 a997189c Author: Khushboo Rajput Date: Thu Mar 4 09:39:54 2021 -0800 Merge branch 'main' into aes_7.10_nightly commit a997189cb93532e2efb33b1bbafde6febbd78495 Author: Dharmesh Date: Thu Mar 4 10:55:36 2021 +0530 Added metrics publishing logic (#285) commit 28ee8302529c279dfadc47bf9b0d4f6b6d91ac75 Author: Dharmesh Date: Tue Mar 2 23:49:47 2021 +0530 Adding a additional check to just process first 50 objects of hot store (#284) commit 0d2566a61bab1531400eaae486c3734bcea386fe Author: Dharmesh <12bce021@nirmauni.ac.in> Date: Tue Mar 2 11:33:35 2021 +0530 Changes to collect shard indexing pressure metrics and push to shared location (#278) * Changes to collect shard indexing pressure metrics and push to shared location * Code changes to remove pushing the cold store objects and removing isActiveShard flag as it becomes irrelevant now * Addressing comment to change NodeRole to IndexingStage as node role already has a predefined meaning * Checking if class is present while adding the collector in scheduled executor so that the collector will never be invoked if class in not present * Addressing comments. commit 819d48528c24438fb00af9aadafe41473e440949 Author: Dharmesh Date: Tue Mar 2 23:49:47 2021 +0530 Adding a additional check to just process first 50 objects of hot store (#284) commit 2f3df02f6b6a03f44782af0c1b8b63d02ff7df0a Author: Dharmesh <12bce021@nirmauni.ac.in> Date: Tue Mar 2 11:33:35 2021 +0530 Changes to collect shard indexing pressure metrics and push to shared location (#278) * Changes to collect shard indexing pressure metrics and push to shared location * Code changes to remove pushing the cold store objects and removing isActiveShard flag as it becomes irrelevant now * Addressing comment to change NodeRole to IndexingStage as node role already has a predefined meaning * Checking if class is present while adding the collector in scheduled executor so that the collector will never be invoked if class in not present * Addressing comments. commit 9f830593b58e0c38fdf4bb5fa225f6899685c9c2 Author: Yu Sun <11154022+yu-sun-77@users.noreply.github.com> Date: Fri Feb 26 14:36:02 2021 -0800 Log error once only (#280) commit 3377c2f3cdefd68dc8d13e5640200c58ec2b4dc4 Author: Karthik Kumarguru <52506191+ktkrg@users.noreply.github.com> Date: Mon Feb 22 15:11:45 2021 -0800 Increase the max heap limit to check against in the integ tests (#277) commit 641b403f941596f543e95a01b9a5795b73d7fd65 Author: Khushboo Rajput Date: Sat Feb 20 12:45:57 2021 -0800 Creating new brand for ES 7.10 version --- licenses/bcpkix-jdk15on-1.66.jar.sha1 | 1 - licenses/bcprov-jdk15on-1.66.jar.sha1 | 1 - licenses/jackson-annotations-2.10.4.jar.sha1 | 1 - licenses/jackson-databind-2.10.4.jar.sha1 | 1 - .../performanceanalyzer-rca-1.13.jar.sha1 | 2 +- .../PerformanceAnalyzerPlugin.java | 11 +- .../AdmissionControlMetricsCollector.java | 43 ++++- .../ClusterApplierServiceStatsCollector.java | 2 +- .../collectors/ElectionTermCollector.java | 2 +- .../collectors/MasterServiceMetrics.java | 2 + .../performanceanalyzer/util/Utils.java | 3 +- ...AdmissionControlMetricsCollectorTests.java | 44 +++-- ...sterApplierServiceStatsCollectorTests.java | 168 +++++++++++++++--- .../ElectionTermCollectorTests.java | 2 +- 14 files changed, 225 insertions(+), 58 deletions(-) delete mode 100644 licenses/bcpkix-jdk15on-1.66.jar.sha1 delete mode 100644 licenses/bcprov-jdk15on-1.66.jar.sha1 delete mode 100644 licenses/jackson-annotations-2.10.4.jar.sha1 delete mode 100644 licenses/jackson-databind-2.10.4.jar.sha1 diff --git a/licenses/bcpkix-jdk15on-1.66.jar.sha1 b/licenses/bcpkix-jdk15on-1.66.jar.sha1 deleted file mode 100644 index 7a5b700b..00000000 --- a/licenses/bcpkix-jdk15on-1.66.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -86cca2c1a32775abe92316d9628b7ee50b6f19ad \ No newline at end of file diff --git a/licenses/bcprov-jdk15on-1.66.jar.sha1 b/licenses/bcprov-jdk15on-1.66.jar.sha1 deleted file mode 100644 index 009cae20..00000000 --- a/licenses/bcprov-jdk15on-1.66.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed564ade61defca27e26fb1378a70b22831fc5c1 \ No newline at end of file diff --git a/licenses/jackson-annotations-2.10.4.jar.sha1 b/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0..00000000 --- a/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/licenses/jackson-databind-2.10.4.jar.sha1 b/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72c..00000000 --- a/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/licenses/performanceanalyzer-rca-1.13.jar.sha1 b/licenses/performanceanalyzer-rca-1.13.jar.sha1 index 1959cbc9..0c4379d5 100644 --- a/licenses/performanceanalyzer-rca-1.13.jar.sha1 +++ b/licenses/performanceanalyzer-rca-1.13.jar.sha1 @@ -1 +1 @@ -0ca252df95b9c0743d7b29452c7637504d5ba50f \ No newline at end of file +0d0d568122db55b2a11cf5d0f562d754e2bcf898 \ No newline at end of file diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index ada7d7a8..76a63c28 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2019-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.FaultDetectionMetricsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.GCInfoCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ShardIndexingPressureMetricsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterThrottlingMetricsCollector; @@ -214,13 +213,7 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ClusterApplierServiceStatsCollector( performanceAnalyzerController,configOverridesWrapper)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new AdmissionControlMetricsCollector()); - try { - Class.forName(ShardIndexingPressureMetricsCollector.SHARD_INDEXING_PRESSURE_CLASS_NAME); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardIndexingPressureMetricsCollector( - performanceAnalyzerController,configOverridesWrapper)); - } catch (ClassNotFoundException e) { - LOG.info("Shard IndexingPressure not present in this ES version. Skipping ShardIndexingPressureMetricsCollector"); - } + try { Class.forName(MasterClusterStateUpdateStatsCollector.MASTER_CLUSTER_UPDATE_STATS_CLASS_NAME); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterClusterStateUpdateStatsCollector( diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java index 116d08c5..70a24728 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java @@ -1,3 +1,18 @@ +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp; @@ -6,6 +21,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.WriterMetrics; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -14,7 +30,7 @@ import java.util.Objects; /** - * AdmissionControlMetricsCollector collects `UsedQuota`, `TotalQuota`, RejectionCount from ElasticSearch-Jetty - AdmissionController + * AdmissionControlMetricsCollector collects `UsedQuota`, `TotalQuota`, RejectionCount */ public class AdmissionControlMetricsCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { @@ -41,7 +57,15 @@ public AdmissionControlMetricsCollector() { } @Override + @SuppressWarnings("unchecked") public void collectMetrics(long startTime) { + if(!isAdmissionControlFeatureAvailable()) { + LOG.debug("AdmissionControl is not available for this domain"); + PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat( + WriterMetrics.ADMISSION_CONTROL_COLLECTOR_NOT_AVAILABLE, "", 1); + return; + } + long startTimeMillis = System.currentTimeMillis(); try { Class admissionController = Class.forName(ADMISSION_CONTROLLER); @@ -87,11 +111,17 @@ public void collectMetrics(long startTime) { } saveMetricValues(value.toString(), startTime); + + PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat( + WriterMetrics.ADMISSION_CONTROL_COLLECTOR_EXECUTION_TIME, "", + System.currentTimeMillis() - startTimeMillis); + } catch(Exception ex) { PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat( ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, getCollectorName(), System.currentTimeMillis() - startTimeMillis); - LOG.debug("Exception in collecting AdmissionControl Metrics: {} for startTime {}", ex::toString, () -> startTime); + LOG.debug("Exception in collecting AdmissionControl Metrics: {} for startTime {}", + ex::toString, () -> startTime); } } @@ -144,4 +174,13 @@ public long getRejectionCount() { } } + private boolean isAdmissionControlFeatureAvailable() { + try { + Class.forName(ADMISSION_CONTROLLER); + Class.forName(ADMISSION_CONTROL_SERVICE); + } catch (ClassNotFoundException e) { + return false; + } + return true; + } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java index fc92feb4..aca0bfcd 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java @@ -194,4 +194,4 @@ public double getClusterApplierServiceFailed() { return clusterStateAppliedFailedCount; } } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollector.java index 650558ce..334e8c6b 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollector.java @@ -103,4 +103,4 @@ public long getElectionTerm() { return electionTerm; } } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java index 95d555a7..bb2067a8 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java @@ -23,6 +23,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,6 +31,7 @@ import java.util.HashMap; import java.util.List; + @SuppressWarnings("unchecked") public class MasterServiceMetrics extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(MasterServiceMetrics.class).samplingInterval; diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java index b62d3f97..426c9a5a 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java @@ -64,9 +64,10 @@ public static void configureMetrics() { MetricsConfiguration.CONFIG_MAP.put(FaultDetectionMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ShardStateCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(MasterThrottlingMetricsCollector.class, cdefault); - MetricsConfiguration.CONFIG_MAP.put(ShardIndexingPressureMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(MasterClusterStateUpdateStatsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ClusterApplierServiceStatsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(ElectionTermCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(ShardIndexingPressureMetricsCollector.class, cdefault); } // These methods are utility functions for the Node Stat Metrics Collectors. These methods are used by both the all diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollectorTests.java index 8edd8210..979ab6a6 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollectorTests.java @@ -1,28 +1,52 @@ +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; +import static org.junit.Assert.assertEquals; + import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; -import org.junit.Test; - -import java.util.ArrayList; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.TestUtil; import java.util.List; - -import static org.junit.Assert.assertEquals; +import org.junit.Before; +import org.junit.Test; public class AdmissionControlMetricsCollectorTests extends CustomMetricsLocationTestBase { + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + // clean metricQueue before running every test + TestUtil.readEvents(); + System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); + } + @Test public void admissionControlMetricsCollector() { - MetricsConfiguration.CONFIG_MAP.put(AdmissionControlMetricsCollector.class, MetricsConfiguration.cdefault); - AdmissionControlMetricsCollector admissionControlMetricsCollector = new AdmissionControlMetricsCollector(); + MetricsConfiguration.CONFIG_MAP.put( + AdmissionControlMetricsCollector.class, MetricsConfiguration.cdefault); + AdmissionControlMetricsCollector admissionControlMetricsCollector = + new AdmissionControlMetricsCollector(); long startTimeInMills = System.currentTimeMillis(); admissionControlMetricsCollector.saveMetricValues("testMetric", startTimeInMills); - List metrics = new ArrayList<>(); - PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + List metrics = TestUtil.readEvents(); assertEquals(1, metrics.size()); assertEquals("testMetric", metrics.get(0).value); } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollectorTests.java index 7a40cf54..88d47a1d 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollectorTests.java @@ -1,61 +1,173 @@ /* - * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; -import org.junit.Test; -import org.mockito.Mockito; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import org.elasticsearch.cluster.service.ClusterApplierService; +import org.elasticsearch.cluster.service.ClusterService; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - +@RunWith(PowerMockRunner.class) +@PrepareForTest({ESResources.class}) public class ClusterApplierServiceStatsCollectorTests extends CustomMetricsLocationTestBase { + ObjectMapper mapper = new ObjectMapper(); @Test - public void testClusterApplierServiceStatsMetrics() { - MetricsConfiguration.CONFIG_MAP.put(ClusterApplierServiceStatsCollector.class, MetricsConfiguration.cdefault); + public void testClusterApplierServiceStats_saveMetricValues() { + MetricsConfiguration.CONFIG_MAP.put( + ClusterApplierServiceStatsCollector.class, MetricsConfiguration.cdefault); System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); - long startTimeInMills = 1153721339; - PerformanceAnalyzerController controller = Mockito.mock(PerformanceAnalyzerController.class); + PerformanceAnalyzerController controller = + Mockito.mock(PerformanceAnalyzerController.class); ConfigOverridesWrapper configOverrides = Mockito.mock(ConfigOverridesWrapper.class); - Mockito.when(controller.isCollectorEnabled(configOverrides, "ClusterApplierServiceStatsCollector")) + ClusterApplierServiceStatsCollector clusterApplierServiceStatsCollector = + new ClusterApplierServiceStatsCollector(controller, configOverrides); + Mockito.when( + controller.isCollectorEnabled( + configOverrides, "ClusterApplierServiceStatsCollector")) .thenReturn(true); - ClusterApplierServiceStatsCollector clusterApplierServiceStatsCollector = new ClusterApplierServiceStatsCollector( - controller, configOverrides); - clusterApplierServiceStatsCollector.saveMetricValues("testMetric", startTimeInMills); - - List metrics = new ArrayList<>(); + clusterApplierServiceStatsCollector.saveMetricValues( + "cluster_applier_service", startTimeInMills); + List metrics = new ArrayList<>(); PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + assertEquals(1, metrics.size()); - assertEquals("testMetric", metrics.get(0).value); + assertEquals("cluster_applier_service", metrics.get(0).value); try { - clusterApplierServiceStatsCollector.saveMetricValues("Cluster_applier", startTimeInMills, "123"); + clusterApplierServiceStatsCollector.saveMetricValues( + "cluster_applier_service", startTimeInMills, "dummy"); assertTrue("Negative scenario test: Should have been a RuntimeException", true); } catch (RuntimeException ex) { - //- expecting exception...1 values passed; 0 expected + // - expecting exception...1 values passed; 0 expected } } + + @SuppressWarnings("unchecked") + @Ignore + @Test + public void testClusterApplierServiceStats_collectMetrics() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + JsonProcessingException { + System.out.println("test 1"); + MetricsConfiguration.CONFIG_MAP.put( + ClusterApplierServiceStatsCollector.class, MetricsConfiguration.cdefault); + System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); + long startTimeInMills = 1153721339; + PerformanceAnalyzerController controller = + Mockito.mock(PerformanceAnalyzerController.class); + ConfigOverridesWrapper configOverrides = Mockito.mock(ConfigOverridesWrapper.class); + ClusterApplierServiceStatsCollector clusterApplierServiceStatsCollector = + new ClusterApplierServiceStatsCollector(controller, configOverrides); + ClusterApplierServiceStatsCollector spyCollector = + Mockito.spy(clusterApplierServiceStatsCollector); + Mockito.doReturn( + new ClusterApplierServiceStatsCollector.ClusterApplierServiceStats( + 23L, 15L, 2L, -1L)) + .when(spyCollector) + .getClusterApplierServiceStats(); + Mockito.when( + controller.isCollectorEnabled( + configOverrides, + ClusterApplierServiceStatsCollector.class.getSimpleName())) + .thenReturn(true); + + ESResources esResources = Mockito.mock(ESResources.class); + ClusterService clusterService = Mockito.mock(ClusterService.class); + ClusterApplierService clusterApplierService = Mockito.mock(ClusterApplierService.class); + Whitebox.setInternalState(ESResources.class, "INSTANCE", esResources); + Mockito.when(esResources.getClusterService()).thenReturn(clusterService); + Mockito.when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); + + spyCollector.collectMetrics(startTimeInMills); + + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + + assertEquals(1, metrics.size()); + String[] lines = metrics.get(0).value.split(System.lineSeparator()); + Map map = mapper.readValue(lines[1], Map.class); + assertEquals( + 0.6521739130434783, + map.get( + AllMetrics.ClusterApplierServiceStatsValue.CLUSTER_APPLIER_SERVICE_LATENCY + .toString())); + assertEquals( + 2.0, + map.get( + AllMetrics.ClusterApplierServiceStatsValue.CLUSTER_APPLIER_SERVICE_FAILURE + .toString())); + } + + @SuppressWarnings("unchecked") + @Test + public void testClusterApplierServiceStats_collectMetrics_ClassNotFoundException() { + MetricsConfiguration.CONFIG_MAP.put( + ClusterApplierServiceStatsCollector.class, MetricsConfiguration.cdefault); + System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); + long startTimeInMills = 1153721339; + PerformanceAnalyzerController controller = + Mockito.mock(PerformanceAnalyzerController.class); + ConfigOverridesWrapper configOverrides = Mockito.mock(ConfigOverridesWrapper.class); + ClusterApplierServiceStatsCollector clusterApplierServiceStatsCollector = + new ClusterApplierServiceStatsCollector(controller, configOverrides); + ClusterApplierServiceStatsCollector spyCollector = + Mockito.spy(clusterApplierServiceStatsCollector); + Mockito.when( + controller.isCollectorEnabled( + configOverrides, + ClusterApplierServiceStatsCollector.class.getSimpleName())) + .thenReturn(true); + + ESResources esResources = Mockito.mock(ESResources.class); + ClusterService clusterService = Mockito.mock(ClusterService.class); + ClusterApplierService clusterApplierService = Mockito.mock(ClusterApplierService.class); + Whitebox.setInternalState(ESResources.class, "INSTANCE", esResources); + Mockito.when(esResources.getClusterService()).thenReturn(clusterService); + Mockito.when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); + + spyCollector.collectMetrics(startTimeInMills); + + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + // No method found to get cluster state applier thread stats. Skipping + // ClusterApplierServiceStatsCollector. + assertEquals(0, metrics.size()); + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollectorTests.java index f085c13e..ec0ccc96 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ElectionTermCollectorTests.java @@ -1,5 +1,5 @@ /* - * Copyright <2021> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License.