Adds ability to exclude deadlocked threads from exported metrics

Finding deadlocked threads might be an expensive operation if there are a large number of threads in the jvm since it requires a safepoint.
prometheus · Jan 13, 2021 · 02132a3 · 02132a3
1 parent 683466f
commit 02132a3
Show file tree

Hide file tree

Showing 2 changed files with 73 additions and 12 deletions.
diff --git a/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java b/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java
@@ -32,6 +32,7 @@
  */
 public class ThreadExports extends Collector {
   private final ThreadMXBean threadBean;
+  private boolean includeDeadlockedThreads = true;
 
   public ThreadExports() {
     this(ManagementFactory.getThreadMXBean());
@@ -41,6 +42,14 @@ public ThreadExports(ThreadMXBean threadBean) {
     this.threadBean = threadBean;
   }
 
+  public final boolean isIncludeDeadlockedThreads() {
+    return includeDeadlockedThreads;
+  }
+
+  public final void setIncludeDeadlockedThreads(boolean includeDeadlockedThreads) {
+    this.includeDeadlockedThreads = includeDeadlockedThreads;
+  }
+
   void addThreadMetrics(List<MetricFamilySamples> sampleFamilies) {
     sampleFamilies.add(
         new GaugeMetricFamily(
@@ -66,18 +75,20 @@ void addThreadMetrics(List<MetricFamilySamples> sampleFamilies) {
           "Started thread count of a JVM",
           threadBean.getTotalStartedThreadCount()));
 
-    sampleFamilies.add(
-        new GaugeMetricFamily(
-        "jvm_threads_deadlocked",
-        "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers",
-        nullSafeArrayLength(threadBean.findDeadlockedThreads())));
-
-    sampleFamilies.add(
-        new GaugeMetricFamily(
-        "jvm_threads_deadlocked_monitor",
-        "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors",
-        nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads())));
-
+    if (includeDeadlockedThreads) {
+        sampleFamilies.add(
+            new GaugeMetricFamily(
+            "jvm_threads_deadlocked",
+            "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers",
+            nullSafeArrayLength(threadBean.findDeadlockedThreads())));
+
+        sampleFamilies.add(
+            new GaugeMetricFamily(
+            "jvm_threads_deadlocked_monitor",
+            "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors",
+            nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads())));
+    }
+
     GaugeMetricFamily threadStateFamily = new GaugeMetricFamily(
       "jvm_threads_state",
       "Current count of threads by state",

diff --git a/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java b/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java
@@ -10,6 +10,7 @@
 import java.util.Arrays;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.when;
 
 public class ThreadExportsTest {
@@ -97,4 +98,53 @@ public void testThreadPools() {
                     "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL),
             .0000001);
   }
+
+  @Test
+	public void testThreadPoolsNoDeadlocked() {
+    collectorUnderTest.setIncludeDeadlockedThreads(false);
+
+    assertNull(registry.getSampleValue(
+                    "jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL));
+    assertNull(registry.getSampleValue(
+                    "jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL));
+
+    assertEquals(
+            300L,
+            registry.getSampleValue(
+                    "jvm_threads_current", EMPTY_LABEL, EMPTY_LABEL),
+            .0000001);
+    assertEquals(
+            200L,
+            registry.getSampleValue(
+                    "jvm_threads_daemon", EMPTY_LABEL, EMPTY_LABEL),
+            .0000001);
+    assertEquals(
+            301L,
+            registry.getSampleValue(
+                    "jvm_threads_peak", EMPTY_LABEL, EMPTY_LABEL),
+            .0000001);
+    assertEquals(
+            503L,
+            registry.getSampleValue(
+                    "jvm_threads_started_total", EMPTY_LABEL, EMPTY_LABEL),
+            .0000001);
+
+    assertEquals(
+            1L,
+            registry.getSampleValue(
+                    "jvm_threads_state", STATE_LABEL, STATE_BLOCKED_LABEL),
+            .0000001);
+
+    assertEquals(
+            2L,
+            registry.getSampleValue(
+                    "jvm_threads_state", STATE_LABEL, STATE_RUNNABLE_LABEL),
+            .0000001);
+
+    assertEquals(
+            0L,
+            registry.getSampleValue(
+                    "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL),
+            .0000001);
+  }
 }