Allow dynamic update of agent parameters

Made the agent profilers re-entrant allowing to attach and update profiling parameters at runtime This is useful when the VM is not started with the agent but can be later attached and controlled, e.g: - update profiling interval - enabling/disabling profilers For time being only profilers parameters can be updated. Transformers, reporter not
uber-common · May 2, 2019 · d5b0ade · d5b0ade
1 parent cb9b289
commit d5b0ade
Show file tree

Hide file tree

Showing 23 changed files with 375 additions and 257 deletions.
diff --git a/attach-client.sh b/attach-client.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+AGENT_PATH=`ls -1 $DIR/target/jvm-profiler-*.jar`
+echo $AGENT_PATH
+
+java -cp $AGENT_PATH:$JAVA_HOME/lib/tools.jar com.uber.profiling.tools.AttachClient $AGENT_PATH $*
+
diff --git a/pom.xml b/pom.xml
@@ -81,7 +81,13 @@
             <version>4.8.1</version>
             <scope>test</scope>
         </dependency>
-
+
+        <!-- required for AttachClient build -->
+        <dependency>
+            <groupId>com.github.olivergondza</groupId>
+            <artifactId>maven-jdk-tools-wrapper</artifactId>
+            <version>0.1</version>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/src/main/java/com/uber/profiling/AgentImpl.java b/src/main/java/com/uber/profiling/AgentImpl.java
diff --git a/src/main/java/com/uber/profiling/Arguments.java b/src/main/java/com/uber/profiling/Arguments.java
@@ -23,7 +23,6 @@
 import com.uber.profiling.util.DummyConfigProvider;
 import com.uber.profiling.util.JsonUtils;
 import com.uber.profiling.util.ReflectionUtils;
-
 import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -59,6 +58,7 @@ public class Arguments {
     private boolean noop = false;
 
     private Constructor<Reporter> reporterConstructor;
+    private Reporter reporter;
     private Constructor<ConfigProvider> configProviderConstructor;
     private String configFile;
 
@@ -260,11 +260,14 @@ public Map<String, List<String>> getRawArgValues() {
     }
 
     public Reporter getReporter() {
+        if (reporter != null) {
+            return reporter;
+        }
         if (reporterConstructor == null) {
             return new ConsoleOutputReporter();
         } else {
             try {
-                Reporter reporter = reporterConstructor.newInstance();
+                reporter = reporterConstructor.newInstance();
                 reporter.updateArguments(getRawArgValues());
                 return reporter;
             } catch (Throwable e) {

diff --git a/src/main/java/com/uber/profiling/Profiler.java b/src/main/java/com/uber/profiling/Profiler.java
@@ -16,10 +16,45 @@
 
 package com.uber.profiling;
 
-public interface Profiler {
-    long getIntervalMillis();
+import com.uber.profiling.util.AgentLogger;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicLong;
 
-    void setReporter(Reporter reporter);
+public abstract class Profiler implements Runnable {
 
-    void profile();
+  private static final AgentLogger logger = AgentLogger.getLogger(Profiler.class.getName());
+  private static final int MAX_ERROR_COUNT_TO_LOG = 100;
+
+  private final AtomicLong errorCounter = new AtomicLong(0);
+  private ScheduledFuture<?> scheduleHandler;
+
+  public abstract long getIntervalMillis();
+
+  public abstract void setIntervalMillis(long millis);
+
+  public abstract void setReporter(Reporter reporter);
+
+  public abstract void profile();
+
+  @Override
+  public void run() {
+    try {
+      this.profile();
+    } catch (Throwable e) {
+      long errorCountValue = errorCounter.incrementAndGet();
+      if (errorCountValue <= MAX_ERROR_COUNT_TO_LOG) {
+        logger.warn("Failed to run profile: " + this, e);
+      } else {
+        e.printStackTrace();
+      }
+    }
+  }
+
+  public ScheduledFuture<?> getScheduleHandler() {
+    return scheduleHandler;
+  }
+
+  public void setScheduleHandler(ScheduledFuture<?> handler) {
+    this.scheduleHandler = handler;
+  }
 }
diff --git a/src/main/java/com/uber/profiling/ProfilerGroup.java b/src/main/java/com/uber/profiling/ProfilerGroup.java
diff --git a/src/main/java/com/uber/profiling/ProfilerRunner.java b/src/main/java/com/uber/profiling/ProfilerRunner.java
diff --git a/src/main/java/com/uber/profiling/ShutdownHookRunner.java b/src/main/java/com/uber/profiling/ShutdownHookRunner.java
@@ -42,6 +42,10 @@ public void run() {
 
         for (Profiler profiler : profilers) {
             try {
+                if (profiler.getIntervalMillis() <=0 ){
+                    //one time profiler, skip
+                    continue;
+                }
                 logShutdownMessage("Running periodic profiler (last run): " + profiler);
                 profiler.profile();
                 logShutdownMessage("Ran periodic profiler (last run): " + profiler);

diff --git a/src/main/java/com/uber/profiling/profilers/CpuAndMemoryProfiler.java b/src/main/java/com/uber/profiling/profilers/CpuAndMemoryProfiler.java
@@ -16,7 +16,6 @@
 
 package com.uber.profiling.profilers;
 
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.util.AgentLogger;
 import com.uber.profiling.util.ProcFileUtils;
@@ -36,7 +35,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class CpuAndMemoryProfiler extends ProfilerBase implements Profiler {
+public class CpuAndMemoryProfiler extends ProfilerBase {
     public final static String PROFILER_NAME = "CpuAndMemory";
 
     private static final AgentLogger logger = AgentLogger.getLogger(CpuAndMemoryProfiler.class.getName());
@@ -70,6 +69,7 @@ public long getIntervalMillis() {
         return intervalMillis;
     }
 
+    @Override
     public void setIntervalMillis(long intervalMillis) {
         this.intervalMillis = intervalMillis;
     }

diff --git a/src/main/java/com/uber/profiling/profilers/IOProfiler.java b/src/main/java/com/uber/profiling/profilers/IOProfiler.java
@@ -16,15 +16,14 @@
 
 package com.uber.profiling.profilers;
 
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.util.ProcFileUtils;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-public class IOProfiler extends ProfilerBase implements Profiler {
+public class IOProfiler extends ProfilerBase {
     public final static String PROFILER_NAME = "IO";
 
     private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL;

diff --git a/src/main/java/com/uber/profiling/profilers/MethodArgumentCollector.java b/src/main/java/com/uber/profiling/profilers/MethodArgumentCollector.java
@@ -19,6 +19,7 @@
 import com.uber.profiling.util.ClassMethodArgumentMetricBuffer;
 
 public class MethodArgumentCollector {
+    public static final String PROFILER_NAME = "MethodArgumentCollector";
     private ClassMethodArgumentMetricBuffer buffer;
 
     public MethodArgumentCollector(ClassMethodArgumentMetricBuffer buffer) {

diff --git a/src/main/java/com/uber/profiling/profilers/MethodArgumentProfiler.java b/src/main/java/com/uber/profiling/profilers/MethodArgumentProfiler.java
@@ -16,7 +16,6 @@
 
 package com.uber.profiling.profilers;
 
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.reporters.ConsoleOutputReporter;
 import com.uber.profiling.util.ClassAndMethodMetricKey;
@@ -26,7 +25,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
-public class MethodArgumentProfiler extends ProfilerBase implements Profiler {
+public class MethodArgumentProfiler extends ProfilerBase {
     public static final String PROFILER_NAME = "MethodArgument";
 
     private ClassMethodArgumentMetricBuffer buffer;

diff --git a/src/main/java/com/uber/profiling/profilers/MethodDurationProfiler.java b/src/main/java/com/uber/profiling/profilers/MethodDurationProfiler.java
@@ -16,7 +16,6 @@
 
 package com.uber.profiling.profilers;
 
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.reporters.ConsoleOutputReporter;
 import com.uber.profiling.util.ClassAndMethodLongMetricBuffer;
@@ -26,7 +25,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class MethodDurationProfiler extends ProfilerBase implements Profiler {
+public class MethodDurationProfiler extends ProfilerBase {
     public static final String PROFILER_NAME = "MethodDuration";
 
     private ClassAndMethodLongMetricBuffer buffer;

diff --git a/src/main/java/com/uber/profiling/profilers/ProcessInfoProfiler.java b/src/main/java/com/uber/profiling/profilers/ProcessInfoProfiler.java
@@ -17,7 +17,6 @@
 package com.uber.profiling.profilers;
 
 import com.uber.profiling.AgentImpl;
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.util.AgentLogger;
 import com.uber.profiling.util.ProcFileUtils;
@@ -30,7 +29,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class ProcessInfoProfiler extends ProfilerBase implements Profiler {
+public class ProcessInfoProfiler extends ProfilerBase {
     public final static String PROFILER_NAME = "ProcessInfo";
 
     private static final AgentLogger logger = AgentLogger.getLogger(ProcessInfoProfiler.class.getName());
@@ -50,7 +49,14 @@ public ProcessInfoProfiler(Reporter reporter) {
 
     @Override
     public long getIntervalMillis() {
-        return 0;
+        return -1;
+    }
+
+    @Override
+    public void setIntervalMillis(long millis) {
+        if (millis > 0) {
+            throw new IllegalStateException("Cannot schedule " + PROFILER_NAME + " profiler");
+        }
     }
 
     @Override

diff --git a/src/main/java/com/uber/profiling/profilers/ProfilerBase.java b/src/main/java/com/uber/profiling/profilers/ProfilerBase.java
@@ -16,14 +16,15 @@
 
 package com.uber.profiling.profilers;
 
+import com.uber.profiling.Profiler;
 import com.uber.profiling.util.NetworkUtils;
 import com.uber.profiling.util.ProcFileUtils;
 import com.uber.profiling.util.ProcessUtils;
 import com.uber.profiling.util.SparkUtils;
 
 import java.util.UUID;
 
-public class ProfilerBase {
+public abstract class ProfilerBase extends Profiler {
     private String tag = null;
     private String cluster = null;
     private String hostName = null;

diff --git a/src/main/java/com/uber/profiling/profilers/StacktraceCollectorProfiler.java b/src/main/java/com/uber/profiling/profilers/StacktraceCollectorProfiler.java
@@ -31,7 +31,8 @@
 /**
  * This class collects stacktraces by getting thread dump via JMX, and stores the stacktraces into the given buffer.
  */
-public class StacktraceCollectorProfiler implements Profiler {
+public class StacktraceCollectorProfiler extends Profiler {
+    public static final String PROFILER_NAME = "StacktraceCollector";
     private long intervalMillis;
     private StacktraceMetricBuffer buffer;
     private String ignoreThreadNamePrefix = "";
@@ -63,6 +64,9 @@ public void setReporter(Reporter reporter) {
 
     @Override
     public void profile() {
+        if (getIntervalMillis() <=0){
+            return;
+        }
         ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(false, false);
         if (threadInfos == null) {
             return;

diff --git a/src/main/java/com/uber/profiling/profilers/StacktraceReporterProfiler.java b/src/main/java/com/uber/profiling/profilers/StacktraceReporterProfiler.java
@@ -16,7 +16,6 @@
 
 package com.uber.profiling.profilers;
 
-import com.uber.profiling.Profiler;
 import com.uber.profiling.Reporter;
 import com.uber.profiling.reporters.ConsoleOutputReporter;
 import com.uber.profiling.util.ClassAndMethod;
@@ -32,7 +31,7 @@
 /**
  * This class reads the stacktraces from the given buffer and send out via given reporter.
  */
-public class StacktraceReporterProfiler extends ProfilerBase implements Profiler {
+public class StacktraceReporterProfiler extends ProfilerBase {
     public static final String PROFILER_NAME = "Stacktrace";
 
     private StacktraceMetricBuffer buffer;
@@ -41,6 +40,10 @@ public class StacktraceReporterProfiler extends ProfilerBase implements Profiler
 
     private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL;
 
+    public StacktraceMetricBuffer getBuffer() {
+        return buffer;
+    }
+
     public StacktraceReporterProfiler(StacktraceMetricBuffer buffer, Reporter reporter) {
         this.buffer = buffer;
         this.reporter = reporter;

diff --git a/src/main/java/com/uber/profiling/reporters/ConsoleOutputReporter.java b/src/main/java/com/uber/profiling/reporters/ConsoleOutputReporter.java
@@ -18,8 +18,6 @@
 
 import com.uber.profiling.Reporter;
 import com.uber.profiling.util.JsonUtils;
-
-import java.util.List;
 import java.util.Map;
 
 public class ConsoleOutputReporter implements Reporter {