Use semantic conventions for GC metrics + fix lifecycle of Otel metrics instruments recreating metrics instruments upon reconfiguration of Otel SDK. (#426)

* Use semantic conventions for GC metrics + fix lifecycle of Otel metrics instruments recreating metrics instruments upon reconfiguration of Otel SDK.
Old GC metric names: "runtime.jvm.memory.area"
New GC metric names: "process.runtime.jvm.memory.*"
* Disable broken unit tests
* Bump dependencies
* Use smaller GitHub repository in tests. Fix #427
Co-authored-by: Ivan Fernandez Calvo <[email protected]>

Author: cyrille-leclerc

pom.xml

@@ -39,7 +39,7 @@
             <dependency>
                 <groupId>io.jenkins.tools.bom</groupId>
                 <artifactId>bom-2.289.x</artifactId>
-                <version>1280.vd669827e38cd</version>
+                <version>1342.v729ca_3818e88</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

src/main/java/io/jenkins/plugins/opentelemetry/AbstractOtelComponent.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright The Original Author or Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.jenkins.plugins.opentelemetry;
+
+import io.opentelemetry.api.metrics.ObservableDoubleGauge;
+import io.opentelemetry.api.metrics.ObservableLongCounter;
+import io.opentelemetry.api.metrics.ObservableLongGauge;
+import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
+
+import java.util.logging.Logger;
+
+/**
+ * Helper to implement {@link OtelComponent} and to manage metric instruments
+ */
+public abstract class AbstractOtelComponent implements OtelComponent {
+    
+    private final OtelComponent.State state = new OtelComponent.State();
+
+    protected void registerInstrument(ObservableLongCounter instrument) {
+        state.registerInstrument(instrument);
+    }
+
+    protected void registerInstrument(ObservableLongGauge instrument) {
+        state.registerInstrument(instrument);
+    }
+
+    protected void registerInstrument(ObservableLongUpDownCounter instrument) {
+        state.registerInstrument(instrument);
+    }
+
+    protected void registerInstrument(ObservableDoubleGauge instrument) {
+        state.registerInstrument(instrument);
+    }
+
+    @Override
+    public void beforeSdkShutdown() {
+        state.closeInstruments();
+    }
+}

src/main/java/io/jenkins/plugins/opentelemetry/OpenTelemetrySdkProvider.java

@@ -9,6 +9,8 @@
 import com.google.common.base.Preconditions;
 import hudson.Extension;
 import hudson.ExtensionList;
+import hudson.init.InitMilestone;
+import hudson.init.Initializer;
 import io.jenkins.plugins.opentelemetry.opentelemetry.GlobalOpenTelemetrySdk;
 import io.jenkins.plugins.opentelemetry.opentelemetry.autoconfigure.ConfigPropertiesUtils;
 import io.jenkins.plugins.opentelemetry.opentelemetry.trace.TracerDelegate;
@@ -99,6 +101,10 @@ protected OpenTelemetrySdk getOpenTelemetrySdk() {
     @PreDestroy
     public void preDestroy() {
         if (this.openTelemetrySdk != null) {
+            for(OtelComponent otelComponent: ExtensionList.lookup(OtelComponent.class)) {
+                LOGGER.log(Level.FINE, () -> "beforeSdkShutdown() " + otelComponent);
+                otelComponent.beforeSdkShutdown();
+            }
             this.openTelemetrySdk.getSdkTracerProvider().shutdown();
             this.openTelemetrySdk.getSdkMeterProvider().shutdown();
             this.openTelemetrySdk.getSdkLogEmitterProvider().shutdown();
@@ -159,6 +165,14 @@ public void initializeNoOp() {
         LOGGER.log(Level.FINE, "OpenTelemetry initialized as NoOp");
     }
 
+    @Initializer(after = InitMilestone.JOB_CONFIG_ADAPTED)
+    public void postInitialize() {
+        for (OtelComponent otelComponent : ExtensionList.lookup(OtelComponent.class)) {
+            LOGGER.log(Level.FINE, () -> "Initialize Otel SDK on " + otelComponent);
+            otelComponent.afterSdkInitialized(meter, logEmitter, tracer, config);
+        }
+    }
+
     static public OpenTelemetrySdkProvider get() {
         return ExtensionList.lookupSingleton(OpenTelemetrySdkProvider.class);
     }

src/main/java/io/jenkins/plugins/opentelemetry/OtelComponent.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright The Original Author or Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.jenkins.plugins.opentelemetry;
+
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableDoubleGauge;
+import io.opentelemetry.api.metrics.ObservableLongCounter;
+import io.opentelemetry.api.metrics.ObservableLongGauge;
+import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import io.opentelemetry.sdk.logs.LogEmitter;
+import io.opentelemetry.sdk.logs.SdkLogEmitterProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Interface for components that want to be notified when the Otel SDK has been initialized or will be shutdown.
+ *
+ * The life cycle of consumers of the OpenTelemetry SDK (consumers of {@link io.opentelemetry.api.trace.TracerProvider},
+ * {@link io.opentelemetry.api.metrics.MeterProvider}, and {@link SdkLogEmitterProvider}) can NOT use the Jenkins life
+ * cycle because those consumers of the Otel SDK need to perform initialization tasks after the Otel SDK has been
+ * initialized and have to shut down things before the Otel SDK is shutdown due to a reconfiguration.
+ *
+ * Used by components that create counters...
+ */
+public interface OtelComponent {
+
+    /**
+     * Invoked soon after the Otel SDK has been initialized.
+     *
+     * @param meter {@link Meter} of the newly initialized Otel SDK
+     * @param logEmitter {@link LogEmitter} of the newly initialized Otel SDK
+     * @param tracer {@link Tracer} of the newly initialized Otel SDK
+     * @param configProperties {@link ConfigProperties} of the newly initialized Otel SDK
+     */
+    void afterSdkInitialized(Meter meter, LogEmitter logEmitter, Tracer tracer, ConfigProperties configProperties);
+
+    /**
+     * Invoked just before the Otel SDK is shutdown
+     */
+    void beforeSdkShutdown();
+
+    /**
+     * Helper for {@link OtelComponent} implementations to manage the created metric instruments
+     */
+    class State {
+        private final static Logger logger = Logger.getLogger(State.class.getName());
+        private final List<AutoCloseable> instruments = new ArrayList<>();
+
+        public void registerInstrument(ObservableLongCounter instrument) {
+            instruments.add(instrument);
+        }
+
+        public void registerInstrument(ObservableLongGauge instrument) {
+            instruments.add(instrument);
+        }
+
+        public void registerInstrument(ObservableLongUpDownCounter instrument) {
+            instruments.add(instrument);
+        }
+
+        public void registerInstrument(ObservableDoubleGauge instrument) {
+            instruments.add(instrument);
+        }
+
+        public void closeInstruments() {
+            List<AutoCloseable> instruments = this.instruments;
+            this.instruments.clear(); // reset the list of instruments for reuse
+            for (AutoCloseable instrument : instruments) {
+                try {
+                    instrument.close();
+                } catch (Exception e) {
+                    // should never happen, Otel instruments override the #close method to indicate they don't throw exceptions
+                    logger.log(Level.INFO, "Exception closing instrument " + instrument, e);
+                }
+            }
+        }
+    }
+}

src/main/java/io/jenkins/plugins/opentelemetry/computer/MonitoringCloudListener.java

@@ -11,9 +11,13 @@
 import hudson.slaves.CloudProvisioningListener;
 import hudson.slaves.NodeProvisioner;
 import io.jenkins.plugins.opentelemetry.OpenTelemetrySdkProvider;
+import io.jenkins.plugins.opentelemetry.OtelComponent;
 import io.jenkins.plugins.opentelemetry.semconv.JenkinsSemanticMetrics;
 import io.opentelemetry.api.metrics.LongCounter;
 import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import io.opentelemetry.sdk.logs.LogEmitter;
 
 import javax.annotation.Nonnull;
 import javax.annotation.PostConstruct;
@@ -22,16 +26,14 @@
 import java.util.logging.Logger;
 
 @Extension
-public class MonitoringCloudListener extends CloudProvisioningListener {
+public class MonitoringCloudListener extends CloudProvisioningListener implements OtelComponent {
     private final static Logger LOGGER = Logger.getLogger(MonitoringCloudListener.class.getName());
 
-    protected Meter meter;
-
     private LongCounter failureCloudCounter;
     private LongCounter totalCloudCount;
 
-    @PostConstruct
-    public void postConstruct() {
+    @Override
+    public void afterSdkInitialized(Meter meter, LogEmitter logEmitter, Tracer tracer, ConfigProperties configProperties) {
         failureCloudCounter = meter.counterBuilder(JenkinsSemanticMetrics.JENKINS_CLOUD_AGENTS_FAILURE)
             .setDescription("Number of failed cloud agents when provisioning")
             .setUnit("1")
@@ -61,11 +63,8 @@ public void onComplete(NodeProvisioner.PlannedNode plannedNode, Node node) {
         LOGGER.log(Level.FINE, () -> "onComplete(" + plannedNode + ")");
     }
 
-    /**
-     * Jenkins doesn't support {@link com.google.inject.Provides} so we manually wire dependencies :-(
-     */
-    @Inject
-    public void setMeter(@Nonnull OpenTelemetrySdkProvider openTelemetrySdkProvider) {
-        this.meter = openTelemetrySdkProvider.getMeter();
+    @Override
+    public void beforeSdkShutdown() {
+
     }
 }

src/main/java/io/jenkins/plugins/opentelemetry/computer/MonitoringComputerListener.java

@@ -12,19 +12,19 @@
 import hudson.remoting.Channel;
 import hudson.slaves.ComputerListener;
 import io.jenkins.plugins.opentelemetry.OpenTelemetryAttributesAction;
-import io.jenkins.plugins.opentelemetry.OpenTelemetrySdkProvider;
+import io.jenkins.plugins.opentelemetry.OtelComponent;
 import io.jenkins.plugins.opentelemetry.semconv.JenkinsOtelSemanticAttributes;
 import io.jenkins.plugins.opentelemetry.semconv.JenkinsSemanticMetrics;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.metrics.LongCounter;
 import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import io.opentelemetry.sdk.logs.LogEmitter;
 import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
 import jenkins.model.Jenkins;
 import jenkins.security.MasterToSlaveCallable;
 
-import javax.annotation.Nonnull;
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.Arrays;
@@ -34,15 +34,15 @@
 import java.util.logging.Logger;
 
 @Extension
-public class MonitoringComputerListener extends ComputerListener {
+public class MonitoringComputerListener extends ComputerListener implements OtelComponent {
     private final static Logger LOGGER = Logger.getLogger(MonitoringComputerListener.class.getName());
 
-    protected Meter meter;
-
     private LongCounter failureAgentCounter;
 
-    @PostConstruct
-    public void postConstruct() {
+    private OtelComponent.State state = new State();
+
+    @Override
+    public void afterSdkInitialized(Meter meter, LogEmitter logEmitter, Tracer tracer, ConfigProperties configProperties) {
         final Jenkins jenkins = Jenkins.get();
         Computer controllerComputer = jenkins.getComputer("");
         if (controllerComputer == null) {
@@ -65,21 +65,24 @@ public void postConstruct() {
                 LOGGER.log(Level.WARNING, "Failure getting attributes for Jenkins Controller computer " + controllerComputer, e);
             }
         }
+        state.registerInstrument(
         meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_OFFLINE)
             .ofLongs()
             .setDescription("Number of offline agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getOfflineAgentsCount()));
-        meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_ONLINE)
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getOfflineAgentsCount())));
+        state.registerInstrument(
+            meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_ONLINE)
             .ofLongs()
             .setDescription("Number of online agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getOnlineAgentsCount()));
-        meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_TOTAL)
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getOnlineAgentsCount())));
+        state.registerInstrument(
+            meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_TOTAL)
             .ofLongs()
             .setDescription("Number of agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getAgentsCount()));
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getAgentsCount())));
         failureAgentCounter = meter.counterBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_LAUNCH_FAILURE)
             .setDescription("Number of ComputerLauncher failures")
             .setUnit("1")
@@ -130,6 +133,11 @@ public void onLaunchFailure(Computer computer, TaskListener taskListener) throws
         LOGGER.log(Level.FINE, () -> "onLaunchFailure(" + computer + "): ");
     }
 
+    @Override
+    public void beforeSdkShutdown() {
+        state.closeInstruments();
+    }
+
     private static class GetComputerAttributes extends MasterToSlaveCallable<Map<String, String>, IOException> {
         @Override
         public Map<String, String> call() throws IOException {
@@ -152,12 +160,4 @@ public Map<String, String> call() throws IOException {
             return attributes;
         }
     }
-
-    /**
-     * Jenkins doesn't support {@link com.google.inject.Provides} so we manually wire dependencies :-(
-     */
-    @Inject
-    public void setMeter(@Nonnull OpenTelemetrySdkProvider openTelemetrySdkProvider) {
-        this.meter = openTelemetrySdkProvider.getMeter();
-    }
 }