diff --git a/docs/http-requests-traces.md b/docs/http-requests-traces.md index ced9bb32..0a23f475 100644 --- a/docs/http-requests-traces.md +++ b/docs/http-requests-traces.md @@ -14,6 +14,12 @@ Monitor and troubleshoot performance problems of the Jenkins Controller using tr ``` otel.instrumentation.jenkins.web.enabled=false ``` +* HTTP request parameters can be captured using the standard configuration parameter + ([docs](https://opentelemetry.io/docs/zero-code/java/agent/configuration/#capturing-servlet-request-parameters)): + +``` +otel.instrumentation.servlet.experimental.capture-request-parameters=<> +``` * Observability solutions provide aggregated views on the overall activity on the Jenkins Controller UI, often enabling monitoring dashboards, alerting, and automated anomaly detection diff --git a/pom.xml b/pom.xml index d855eca7..14a6992d 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,8 @@ Monitor, troubleshoot and observe Jenkins with OpenTelemetry. Monitor Jenkins with dashboards and alerts on critical health metrics. Troubleshoot Jenkins problems using traces of job executions and HTTP requests. - Store pipeline build logs in an Observability backend like Elastic to improve Jenkins reliability and scalability while improving the traceability of builds. + Store pipeline build logs in an Observability backend like Elastic or Grafana/Loki + to improve Jenkins reliability and scalability while improving the traceability of builds. https://github.com/jenkinsci/${project.artifactId}-plugin @@ -73,6 +74,20 @@ pom import + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom + ${opentelemetry-instrumentation.version} + pom + import + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom-alpha + ${opentelemetry-instrumentation.version}-alpha + pom + import + jakarta.json jakarta.json-api @@ -83,6 +98,11 @@ parsson 1.1.6 + + com.jayway.jsonpath + json-path + 2.9.0 + io.prometheus simpleclient_bom @@ -157,7 +177,6 @@ io.opentelemetry.instrumentation opentelemetry-runtime-telemetry-java17 - ${opentelemetry-instrumentation-alpha.version} --> @@ -264,6 +283,14 @@ org.jenkins-ci.plugins.workflow workflow-durable-task-step + + org.jenkins-ci.plugins + apache-httpcomponents-client-4-api + + + io.opentelemetry.instrumentation + opentelemetry-apache-httpclient-4.3 + org.jenkins-ci.plugins cloudbees-disk-usage-simple diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackend.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackend.java index acae930c..9dea9a34 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackend.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackend.java @@ -56,7 +56,7 @@ public String getDisplayName() { private transient Template buildLogsVisualizationUrlGTemplate; - protected LokiOTelLogFormat lokiOTelLogFormat = LokiOTelLogFormat.LOKI_V2_JSON_OTEL_FORMAT; + protected LokiOTelLogFormat lokiOTelLogFormat; /** * Returns {@code null} if the backend is not capable of retrieving logs(ie the {@link NoGrafanaLogsBackend} @@ -155,7 +155,7 @@ public DescriptorImpl getDescriptor() { @NonNull public String getLokiOTelLogFormat() { - return Optional.ofNullable(lokiOTelLogFormat).map(Enum::name).orElse(LokiOTelLogFormat.LOKI_V2_JSON_OTEL_FORMAT.name()); + return Optional.ofNullable(lokiOTelLogFormat).map(Enum::name).orElse(getDescriptor().getDefaultLokiOTelLogFormat()); } @DataBoundSetter @@ -178,5 +178,7 @@ public ListBoxModel doFillLokiOTelLogFormatItems() { } return items; } + + public abstract String getDefaultLokiOTelLogFormat(); } } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendBackendWithLogMirroringInJenkins.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendBackendWithLogMirroringInJenkins.java index f6d6b427..13866506 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendBackendWithLogMirroringInJenkins.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendBackendWithLogMirroringInJenkins.java @@ -56,6 +56,11 @@ public int hashCode() { @Extension(ordinal = 100) public static class DescriptorImpl extends GrafanaLogsBackend.DescriptorImpl { + @Override + public String getDefaultLokiOTelLogFormat() { + return LokiOTelLogFormat.LOKI_V2_JSON_OTEL_FORMAT.name(); + } + @Nonnull @Override public String getDisplayName() { diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization.java new file mode 100644 index 00000000..76789683 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization.java @@ -0,0 +1,263 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import com.cloudbees.plugins.credentials.common.StandardListBoxModel; +import com.cloudbees.plugins.credentials.common.StandardUsernameCredentials; +import com.google.errorprone.annotations.MustBeClosed; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.Extension; +import hudson.model.Item; +import hudson.security.ACL; +import hudson.util.FormValidation; +import hudson.util.ListBoxModel; +import io.jenkins.plugins.opentelemetry.JenkinsControllerOpenTelemetry; +import io.jenkins.plugins.opentelemetry.TemplateBindingsProvider; +import io.jenkins.plugins.opentelemetry.backend.GrafanaBackend; +import io.jenkins.plugins.opentelemetry.backend.ObservabilityBackend; +import io.jenkins.plugins.opentelemetry.jenkins.CredentialsNotFoundException; +import io.jenkins.plugins.opentelemetry.jenkins.JenkinsCredentialsToApacheHttpCredentialsAdapter; +import io.jenkins.plugins.opentelemetry.job.log.LogStorageRetriever; +import io.opentelemetry.api.OpenTelemetry; +import jenkins.model.Jenkins; +import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.Credentials; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; + import org.kohsuke.stapler.verb.POST; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.logging.Logger; + +public class GrafanaLogsBackendWithJenkinsVisualization extends GrafanaLogsBackend implements TemplateBindingsProvider { + private final static String MSG_LOKI_URL_IS_BLANK = "Loki URL is blank, logs will not be stored in Elasticsearch"; + private final static Logger logger = Logger.getLogger(GrafanaLogsBackendWithJenkinsVisualization.class.getName()); + + private String grafanaLokiDatasourceIdentifier = GrafanaBackend.DEFAULT_LOKI_DATA_SOURCE_IDENTIFIER; + + private String lokiUrl; + private boolean disableSslVerifications; + private String lokiCredentialsId; + private String lokiTenantId; + + @DataBoundConstructor + public GrafanaLogsBackendWithJenkinsVisualization() { + + } + + public String getGrafanaLokiDatasourceIdentifier() { + return grafanaLokiDatasourceIdentifier; + } + + @DataBoundSetter + public void setGrafanaLokiDatasourceIdentifier(String grafanaLokiDatasourceIdentifier) { + this.grafanaLokiDatasourceIdentifier = grafanaLokiDatasourceIdentifier; + } + + @Override + @MustBeClosed + public LogStorageRetriever newLogStorageRetriever(TemplateBindingsProvider templateBindingsProvider) { + if (StringUtils.isBlank(lokiUrl)) { + throw new IllegalStateException(MSG_LOKI_URL_IS_BLANK); + } + // TODO shall we inject this through @Inject? + OpenTelemetry openTelemetry = JenkinsControllerOpenTelemetry.get(); + + String serviceName = templateBindingsProvider.getBindings().get(ObservabilityBackend.TemplateBindings.SERVICE_NAME).toString(); + Optional serviceNamespace = Optional.ofNullable(templateBindingsProvider.getBindings().get(ObservabilityBackend.TemplateBindings.SERVICE_NAMESPACE)).map(Object::toString); + Optional lokiTenantId = Optional.ofNullable(this.lokiTenantId).filter(StringUtils::isNotBlank); + return new LokiLogStorageRetriever( + lokiUrl, + disableSslVerifications, + getLokiApacheHttpCredentials(lokiCredentialsId), lokiTenantId, + getBuildLogsVisualizationUrlTemplate(), + TemplateBindingsProvider.compose(templateBindingsProvider, this.getBindings()), + serviceName, + serviceNamespace, + openTelemetry); + } + + public String getLokiUrl() { + return lokiUrl; + } + + @DataBoundSetter + public void setLokiUrl(String lokiUrl) { + this.lokiUrl = lokiUrl; + } + + public boolean isDisableSslVerifications() { + return disableSslVerifications; + } + + + @DataBoundSetter + public void setDisableSslVerifications(boolean disableSslVerifications) { + this.disableSslVerifications = disableSslVerifications; + } + + public String getLokiTenantId() { + return lokiTenantId; + } + + @DataBoundSetter + public void setLokiTenantId(String lokiTenantId) { + this.lokiTenantId = lokiTenantId; + } + + public String getLokiCredentialsId() { + return lokiCredentialsId; + } + + /** + * + * @param lokiCredentialsId Jenkins credentials id + */ + @NonNull + protected static Optional getLokiApacheHttpCredentials(@Nullable String lokiCredentialsId) { + return Optional.ofNullable(lokiCredentialsId).filter(StringUtils::isNotBlank).map(JenkinsCredentialsToApacheHttpCredentialsAdapter::new); + } + + @DataBoundSetter + public void setLokiCredentialsId(String lokiCredentialsId) { + this.lokiCredentialsId = lokiCredentialsId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GrafanaLogsBackendWithJenkinsVisualization that = (GrafanaLogsBackendWithJenkinsVisualization) o; + return Objects.equals(grafanaLokiDatasourceIdentifier, that.grafanaLokiDatasourceIdentifier); + } + + @Override + public int hashCode() { + return Objects.hash(grafanaLokiDatasourceIdentifier); + } + + @Override + public String toString() { + return "GrafanaLogsBackendWithJenkinsVisualization{" + + "grafanaLokiDatasourceIdentifier='" + grafanaLokiDatasourceIdentifier + '\'' + + ", lokiUrl='" + lokiUrl + '\'' + + ", disableSslVerifications=" + disableSslVerifications + + ", lokiCredentialsId='" + lokiCredentialsId + '\'' + + '}'; + } + + @Override + public Map getBindings() { + return Map.of( + GrafanaBackend.TemplateBindings.GRAFANA_LOKI_DATASOURCE_IDENTIFIER, getGrafanaLokiDatasourceIdentifier()); + } + + @Extension(ordinal = 50) + public static class DescriptorImpl extends GrafanaLogsBackend.DescriptorImpl { + + public FormValidation doCheckLokiUrl(@QueryParameter("lokiUrl") String url) { + if (StringUtils.isEmpty(url)) { + return FormValidation.ok(); + } + try { + new URI(url).toURL(); + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { + return FormValidation.error("Invalid Loki URL: " + e.getMessage()); + } + return FormValidation.ok(); + } + + public ListBoxModel doFillLokiCredentialsIdItems(Item context, @QueryParameter String lokiCredentialsId) { + if (context == null && !Jenkins.get().hasPermission(Jenkins.ADMINISTER) + || context != null && !context.hasPermission(context.CONFIGURE)) { + return new StandardListBoxModel(); + } + + return new StandardListBoxModel().includeEmptyValue() + .includeAs(ACL.SYSTEM, context, StandardUsernameCredentials.class) + .includeCurrentValue(lokiCredentialsId); + } + + @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT", + justification = "We don't care about the return value, we just want to check that the credentials are valid") + @POST + public FormValidation doCheckLokiCredentialsId(Item context, @QueryParameter String lokiCredentialsId) { + Jenkins.get().checkPermission(Jenkins.ADMINISTER); + + if (context == null && !Jenkins.get().hasPermission(Jenkins.ADMINISTER) + || context != null && !context.hasPermission(context.CONFIGURE)) { + return FormValidation.ok(); + } + + if (lokiCredentialsId == null || lokiCredentialsId.isEmpty()) { + return FormValidation.ok(); // support anonymous access + } + try { + new JenkinsCredentialsToApacheHttpCredentialsAdapter(lokiCredentialsId) + .getUserPrincipal().getName(); + } catch (CredentialsNotFoundException e) { + return FormValidation.error("Loki credentials are not valid: " + e.getMessage()); + } + return FormValidation.ok(); + } + + @POST + public FormValidation doValidate(@QueryParameter String lokiUrl, + @QueryParameter boolean disableSslVerifications, @QueryParameter String lokiCredentialsId, + @QueryParameter String lokiTenantId) { + Jenkins.get().checkPermission(Jenkins.ADMINISTER); + + FormValidation lokiUrlValidation = doCheckLokiUrl(lokiUrl); + if (lokiUrlValidation.kind != FormValidation.Kind.OK) { + return lokiUrlValidation; + } + OpenTelemetry openTelemetry = JenkinsControllerOpenTelemetry.get(); + try (LokiLogStorageRetriever lokiLogStorageRetriever = new LokiLogStorageRetriever( + lokiUrl, + disableSslVerifications, + getLokiApacheHttpCredentials(lokiCredentialsId), + Optional.ofNullable(lokiTenantId).filter(Predicate.not(String::isBlank)), + ObservabilityBackend.ERROR_TEMPLATE, + TemplateBindingsProvider.empty(), + "##not-needed-to-invoke-check-loki-setup##", + Optional.empty(), + openTelemetry)) { + return FormValidation.aggregate(lokiLogStorageRetriever.checkLokiSetup()); + } catch (NoSuchElementException e) { + return FormValidation.error("No credentials found for id '" + lokiCredentialsId + "'"); + } catch (Exception e) { + return FormValidation.error(e, e.getMessage()); + } + } + + + @NonNull + public String getDefaultLokiDataSourceIdentifier() { + return GrafanaBackend.DEFAULT_LOKI_DATA_SOURCE_IDENTIFIER; + } + + @Override + public String getDefaultLokiOTelLogFormat() { + return LokiOTelLogFormat.LOKI_V2_JSON_OTEL_FORMAT.name(); + } + + @NonNull + @Override + public String getDisplayName() { + return "Store pipeline logs In Loki and visualize logs both in Grafana and through Jenkins "; + } + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization.java index 370b39f0..381f24bf 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization.java @@ -75,6 +75,11 @@ public String getDefaultLokiDataSourceIdentifier(){ return GrafanaBackend.DEFAULT_LOKI_DATA_SOURCE_IDENTIFIER; } + @Override + public String getDefaultLokiOTelLogFormat() { + return LokiOTelLogFormat.LOKI_V3_OTEL_FORMAT.name(); + } + @NonNull @Override public String getDisplayName() { diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIterator.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIterator.java new file mode 100644 index 00000000..f9f606a9 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIterator.java @@ -0,0 +1,229 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import com.google.common.annotations.VisibleForTesting; +import com.jayway.jsonpath.JsonPath; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.jenkins.plugins.opentelemetry.job.log.LogLine; +import io.jenkins.plugins.opentelemetry.job.log.util.CloseableIterator; +import io.jenkins.plugins.opentelemetry.job.log.util.LogLineIterator; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.context.Scope; +import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthenticationException; +import org.apache.http.auth.Credentials; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/* + * HttpClient can't do preemptive auth and Loki doesn't return `WWW-Authenticate` header when authentication is + * needed so use Apache HTTP Client instead. + */ +public class LokiBuildLogsLineIterator implements LogLineIterator, AutoCloseable { + + private final static Logger logger = Logger.getLogger(LokiBuildLogsLineIterator.class.getName()); + public final static int MAX_QUERIES = 100; + + final LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters; + + final String lokiUrl; + final Optional lokiCredentials; + final Optional lokiTenantId; + + final CloseableHttpClient httpClient; + final HttpContext httpContext; + + + @VisibleForTesting + int queryCounter; + + final Tracer tracer; + + Iterator> delegate; + boolean endOfStream; + + public LokiBuildLogsLineIterator( + + @NonNull LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters, + + @NonNull CloseableHttpClient httpClient, + @NonNull HttpContext httpContext, + + @NonNull String lokiUrl, + @NonNull Optional lokiCredentials, + @NonNull Optional lokiTenantId, + @NonNull Tracer tracer) { + + this.lokiQueryParameters = lokiQueryParameters; + + this.lokiUrl = lokiUrl; + this.lokiCredentials = lokiCredentials; + this.lokiTenantId = lokiTenantId; + + this.httpClient = httpClient; + this.httpContext = httpContext; + + this.tracer = tracer; + } + + @NonNull + Iterator> getCurrentIterator() { + try { + if (endOfStream) { + // don't try to load more + return delegate; + } + if (delegate == null) { + delegate = loadNextLogLines(); + } + if (delegate.hasNext()) { + return delegate; + } + delegate = loadNextLogLines(); + if (!delegate.hasNext()) { + endOfStream = true; + } + return delegate; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected Iterator> loadNextLogLines() throws IOException { + if (queryCounter > MAX_QUERIES) { + logger.log(Level.INFO, () -> "Circuit breaker: " + + queryCounter + " queries for " + this.lokiQueryParameters); + return Collections.emptyIterator(); + } + + Span loadNextLogLinesSpan = tracer.spanBuilder("LokiBuildLogsLineIterator.loadNextLogLines") + .setAllAttributes(this.lokiQueryParameters.toAttributes()) + .startSpan(); + try (Scope loadNextLogLinesScope = loadNextLogLinesSpan.makeCurrent()) { + + + HttpUriRequest lokiQueryRangeRequest = this.lokiQueryParameters.toHttpRequest(lokiUrl); + lokiCredentials.ifPresent(credentials -> { + // preemptive authentication due to a limitation of Grafana Cloud Logs (Loki) that doesn't return + // `WWW-Authenticate` header to trigger traditional authentication + try { + lokiQueryRangeRequest.addHeader(new BasicScheme().authenticate(credentials, lokiQueryRangeRequest, httpContext)); + } catch (AuthenticationException e) { + throw new RuntimeException(e); + } + }); + lokiTenantId.ifPresent(tenantId -> lokiQueryRangeRequest.addHeader(new LokiTenantHeader(tenantId))); + + queryCounter++; + try (CloseableHttpResponse lokiQueryRangeResponse = httpClient.execute(lokiQueryRangeRequest, this.httpContext)) { + if (lokiQueryRangeResponse.getStatusLine().getStatusCode() != 200) { + throw new IOException("Loki logs query failure: " + lokiQueryRangeResponse.getStatusLine() + " - " + EntityUtils.toString(lokiQueryRangeResponse.getEntity())); + } + HttpEntity entity = lokiQueryRangeResponse.getEntity(); + if (entity == null) { + logger.log(Level.INFO, () -> "No content in response for " + this.lokiQueryParameters); + return Collections.emptyIterator(); + } + InputStream lokiQueryLogsResponseStream = entity.getContent(); + return loadLogLines(lokiQueryLogsResponseStream); + } + } + } + + @Nonnull + @VisibleForTesting + protected Iterator> loadLogLines(InputStream lokiQueryResponseInputStream) throws IOException { + Iterator> logLineIterator = JsonPath.>>read(lokiQueryResponseInputStream, "$.data.result[*].values[*]").stream().map(valueKeyPair -> { + long timestampInNanos = Long.parseLong(valueKeyPair.get(0)); + String msg = valueKeyPair.get(1); + if (timestampInNanos < lokiQueryParameters.getStartTimeInNanos()) { + logger.log(Level.INFO, () -> "Unordered timestamps " + timestampInNanos + " < " + lokiQueryParameters.getStartTimeInNanos() + + " for " + lokiQueryParameters); + } else { + lokiQueryParameters.setStartTimeInNanos(timestampInNanos + 1); // +1 because `start` is >= + } + return new LogLine<>(timestampInNanos, msg); + }).iterator(); + + return new CloseableIterator<>(logLineIterator, lokiQueryResponseInputStream); + } + + @Override + public void skipLines(Long lastLogTimestampInNanos) { + Tracer tracer = logger.isLoggable(Level.FINE) ? this.tracer : TracerProvider.noop().get("noop"); + Span span = tracer.spanBuilder("LokiBuildLogsLineIterator.skip") + .setAllAttributes(this.lokiQueryParameters.toAttributes()) + .setAttribute("lastLogTimestampInNanos", lastLogTimestampInNanos) + .startSpan(); + long newStartTimeInNanos = lastLogTimestampInNanos + 1; + try { + if (this.delegate == null) { + span.setAttribute("skippedLines", -1); + lokiQueryParameters.setStartTimeInNanos(newStartTimeInNanos); + } else { + /* + * Happens when invoked by: + * GET /job/:jobFullName/:runNumber/consoleText + * |- org.jenkinsci.plugins.workflow.job.WorkflowRun.doConsoleText + * |- io.jenkins.plugins.opentelemetry.job.log.OverallLog.writeLogTo(long, java.io.OutputStream) + * |- org.kohsuke.stapler.framework.io.LargeText.writeLogTo(long, java.io.OutputStream) + * GET /blue/rest/organizations/:organization/pipelines/:pipeline/branches/:branch/runs/:runNumber/log?start=0 + * + * When invoked by "/job/:jobFullName/:runNumber/consoleText", it's the second call to LargeText.writeLogTo() and it's EOF + */ + span.setAttribute("skippedLines", -1); + lokiQueryParameters.setStartTimeInNanos(newStartTimeInNanos); + this.delegate = null; // TODO optimize to skip lines in the current delegate + } + } finally { + span.end(); + } + } + + @Override + public boolean hasNext() { + return getCurrentIterator().hasNext(); + } + + @Override + public LogLine next() { + return getCurrentIterator().next(); + } + + @Override + public void close() throws Exception { + if (delegate instanceof AutoCloseable) { + try { + ((AutoCloseable) delegate).close(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to close delegate for " + lokiQueryParameters, e); + } + } + try { + this.httpClient.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParameters.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParameters.java new file mode 100644 index 00000000..42a279ed --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParameters.java @@ -0,0 +1,126 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import edu.umd.cs.findbugs.annotations.NonNull; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; + +import javax.annotation.Nonnull; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import static io.jenkins.plugins.opentelemetry.backend.grafana.LokiMetadata.*; +import static io.jenkins.plugins.opentelemetry.backend.grafana.LokiMetadata.META_DATA_JENKINS_PIPELINE_STEP_ID; + +public class LokiGetJenkinsBuildLogsQueryParameters { + @NonNull + private final String jobFullName; + private final int runNumber; + @NonNull + private final String traceId; + @NonNull + private final Optional flowNodeId; + @NonNull + private Long startTimeInNanos; + @NonNull + private final Optional endTimeInNanos; + @NonNull + private final String serviceName; + @NonNull + private final Optional serviceNamespace; + + public LokiGetJenkinsBuildLogsQueryParameters(@NonNull String jobFullName, int runNumber, @NonNull String traceId, @NonNull Optional flowNodeId, @NonNull Instant startTimeInNanos, @NonNull Optional endTime, @NonNull String serviceName, @NonNull Optional serviceNamespace) { + this.jobFullName = jobFullName; + this.runNumber = runNumber; + this.traceId = traceId; + this.flowNodeId = flowNodeId; + this.startTimeInNanos = instantToEpochNanos(startTimeInNanos); + this.endTimeInNanos = endTime.map(new InstantToEpochInNanos()); + this.serviceName = serviceName; + this.serviceNamespace = serviceNamespace; + } + + public HttpUriRequest toHttpRequest(@Nonnull String lokiUrl) { + // https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + + final StringBuilder logQl = new StringBuilder("{"); + serviceNamespace.ifPresent(serviceNamespace -> logQl.append(LABEL_SERVICE_NAMESPACE).append("=\"").append(serviceNamespace).append("\", ")); + logQl.append(LABEL_SERVICE_NAME + "=\"" + serviceName + "\"}"); + + logQl.append("|" + + META_DATA_TRACE_ID + "=\"" + traceId + "\", " + + META_DATA_CI_PIPELINE_ID + "=\"" + jobFullName + "\", " + + META_DATA_CI_PIPELINE_RUN_NUMBER + "=" + runNumber); + flowNodeId.ifPresent(flowNodeId -> logQl.append(", " + META_DATA_JENKINS_PIPELINE_STEP_ID + "=\"" + flowNodeId + "\"")); + + logQl.append(" | keep __line__"); + + RequestBuilder lokiQueryRangeRequestBuilder = RequestBuilder + .get() + .setUri(lokiUrl + "/loki/api/v1/query_range") + .addParameter("query", logQl.toString()) + .addParameter("start", startTimeInNanos + "") + .addParameter("direction", "forward"); + + endTimeInNanos + .ifPresent(endTimeInNanos -> lokiQueryRangeRequestBuilder.addParameter("end", String.valueOf(endTimeInNanos))); + + return lokiQueryRangeRequestBuilder.build(); + } + + public Attributes toAttributes() { + final AttributesBuilder attributesBuilder = Attributes.builder(); + attributesBuilder.put("query." + META_DATA_TRACE_ID, traceId); + attributesBuilder.put("query." + META_DATA_CI_PIPELINE_ID, jobFullName); + attributesBuilder.put("query." + META_DATA_CI_PIPELINE_RUN_NUMBER, runNumber); + flowNodeId.ifPresent(flowNodeId -> attributesBuilder.put("query." +META_DATA_JENKINS_PIPELINE_STEP_ID, flowNodeId)); + + attributesBuilder.put("query.startTimeInNanos", startTimeInNanos); + endTimeInNanos.ifPresent(endTimeInNanos -> attributesBuilder.put("query.endTimeInNanos", endTimeInNanos)); + + return attributesBuilder.build(); + } + + public void setStartTimeInNanos(long startTimeInNanos) { + this.startTimeInNanos = startTimeInNanos; + } + + @NonNull + public Long getStartTimeInNanos() { + return startTimeInNanos; + } + + @Override + public String toString() { + return "LokiGetJenkinsBuildLogsQueryParameters{" + + "jobFullName='" + jobFullName + '\'' + + ", runNumber=" + runNumber + + ", traceId='" + traceId + '\'' + + ", flowNodeId=" + flowNodeId + + ", startTimeInNanos=" + startTimeInNanos + + ", endTimeInNanos=" + endTimeInNanos + + ", serviceName='" + serviceName + '\'' + + ", serviceNamespace=" + serviceNamespace + + '}'; + } + + static long instantToEpochNanos(Instant instant) { + return new InstantToEpochInNanos().apply(instant); + } + + static class InstantToEpochInNanos implements Function { + @Override + @Nonnull + public Long apply(Instant instant) { + return TimeUnit.NANOSECONDS.convert(instant.toEpochMilli(), TimeUnit.MILLISECONDS) + instant.getNano(); + } + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParametersBuilder.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParametersBuilder.java new file mode 100644 index 00000000..c2951337 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiGetJenkinsBuildLogsQueryParametersBuilder.java @@ -0,0 +1,68 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import java.time.Instant; +import java.util.Optional; + +public class LokiGetJenkinsBuildLogsQueryParametersBuilder { + private String jobFullName; + private int runNumber; + private String traceId; + private Optional flowNodeId = Optional.empty(); + private Instant startTime; + private Optional endTime = Optional.empty(); + private String serviceName; + private Optional serviceNamespace = Optional.empty(); + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setJobFullName(String jobFullName) { + this.jobFullName = jobFullName; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setRunNumber(int runNumber) { + this.runNumber = runNumber; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setTraceId(String traceId) { + this.traceId = traceId; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setFlowNodeId(String flowNodeId) { + this.flowNodeId = Optional.ofNullable(flowNodeId); + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setStartTime(Instant startTime) { + this.startTime = startTime; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setEndTime(Instant endTime) { + this.endTime = Optional.ofNullable(endTime); + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setServiceName(String serviceName) { + this.serviceName = serviceName; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParametersBuilder setServiceNamespace(String serviceNamespace) { + this.serviceNamespace = Optional.ofNullable(serviceNamespace); + return this; + } + public LokiGetJenkinsBuildLogsQueryParametersBuilder setServiceNamespace(Optional serviceNamespace) { + this.serviceNamespace = serviceNamespace; + return this; + } + + public LokiGetJenkinsBuildLogsQueryParameters build() { + return new LokiGetJenkinsBuildLogsQueryParameters(jobFullName, runNumber, traceId, flowNodeId, startTime, endTime, serviceName, serviceNamespace); + } +} \ No newline at end of file diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetriever.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetriever.java new file mode 100644 index 00000000..8e320f8e --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetriever.java @@ -0,0 +1,275 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import com.google.common.annotations.VisibleForTesting; +import com.google.errorprone.annotations.MustBeClosed; +import edu.umd.cs.findbugs.annotations.NonNull; +import groovy.text.Template; +import hudson.util.FormValidation; +import io.jenkins.plugins.opentelemetry.JenkinsControllerOpenTelemetry; +import io.jenkins.plugins.opentelemetry.TemplateBindingsProvider; +import io.jenkins.plugins.opentelemetry.backend.GrafanaBackend; +import io.jenkins.plugins.opentelemetry.backend.ObservabilityBackend; +import io.jenkins.plugins.opentelemetry.job.log.LogStorageRetriever; +import io.jenkins.plugins.opentelemetry.job.log.LogsQueryResult; +import io.jenkins.plugins.opentelemetry.job.log.LogsViewHeader; +import io.jenkins.plugins.opentelemetry.job.log.util.InputStreamByteBuffer; +import io.jenkins.plugins.opentelemetry.job.log.util.LogLineIterator; +import io.jenkins.plugins.opentelemetry.job.log.util.LogLineIteratorInputStream; +import io.jenkins.plugins.opentelemetry.semconv.JenkinsOtelSemanticAttributes; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.instrumentation.apachehttpclient.v4_3.ApacheHttpClientTelemetry; +import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthenticationException; +import org.apache.http.auth.Credentials; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.BasicHttpContext; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.util.EntityUtils; +import org.kohsuke.stapler.framework.io.ByteBuffer; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.net.ssl.SSLContext; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + + +public class LokiLogStorageRetriever implements LogStorageRetriever, Closeable { + private final static Logger logger = Logger.getLogger(LokiLogStorageRetriever.class.getName()); + + private final Template buildLogsVisualizationUrlTemplate; + + private final TemplateBindingsProvider templateBindingsProvider; + + private final String lokiUrl; + private final String serviceName; + private final Optional serviceNamespace; + private final Optional lokiCredentials; + private final Optional lokiTenantId; + private final CloseableHttpClient httpClient; + private final HttpContext httpContext; + private final OpenTelemetry openTelemetry; + + @MustBeClosed + public LokiLogStorageRetriever(@Nonnull String lokiUrl, boolean disableSslVerifications, + @Nonnull Optional lokiCredentials, + @Nonnull Optional lokiTenantId, + @NonNull Template buildLogsVisualizationUrlTemplate, @NonNull TemplateBindingsProvider templateBindingsProvider, + @Nonnull String serviceName, @Nonnull Optional serviceNamespace, + @Nonnull OpenTelemetry openTelemetry) { + if (StringUtils.isBlank(lokiUrl)) { + throw new IllegalArgumentException("Loki url cannot be blank"); + } + this.lokiUrl = lokiUrl; + this.serviceName = serviceName; + this.serviceNamespace = serviceNamespace; + this.lokiCredentials = lokiCredentials; + this.lokiTenantId = lokiTenantId; + this.httpContext = new BasicHttpContext(); + HttpClientBuilder httpClientBuilder = ApacheHttpClientTelemetry.create(openTelemetry).newHttpClientBuilder(); + if (disableSslVerifications) { + try { + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustAllStrategy()).build(); + httpClientBuilder.setSSLContext(sslContext); + } catch (GeneralSecurityException e) { + logger.log(Level.WARNING, "IllegalStateException: failure to disable SSL certs verification"); + } + httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } + + this.httpClient = httpClientBuilder.build(); + + this.buildLogsVisualizationUrlTemplate = buildLogsVisualizationUrlTemplate; + this.templateBindingsProvider = templateBindingsProvider; + + this.openTelemetry = openTelemetry; + } + + @Nonnull + @Override + public LogsQueryResult overallLog(@Nonnull String jobFullName, int runNumber, @Nonnull String traceId, @Nonnull String spanId, boolean complete, @Nonnull Instant startTime, @Nullable Instant endTime) { + SpanBuilder spanBuilder = getTracer().spanBuilder("LokiLogStorageRetriever.overallLog") + .setAttribute(JenkinsOtelSemanticAttributes.CI_PIPELINE_ID, jobFullName) + .setAttribute(JenkinsOtelSemanticAttributes.CI_PIPELINE_RUN_NUMBER, (long) runNumber) + .setAttribute("complete", complete); + + Span span = spanBuilder.startSpan(); + + try (Scope scope = span.makeCurrent()) { + + LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters = new LokiGetJenkinsBuildLogsQueryParametersBuilder() + .setJobFullName(jobFullName) + .setRunNumber(runNumber) + .setTraceId(traceId) + .setStartTime(startTime) + .setEndTime(endTime) + .setServiceName(serviceName) + .setServiceNamespace(serviceNamespace) + .build(); + LogLineIterator logLines = new LokiBuildLogsLineIterator( + lokiQueryParameters, + httpClient, httpContext, + lokiUrl, lokiCredentials, lokiTenantId, + openTelemetry.getTracer("io.jenkins")); + + LogLineIterator.JenkinsHttpSessionLineBytesToLogLineIdMapper lineBytesToLineNumberConverter = new LogLineIterator.JenkinsHttpSessionLineBytesToLogLineIdMapper<>(jobFullName, runNumber, null); + InputStream lineIteratorInputStream = new LogLineIteratorInputStream<>(logLines, lineBytesToLineNumberConverter, getTracer()); + ByteBuffer byteBuffer = new InputStreamByteBuffer(lineIteratorInputStream, getTracer()); + + Map localBindings = Map.of( + ObservabilityBackend.TemplateBindings.TRACE_ID, traceId, + ObservabilityBackend.TemplateBindings.SPAN_ID, spanId, + ObservabilityBackend.TemplateBindings.START_TIME, startTime, + ObservabilityBackend.TemplateBindings.END_TIME, Optional.ofNullable(endTime).or(() -> Optional.of(Instant.now())).get() + ); + + Map bindings = TemplateBindingsProvider.compose(this.templateBindingsProvider, localBindings).getBindings(); + String logsVisualizationUrl = this.buildLogsVisualizationUrlTemplate.make(bindings).toString(); + + return new LogsQueryResult( + byteBuffer, + new LogsViewHeader( + bindings.get(GrafanaBackend.TemplateBindings.BACKEND_NAME).toString(), + logsVisualizationUrl, + bindings.get(GrafanaBackend.TemplateBindings.BACKEND_24_24_ICON_URL).toString()), + StandardCharsets.UTF_8, complete + ); + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(StatusCode.ERROR, e.getMessage()); + throw e; + } finally { + span.end(); + } + } + + @Nonnull + @Override + public LogsQueryResult stepLog(@Nonnull String jobFullName, int runNumber, @Nonnull String flowNodeId, @Nonnull String traceId, @Nonnull String spanId, boolean complete, @Nonnull Instant startTime, @Nullable Instant endTime) { + SpanBuilder spanBuilder = getTracer().spanBuilder("LokiLogStorageRetriever.stepLog") + .setAttribute(JenkinsOtelSemanticAttributes.CI_PIPELINE_ID, jobFullName) + .setAttribute(JenkinsOtelSemanticAttributes.CI_PIPELINE_RUN_NUMBER, (long) runNumber) + .setAttribute(JenkinsOtelSemanticAttributes.JENKINS_STEP_ID, flowNodeId) + .setAttribute("complete", complete); + + Span span = spanBuilder.startSpan(); + + try (Scope scope = span.makeCurrent()) { + + LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters = new LokiGetJenkinsBuildLogsQueryParametersBuilder() + .setJobFullName(jobFullName) + .setRunNumber(runNumber) + .setTraceId(traceId) + .setFlowNodeId(flowNodeId) + .setStartTime(startTime) + .setEndTime(endTime) + .setServiceName(serviceName) + .setServiceNamespace(serviceNamespace) + .build(); + LogLineIterator logLines = new LokiBuildLogsLineIterator( + lokiQueryParameters, httpClient, httpContext, + lokiUrl, lokiCredentials, lokiTenantId, + openTelemetry.getTracer("io.jenkins")); + + LogLineIterator.LogLineBytesToLogLineIdMapper logLineBytesToLogLineIdMapper = new LogLineIterator.JenkinsHttpSessionLineBytesToLogLineIdMapper<>(jobFullName, runNumber, null); + InputStream logLineIteratorInputStream = new LogLineIteratorInputStream<>(logLines, logLineBytesToLogLineIdMapper, getTracer()); + ByteBuffer byteBuffer = new InputStreamByteBuffer(logLineIteratorInputStream, getTracer()); + + Map localBindings = Map.of( + ObservabilityBackend.TemplateBindings.TRACE_ID, traceId, + ObservabilityBackend.TemplateBindings.SPAN_ID, spanId, + ObservabilityBackend.TemplateBindings.START_TIME, startTime, + ObservabilityBackend.TemplateBindings.END_TIME, Optional.ofNullable(endTime).or(() -> Optional.of(Instant.now())).get() + ); + + Map bindings = TemplateBindingsProvider.compose(this.templateBindingsProvider, localBindings).getBindings(); + String logsVisualizationUrl = this.buildLogsVisualizationUrlTemplate.make(bindings).toString(); + + return new LogsQueryResult( + byteBuffer, + new LogsViewHeader( + bindings.get(GrafanaBackend.TemplateBindings.BACKEND_NAME).toString(), + logsVisualizationUrl, + bindings.get(GrafanaBackend.TemplateBindings.BACKEND_24_24_ICON_URL).toString()), + StandardCharsets.UTF_8, complete + ); + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(StatusCode.ERROR, e.getMessage()); + throw e; + } finally { + span.end(); + } + } + + public List checkLokiSetup() { + List validations = new ArrayList<>(); + + // `/ready`and `/loki/api/v1/status/buildinfo` return a 404 on Grafana Cloud, use the format_query request instead + HttpUriRequest lokiBuildInfoRequest = RequestBuilder.get().setUri(lokiUrl + "/loki/api/v1/format_query").addParameter("query", "{foo= \"bar\"}").build(); + + lokiCredentials.ifPresent(lokiCredentials -> { + try { + // preemptive authentication due to a limitation of Grafana Cloud Logs (Loki) that doesn't return `WWW-Authenticate` + // header to trigger traditional authentication + lokiBuildInfoRequest.addHeader(new BasicScheme().authenticate(lokiCredentials, lokiBuildInfoRequest, httpContext)); + } catch (AuthenticationException e) { + throw new RuntimeException(e); + } + }); + lokiTenantId.ifPresent(tenantId -> lokiBuildInfoRequest.addHeader(new LokiTenantHeader(tenantId))); + try (CloseableHttpResponse lokiReadyResponse = httpClient.execute(lokiBuildInfoRequest, httpContext)) { + if (lokiReadyResponse.getStatusLine().getStatusCode() != 200) { + validations.add(FormValidation.error("Failure to access Loki (" + lokiBuildInfoRequest + "): " + EntityUtils.toString(lokiReadyResponse.getEntity()))); + } else { + validations.add(FormValidation.ok("Loki connection successful")); + } + } catch (IOException e) { + validations.add(FormValidation.error("Failure to access Loki (" + lokiBuildInfoRequest + "): " + e.getMessage())); + } + return validations; + } + + @Override + public void close() throws IOException { + this.httpClient.close(); + } + + @VisibleForTesting + Tracer _tracer; + + private Tracer getTracer() { + if (_tracer == null) { + _tracer = JenkinsControllerOpenTelemetry.get().getDefaultTracer(); + } + return _tracer; + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiMetadata.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiMetadata.java new file mode 100644 index 00000000..0be4b7fc --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiMetadata.java @@ -0,0 +1,19 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import io.jenkins.plugins.opentelemetry.semconv.JenkinsOtelSemanticAttributes; +import io.opentelemetry.semconv.ServiceAttributes; +import io.opentelemetry.semconv.incubating.ServiceIncubatingAttributes; + +public interface LokiMetadata { + String LABEL_SERVICE_NAME = ServiceAttributes.SERVICE_NAME.getKey().replace('.', '_'); + String LABEL_SERVICE_NAMESPACE = ServiceIncubatingAttributes.SERVICE_NAMESPACE.getKey().replace('.', '_'); + String META_DATA_TRACE_ID = "trace_id"; + String META_DATA_CI_PIPELINE_ID = JenkinsOtelSemanticAttributes.CI_PIPELINE_ID.getKey().replace('.', '_'); + String META_DATA_CI_PIPELINE_RUN_NUMBER = JenkinsOtelSemanticAttributes.CI_PIPELINE_RUN_NUMBER.getKey().replace('.', '_'); + String META_DATA_JENKINS_PIPELINE_STEP_ID = JenkinsOtelSemanticAttributes.JENKINS_STEP_ID.getKey().replace('.', '_'); +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiTenantHeader.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiTenantHeader.java new file mode 100644 index 00000000..0f5cf9a7 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiTenantHeader.java @@ -0,0 +1,15 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +public class LokiTenantHeader extends BasicHeader implements Header { + public LokiTenantHeader(String tenantId) { + super("X-Scope-OrgID", tenantId); + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/NoGrafanaLogsBackend.java b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/NoGrafanaLogsBackend.java index 2aa4e360..10236b54 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/NoGrafanaLogsBackend.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/backend/grafana/NoGrafanaLogsBackend.java @@ -10,6 +10,7 @@ import io.jenkins.plugins.opentelemetry.job.log.LogStorageRetriever; import org.kohsuke.stapler.DataBoundConstructor; +import javax.annotation.Nonnull; import java.util.Collections; import java.util.Map; @@ -40,9 +41,16 @@ public int hashCode() { @Extension(ordinal = 100) public static class DescriptorImpl extends GrafanaLogsBackend.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Don't store pipeline logs in Loki"; } + + // doesn't matter what the default is, as this is not really a backend + @Override + public String getDefaultLokiOTelLogFormat() { + return LokiOTelLogFormat.LOKI_V2_JSON_OTEL_FORMAT.name(); + } } } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/init/ServletFilterInitializer.java b/src/main/java/io/jenkins/plugins/opentelemetry/init/ServletFilterInitializer.java index 6a43d553..861f3967 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/init/ServletFilterInitializer.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/init/ServletFilterInitializer.java @@ -20,6 +20,8 @@ import javax.servlet.Filter; import javax.servlet.ServletException; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -54,7 +56,8 @@ public void afterSdkInitialized(Meter meter, LoggerProvider loggerProvider, Even boolean jenkinsWebInstrumentationEnabled = Optional.ofNullable(configProperties.getBoolean(JenkinsOtelSemanticAttributes.OTEL_INSTRUMENTATION_JENKINS_WEB_ENABLED)).orElse(true); if (jenkinsWebInstrumentationEnabled) { - staplerInstrumentationServletFilter = new StaplerInstrumentationServletFilter(tracer); + List capturedRequestParameters = configProperties.getList(JenkinsOtelSemanticAttributes.OTEL_INSTRUMENTATION_SERVLET_CAPTURE_REQUEST_PARAMETERS, Collections.emptyList()); + staplerInstrumentationServletFilter = new StaplerInstrumentationServletFilter(capturedRequestParameters, tracer); addToPluginServletFilter(staplerInstrumentationServletFilter); } else { logger.log(Level.INFO, () -> "Jenkins Web instrumentation disabled. To enable it, set the property " + diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/jenkins/JenkinsCredentialsToApacheHttpCredentialsAdapter.java b/src/main/java/io/jenkins/plugins/opentelemetry/jenkins/JenkinsCredentialsToApacheHttpCredentialsAdapter.java index dde47e1a..c9ae87e5 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/jenkins/JenkinsCredentialsToApacheHttpCredentialsAdapter.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/jenkins/JenkinsCredentialsToApacheHttpCredentialsAdapter.java @@ -18,15 +18,26 @@ import java.util.NoSuchElementException; import java.util.function.Supplier; +/** + * Adapter to convert Jenkins credentials to Apache HTTP credentials. + */ public class JenkinsCredentialsToApacheHttpCredentialsAdapter implements Credentials { Supplier jenkinsCredentialsIdProvider; UsernamePasswordCredentials jenkinsUsernamePasswordCredentials; + /** + * @deprecated use {@link JenkinsCredentialsToApacheHttpCredentialsAdapter#JenkinsCredentialsToApacheHttpCredentialsAdapter(String)} instead + */ + @Deprecated public JenkinsCredentialsToApacheHttpCredentialsAdapter(Supplier jenkinsCredentialsIdProvider) { this.jenkinsCredentialsIdProvider = jenkinsCredentialsIdProvider; } + public JenkinsCredentialsToApacheHttpCredentialsAdapter(String jenkinsCredentialsId) { + this.jenkinsCredentialsIdProvider = () -> jenkinsCredentialsId; + } + @Override public Principal getUserPrincipal() throws CredentialsNotFoundException { return new BasicUserPrincipal(getJenkinsUsernamePasswordCredentials().getUsername()); diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/LogLine.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/LogLine.java new file mode 100644 index 00000000..0b28b0ac --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/LogLine.java @@ -0,0 +1,48 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.job.log; + +import javax.annotation.Nonnull; + +/** + * Represents a build log line. + * + * @param identifier of the log line within the search + * query results (e.g. {@link Long} line number for Elasticsearch + * or the {@link Long} timestamp in nanos for Loki) + */ +public class LogLine { + private final Id id; + private final String message; + + public LogLine(@Nonnull Id id, @Nonnull String message) { + this.id = id; + this.message = message; + } + + /** + * @return the identifier of the log line within the search + * query results (e.g. {@link Long} line number for Elasticsearch + * or the {@link Long} timestamp in nanos for Loki) + */ + @Nonnull + public Id getId() { + return id; + } + + @Nonnull + public String getMessage() { + return message; + } + + @Override + public String toString() { + return "LogLine{" + + "id=" + id + + ", message='" + message + '\'' + + '}'; + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/OtelLogSenderBuildListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/OtelLogSenderBuildListener.java index d3629151..d7e3b74d 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/OtelLogSenderBuildListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/OtelLogSenderBuildListener.java @@ -11,7 +11,7 @@ import hudson.model.BuildListener; import io.jenkins.plugins.opentelemetry.JenkinsControllerOpenTelemetry; import io.jenkins.plugins.opentelemetry.opentelemetry.GlobalOpenTelemetrySdk; -import io.jenkins.plugins.opentelemetry.opentelemetry.common.OffsetClock; +import io.jenkins.plugins.opentelemetry.opentelemetry.common.Clocks; import io.jenkins.plugins.opentelemetry.semconv.JenkinsOtelSemanticAttributes; import io.opentelemetry.sdk.common.Clock; import jenkins.util.JenkinsJVM; @@ -60,7 +60,7 @@ public OtelLogSenderBuildListener(@NonNull RunTraceContext runTraceContext, @Non this.runTraceContext = runTraceContext; this.otelConfigProperties = otelConfigProperties; this.otelResourceAttributes = otelResourceAttributes; - this.clock = Clock.getDefault(); + this.clock = Clocks.monotonicClock(); // Constructor must always be invoked on the Jenkins Controller. // Instantiation on the Jenkins Agents is done via deserialization. JenkinsJVM.checkJenkinsJVM(); @@ -173,7 +173,7 @@ private Object readResolve() { */ if (instantInNanosOnJenkinsControllerBeforeSerialization == 0) { logger.log(Level.INFO, () -> "adjustClock: unexpected timeBeforeSerialization of 0ns, don't adjust the clock"); - this.clock = Clock.getDefault(); + this.clock = Clocks.monotonicClock(); } else { long instantInNanosOnJenkinsAgentAtDeserialization = Clock.getDefault().now(); long offsetInNanosOnJenkinsAgent = instantInNanosOnJenkinsControllerBeforeSerialization - instantInNanosOnJenkinsAgentAtDeserialization; @@ -183,7 +183,7 @@ private Object readResolve() { "A negative offset of few milliseconds is expected due to the latency of the communication from the Jenkins Controller to the Jenkins Agent. " + "Higher offsets indicate a synchronization gap of the system clocks between the Jenkins Controller that will be work arounded by the clock adjustment." ); - this.clock = OffsetClock.offsetClock(offsetInNanosOnJenkinsAgent); + this.clock = Clocks.monotonicOffsetClock(offsetInNanosOnJenkinsAgent); } // Setup OTel diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/CloseableIterator.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/CloseableIterator.java new file mode 100644 index 00000000..faed9be1 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/CloseableIterator.java @@ -0,0 +1,48 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.job.log.util; + +import java.io.Closeable; +import java.io.InputStream; +import java.util.Iterator; + +/** + *

+ * Useful when the {@link Iterator} is backed by an {@link InputStream} and the logic of the code dereferences the + * InputStream, make the {@link Iterator} closeable. + *

+ *

+ * TODO verify that extending {@link AutoCloseable} instead of {@link Closeable} is a good decision? The rationale + * is that it's more general. + *

+ */ +public class CloseableIterator implements Iterator, AutoCloseable { + private final Iterator delegate; + private final Closeable closeable; + + public CloseableIterator(Iterator delegate, Closeable closeable) { + this.delegate = delegate; + this.closeable = closeable; + } + + @Override + public void close() throws Exception { + if (delegate instanceof AutoCloseable) { + ((AutoCloseable) delegate).close(); + } + closeable.close(); + } + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + + @Override + public E next() { + return delegate.next(); + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/InputStreamByteBuffer.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/InputStreamByteBuffer.java index 125f5037..3d1153ea 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/InputStreamByteBuffer.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/InputStreamByteBuffer.java @@ -12,6 +12,8 @@ import org.kohsuke.stapler.framework.io.ByteBuffer; import edu.umd.cs.findbugs.annotations.NonNull; + +import javax.annotation.Nonnull; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -30,7 +32,7 @@ public class InputStreamByteBuffer extends ByteBuffer { @NonNull final InputStream in; - public InputStreamByteBuffer(InputStream in, Tracer tracer) { + public InputStreamByteBuffer(@Nonnull InputStream in, @Nonnull Tracer tracer) { this.in = in; this.tracer = tracer; } @@ -63,9 +65,9 @@ public InputStream newInputStream() { } /** - * Unsupported byt this readonly {@link ByteBuffer} + * Unsupported by this readonly {@link ByteBuffer} * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException always */ @Override public synchronized void write(byte[] b, int off, int len) throws IOException { @@ -73,9 +75,9 @@ public synchronized void write(byte[] b, int off, int len) throws IOException { } /** - * Unsupported byt this readonly {@link ByteBuffer} + * Unsupported by this readonly {@link ByteBuffer} * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException always */ @Override public synchronized void write(int b) throws IOException { @@ -83,9 +85,9 @@ public synchronized void write(int b) throws IOException { } /** - * Unsupported byt this readonly {@link ByteBuffer} + * Unsupported by this readonly {@link ByteBuffer} * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException always */ @Override public synchronized void writeTo(OutputStream os) { @@ -93,9 +95,9 @@ public synchronized void writeTo(OutputStream os) { } /** - * Unsupported byt this readonly {@link ByteBuffer} + * Unsupported by this readonly {@link ByteBuffer} * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException always */ @Override public void write(@NonNull byte[] b) throws IOException { @@ -103,9 +105,9 @@ public void write(@NonNull byte[] b) throws IOException { } /** - * Unsupported byt this readonly {@link ByteBuffer} + * Unsupported by this readonly {@link ByteBuffer} * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException always */ @Override public void flush() throws IOException { diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIterator.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIterator.java index 413087b5..eae77326 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIterator.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIterator.java @@ -20,6 +20,10 @@ import edu.umd.cs.findbugs.annotations.Nullable; import io.jenkins.plugins.opentelemetry.job.RunFlowNodeIdentifier; +/** + * @deprecated use {@link io.jenkins.plugins.opentelemetry.job.log.util.LogLineIterator} instead + */ +@Deprecated public interface LineIterator extends Iterator { void skipLines(long skip); diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIteratorInputStream.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIteratorInputStream.java index 93e1ef68..883bd7b3 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIteratorInputStream.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LineIteratorInputStream.java @@ -11,6 +11,7 @@ import io.opentelemetry.context.Scope; import edu.umd.cs.findbugs.annotations.Nullable; + import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -21,7 +22,10 @@ /** * {@link InputStream} backend by a {@link LineIterator} + * + * @deprecated use {@link LogLineIterator} instead */ +@Deprecated public class LineIteratorInputStream extends InputStream { private final static Logger logger = Logger.getLogger(LineIteratorInputStream.class.getName()); @@ -67,7 +71,7 @@ public int read() throws IOException { * Returns {@code null} if no more data available */ @Nullable - String readLine() throws IOException { + String readLine() { if (lines.hasNext()) { readLines++; return lines.next(); diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIterator.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIterator.java new file mode 100644 index 00000000..346091c6 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIterator.java @@ -0,0 +1,89 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.job.log.util; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.jenkins.plugins.opentelemetry.job.RunFlowNodeIdentifier; +import io.jenkins.plugins.opentelemetry.job.log.LogLine; +import org.kohsuke.stapler.Stapler; +import org.kohsuke.stapler.StaplerRequest; + +import javax.servlet.http.HttpSession; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +public interface LogLineIterator extends Iterator> { + void skipLines(Id toLogLineId); + + interface LogLineBytesToLogLineIdMapper { + /** + * @return {@code null} if unknown + */ + @Nullable + Id getLogLineIdFromLogBytes(long bytes); + + void putLogBytesToLogLineId(long bytes, Id timestampInNanos); + } + + /** + * Converter gets garbage collected when the HTTP session expires + */ + class JenkinsHttpSessionLineBytesToLogLineIdMapper implements LogLineBytesToLogLineIdMapper { + private final static Logger logger = Logger.getLogger(JenkinsHttpSessionLineBytesToLogLineIdMapper.class.getName()); + + public static final String HTTP_SESSION_KEY = "JenkinsHttpSessionLineBytesToLineNumberConverter"; + final String jobFullName; + final int runNumber; + @Nullable + final String flowNodeId; + + public JenkinsHttpSessionLineBytesToLogLineIdMapper(String jobFullName, int runNumber, @Nullable String flowNodeId) { + this.jobFullName = jobFullName; + this.runNumber = runNumber; + this.flowNodeId = flowNodeId; + } + + @Nullable + @Override + public Id getLogLineIdFromLogBytes(long bytes) { + RunFlowNodeIdentifier contextKey = new RunFlowNodeIdentifier(jobFullName, runNumber, flowNodeId); + return Optional + .ofNullable(getContext().get(contextKey)) + .map(d -> d.get(bytes)) + .orElse(null); + + } + + @Override + public void putLogBytesToLogLineId(long bytes, Id logLineId) { + RunFlowNodeIdentifier contextKey = new RunFlowNodeIdentifier(jobFullName, runNumber, flowNodeId); + getContext().computeIfAbsent(contextKey, runFlowNodeIdentifier -> new HashMap<>()).put(bytes, logLineId); + } + + Map> getContext() { + StaplerRequest currentRequest = Stapler.getCurrentRequest(); + if (currentRequest == null) { + // happens when reading logs is not tied to a web request + // (e.g. API call from within a pipeline as described in https://github.com/jenkinsci/opentelemetry-plugin/issues/564) + logger.log(Level.WARNING, "No current request found, default to default LogLineNumber context"); + return new HashMap<>(); + } + HttpSession session = currentRequest.getSession(); + synchronized (session) { + Map> context = (Map>) session.getAttribute(HTTP_SESSION_KEY); + if (context == null) { + context = new HashMap<>(); + session.setAttribute(HTTP_SESSION_KEY, context); + } + return context; + } + } + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIteratorInputStream.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIteratorInputStream.java new file mode 100644 index 00000000..dbb9d319 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/log/util/LogLineIteratorInputStream.java @@ -0,0 +1,136 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.job.log.util; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.jenkins.plugins.opentelemetry.job.log.LogLine; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.context.Scope; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * {@link InputStream} backed by a {@link LogLineIterator} + */ +public class LogLineIteratorInputStream extends InputStream { + private final static Logger logger = Logger.getLogger(LogLineIteratorInputStream.class.getName()); + + private final LogLineIterator.LogLineBytesToLogLineIdMapper logLineBytesToLogLineIdConverter; + private final LogLineIterator logLines; + final protected Tracer tracer; + + private int cursorOnCurrentLine; + private byte[] currentLine; + private long readBytes; + private Id lastLogLineId; + + public LogLineIteratorInputStream(LogLineIterator logLines, LogLineIterator.LogLineBytesToLogLineIdMapper logLineBytesToLogLineIdConverter, Tracer tracer) { + this.logLines = logLines; + this.logLineBytesToLogLineIdConverter = logLineBytesToLogLineIdConverter; + this.tracer = tracer; + } + + @Override + public int read() throws IOException { + if (currentLine == null) { + if (cursorOnCurrentLine != 0) { + throw new IllegalStateException("Current line is null but cursorOnCurrentLine!=0: " + cursorOnCurrentLine); + } + currentLine = Optional.ofNullable(readLine()).map(line -> (line.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)).orElse(null); + if (currentLine == null) { + return -1; + } + } + if (cursorOnCurrentLine > currentLine.length) { + throw new IllegalStateException(); + } + int result = currentLine[cursorOnCurrentLine++]; + if (cursorOnCurrentLine == currentLine.length) { + currentLine = null; + cursorOnCurrentLine = 0; + } + readBytes++; + return result; + } + + /** + * Returns {@code null} if no more data available + */ + @Nullable + LogLine readLine() { + if (logLines.hasNext()) { + LogLine logLine = logLines.next(); + lastLogLineId = logLine.getId(); + return logLine; + } else { + return null; + } + } + + @Override + public long skip(long skipBytes) throws IOException { + Tracer tracer = logger.isLoggable(Level.FINE) ? this.tracer : TracerProvider.noop().get("noop"); + Span span = tracer.spanBuilder("LogLineIteratorInputStream.skip") + .setAttribute("skipBytes", skipBytes) + .startSpan(); + try (Scope scope = span.makeCurrent()) { + Optional logLineId = Optional.ofNullable(logLineBytesToLogLineIdConverter.getLogLineIdFromLogBytes(skipBytes)); + logLineId.ifPresentOrElse(id -> { + span.setAttribute("previousLastLogLineId", String.valueOf(this.lastLogLineId)); + span.setAttribute("lastLogLineId", String.valueOf(id)); + logLines.skipLines(id); + readBytes += skipBytes; + this.lastLogLineId = id; + }, + () -> span.addEvent("LogLine Bytes to LogLine Id conversion not found") + ); + return skipBytes; + } finally { + span.end(); + } + } + + @Override + public int available() throws IOException { + Tracer tracer = logger.isLoggable(Level.FINER) ? this.tracer : TracerProvider.noop().get("noop"); + Span span = tracer.spanBuilder("LogLineIteratorInputStream.available").startSpan(); + try (Scope scope = span.makeCurrent()) { + if (logLines.hasNext()) { + return 1; + } else { + return 0; + } + } finally { + span.end(); + } + } + + @Override + public void close() throws IOException { + Tracer tracer = logger.isLoggable(Level.FINER) ? this.tracer : TracerProvider.noop().get("noop"); + Span span = tracer.spanBuilder("LogLineIteratorInputStream.close") + .setAttribute("readBytes", readBytes) + .setAttribute("lastLogLineId", String.valueOf(lastLogLineId)) + .startSpan(); + try (Scope scope = span.makeCurrent()) { + logLineBytesToLogLineIdConverter.putLogBytesToLogLineId(readBytes, lastLogLineId); + if (logLines instanceof Closeable) { + ((Closeable) logLines).close(); + } + } finally { + span.end(); + } + } + +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/Clocks.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/Clocks.java new file mode 100644 index 00000000..f1838f37 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/Clocks.java @@ -0,0 +1,93 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.opentelemetry.common; + +import io.opentelemetry.sdk.common.Clock; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * Utils for {@link Clock} + */ +public class Clocks { + private Clocks() { + } + + public static Clock monotonicOffsetClock(long offsetInNanos) { + return new MonotonicClock(new OffsetClock(offsetInNanos, Clock.getDefault())); + } + + /** + * @param offsetInNanos the duration to add, in nanos + */ + public static Clock offsetClock(long offsetInNanos, Clock baseClock) { + return new OffsetClock(offsetInNanos, baseClock); + } + + + public static Clock monotonicClock(Clock delegate) { + return new MonotonicClock(delegate); + } + + public static Clock monotonicClock() { + return monotonicClock(Clock.getDefault()); + } + + private static class OffsetClock implements Clock { + private final long offsetInNanos; + private final Clock baseClock; + + /** + * @param offsetInNanos the duration to add, in nanos + * @param baseClock the base clock to add the duration to, not null + */ + private OffsetClock(long offsetInNanos, Clock baseClock) { + this.offsetInNanos = offsetInNanos; + this.baseClock = baseClock; + } + + @Override + public long now() { + return baseClock.now() + offsetInNanos; + } + + @Override + public long nanoTime() { + return baseClock.nanoTime() + offsetInNanos; + } + + @Override + public String toString() { + return "OffsetClock{" + + "offsetInNanos=" + offsetInNanos + + '}'; + } + } + + private static class MonotonicClock implements Clock { + private final Clock delegate; + private final AtomicReference lastNanoTime = new AtomicReference<>(0L); + + public MonotonicClock(Clock delegate) { + this.delegate = delegate; + } + + @Override + public long now() { + return lastNanoTime.updateAndGet(current -> Math.max(current + 1, delegate.now())); + } + + @Override + public long now(boolean highPrecision) { + return lastNanoTime.updateAndGet(current -> Math.max(current + 1, delegate.now(highPrecision))); + } + + @Override + public long nanoTime() { + return lastNanoTime.updateAndGet(current -> Math.max(current + 1, delegate.nanoTime())); + } + } +} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/OffsetClock.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/OffsetClock.java deleted file mode 100644 index c3b86b04..00000000 --- a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/OffsetClock.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright The Original Author or Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.jenkins.plugins.opentelemetry.opentelemetry.common; - -import io.opentelemetry.sdk.common.Clock; - -public final class OffsetClock implements Clock { - final long offsetInNanos; - final Clock baseClock; - - /** - * @param offsetInNanos the duration to add, in nanos - * @param baseClock the base clock to add the duration to, not null - */ - private OffsetClock(long offsetInNanos, Clock baseClock) { - this.offsetInNanos = offsetInNanos; - this.baseClock = baseClock; - } - - @Override - public long now() { - return baseClock.now() + offsetInNanos; - } - - @Override - public long nanoTime() { - return baseClock.nanoTime() + offsetInNanos; - } - - @Override - public String toString() { - return "OffsetClock{" + - "offsetInNanos=" + offsetInNanos + - '}'; - } - - /** - * - * @param offsetInNanos the duration to add, in nanos - */ - public static Clock offsetClock(long offsetInNanos) { - return new OffsetClock(offsetInNanos, Clock.getDefault()); - } -} diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/JenkinsOtelSemanticAttributes.java b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/JenkinsOtelSemanticAttributes.java index 87baf66b..353d9b42 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/JenkinsOtelSemanticAttributes.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/JenkinsOtelSemanticAttributes.java @@ -145,6 +145,10 @@ public final class JenkinsOtelSemanticAttributes { public static final String OTEL_INSTRUMENTATION_JENKINS_WEB_ENABLED = "otel.instrumentation.jenkins.web.enabled"; public static final String OTEL_INSTRUMENTATION_JENKINS_REMOTE_SPAN_ENABLED = "otel.instrumentation.jenkins.remote.span.enabled"; + /** + * https://opentelemetry.io/docs/zero-code/java/agent/configuration/#capturing-servlet-request-parameters + */ + public static final String OTEL_INSTRUMENTATION_SERVLET_CAPTURE_REQUEST_PARAMETERS = "otel.instrumentation.servlet.experimental.capture-request-parameters"; /** diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilter.java b/src/main/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilter.java index 2e8ec616..3f26b9a7 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilter.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilter.java @@ -37,6 +37,7 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.StringTokenizer; import java.util.logging.Level; import java.util.logging.Logger; @@ -46,17 +47,19 @@ /** * Instrumentation of the Stapler MVC framework. - * * TODO find a smarter way to instrument each HTTP request path. It should rely on instrumenting the Stapler framework * TODO adopt StaplerRequest.html#getAncestors() */ public class StaplerInstrumentationServletFilter implements Filter { private final static Logger logger = Logger.getLogger(StaplerInstrumentationServletFilter.class.getName()); + private final List capturedRequestParameters; private final Tracer tracer; + private static final Set SKIP_PATHS = new HashSet<>(Arrays.asList("static", "adjuncts", "scripts", "plugin", "images", "sse-gateway")); - public StaplerInstrumentationServletFilter(Tracer tracer) { + public StaplerInstrumentationServletFilter(List capturedRequestParameters, Tracer tracer) { + this.capturedRequestParameters = capturedRequestParameters; this.tracer = tracer; } @@ -71,7 +74,6 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } public void _doFilter(HttpServletRequest servletRequest, HttpServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - String pathInfo = servletRequest.getPathInfo(); List pathInfoTokens = Collections.list(new StringTokenizer(pathInfo, "/")).stream() .map(token -> (String) token) @@ -95,7 +97,7 @@ public void _doFilter(HttpServletRequest servletRequest, HttpServletResponse ser filterChain.doFilter(servletRequest, servletResponse); return; } - SpanBuilder spanBuilder; + final SpanBuilder spanBuilder; try { if (rootPath.equals("job")) { // e.g /job/my-war/job/master/lastBuild/console @@ -197,9 +199,9 @@ public void _doFilter(HttpServletRequest servletRequest, HttpServletResponse ser spanBuilder = tracer.spanBuilder(servletRequest.getMethod() + " " + pathInfo); } } catch (RuntimeException e) { - httpRoute = "/" + rootPath + "/*"; - spanBuilder = tracer.spanBuilder(servletRequest.getMethod() + " " + pathInfo); - logger.log(Level.INFO, () -> "Exception processing URL " + pathInfo + ", default to httpRoute: '/" + rootPath + "/*': " + e); + logger.log(Level.INFO, () -> "Exception processing URL " + pathInfo + ", skip instrumentation with tracing: " + e); + filterChain.doFilter(servletRequest, servletResponse); + return; } @@ -225,10 +227,12 @@ public void _doFilter(HttpServletRequest servletRequest, HttpServletResponse ser .setAttribute(ThreadIncubatingAttributes.THREAD_ID, currentThread.getId()) .setSpanKind(SpanKind.SERVER); - User user = User.current(); - if (user != null) { - spanBuilder.setAttribute(EnduserIncubatingAttributes.ENDUSER_ID, user.getId()); - } + Optional.ofNullable(User.current()).ifPresent(user -> spanBuilder.setAttribute(EnduserIncubatingAttributes.ENDUSER_ID, user.getId())); + + capturedRequestParameters.forEach( + parameterName -> + Optional.ofNullable(servletRequest.getParameter(parameterName)) + .ifPresent(value -> spanBuilder.setAttribute("http.request.parameter." + parameterName, value))); Span span = spanBuilder.startSpan(); try (Scope scope = span.makeCurrent()) { @@ -624,11 +628,13 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public boolean equals(Object o) { if (this == o) return true; - return o != null && getClass() == o.getClass(); + if (o == null || getClass() != o.getClass()) return false; + StaplerInstrumentationServletFilter that = (StaplerInstrumentationServletFilter) o; + return Objects.equals(capturedRequestParameters, that.capturedRequestParameters); } @Override public int hashCode() { - return StaplerInstrumentationServletFilter.class.hashCode(); + return Objects.hashCode(capturedRequestParameters); } } diff --git a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/GrafanaBackend/config.jelly b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/GrafanaBackend/config.jelly index c8e9dcb3..146d53a2 100644 --- a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/GrafanaBackend/config.jelly +++ b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/GrafanaBackend/config.jelly @@ -2,7 +2,7 @@ - + @@ -15,7 +15,6 @@ @@ -38,8 +37,5 @@ - - - diff --git a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/config.jelly b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/config.jelly new file mode 100644 index 00000000..bfbfc6d1 --- /dev/null +++ b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/config.jelly @@ -0,0 +1,41 @@ + + + + + Pipeline logs are no longer stored in the Jenkins home, they are sent through OpenTelemetry to Loki and visible + in both Grafana and Jenkins. + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiOTelLogFormat.html b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiOTelLogFormat.html new file mode 100644 index 00000000..cdb62a08 --- /dev/null +++ b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiOTelLogFormat.html @@ -0,0 +1,13 @@ + +
+ Before version 3.0, Loki was storing OpenTelemetry logs as JSON messages including the log body, the log fields + (spanID, traceID, severity...), the log attributes, and the resource attributes.
+ Starting from version 3.0, Loki stores OpenTelemetry logs sent to its native OTLP endpoint using labels and + structured metadata for the log fields, attributes, and resource attributes (more details in Loki OpenTelemetry docs ).
+ Indicate the format of the logs stored in Loki to adapt the LogQL queries generated by the Jenkins OpenTelemetry + plugin. +
\ No newline at end of file diff --git a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiTenantId.html b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiTenantId.html new file mode 100644 index 00000000..417b4eca --- /dev/null +++ b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithJenkinsVisualization/help-lokiTenantId.html @@ -0,0 +1,10 @@ + + +
+ When using Loki multi-tenant mode, you need to specify the tenant ID when retrieving logs.
+ The tenant Id is passed in requests to the Loki HTTP API through the X-Scope-OrgID header.
+ More details on Loki multi-tenant mode. +
\ No newline at end of file diff --git a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization/config.jelly b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization/config.jelly index a584aeb3..ee010d62 100644 --- a/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization/config.jelly +++ b/src/main/resources/io/jenkins/plugins/opentelemetry/backend/grafana/GrafanaLogsBackendWithoutJenkinsVisualization/config.jelly @@ -12,7 +12,7 @@ - + diff --git a/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorIT.java b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorIT.java new file mode 100644 index 00000000..c3b057e3 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorIT.java @@ -0,0 +1,68 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import io.jenkins.plugins.opentelemetry.job.log.LogLine; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.internal.JavaVersionSpecific; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.BasicHttpContext; +import org.junit.Test; + +import java.io.InputStream; +import java.time.Instant; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class LokiBuildLogsLineIteratorIT { + @Test + public void overallLog() throws Exception { + System.out.println("OTel Java Specific Version: " + JavaVersionSpecific.get()); + + InputStream env = Thread.currentThread().getContextClassLoader().getResourceAsStream(".env"); + Properties properties = new Properties(); + properties.load(env); + String lokiUser = properties.getProperty("loki.user"); + String lokiPassword = properties.getProperty("loki.apiKey"); + UsernamePasswordCredentials lokiCredentials = new UsernamePasswordCredentials(lokiUser, lokiPassword); + + String lokiUrl = properties.getProperty("loki.url"); + + System.out.println(lokiUrl); + System.out.println(lokiUser); + + CloseableHttpClient httpClient = HttpClientBuilder.create().build(); + + Instant pipelineStartTime = Instant.ofEpochMilli(TimeUnit.MILLISECONDS.convert(1718111754515426000L, TimeUnit.NANOSECONDS)); + + + LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters = new LokiGetJenkinsBuildLogsQueryParametersBuilder() + .setJobFullName("my-war/master") + .setRunNumber(384) + .setTraceId("69a627b7bc02241b6029bed20f4ff8d8") + .setStartTime(pipelineStartTime.minusSeconds(600)) + .setEndTime(pipelineStartTime.plusSeconds(600)) + .setServiceName("jenkins") + .setServiceNamespace("jenkins") + .build(); + try (LokiBuildLogsLineIterator lokiBuildLogsLineIterator = new LokiBuildLogsLineIterator( + lokiQueryParameters, httpClient, + new BasicHttpContext(), + lokiUrl, + Optional.of(lokiCredentials), + Optional.empty(), + OpenTelemetry.noop().getTracer("io.jenkins") + )) { + while (lokiBuildLogsLineIterator.hasNext()) { + LogLine line = lokiBuildLogsLineIterator.next(); + System.out.println(line.getMessage()); + } + } + } +} \ No newline at end of file diff --git a/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorTest.java b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorTest.java new file mode 100644 index 00000000..e715dff1 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiBuildLogsLineIteratorTest.java @@ -0,0 +1,60 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import io.jenkins.plugins.opentelemetry.job.log.LogLine; +import io.opentelemetry.api.OpenTelemetry; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.BasicHttpContext; +import org.junit.Test; + +import java.io.InputStream; +import java.time.Instant; +import java.util.Iterator; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class LokiBuildLogsLineIteratorTest { + + @Test + public void testLoadLokiQueryResponse() { + CloseableHttpClient httpClient = HttpClientBuilder.create().build(); + + Instant pipelineStartTime = Instant.ofEpochMilli(TimeUnit.MILLISECONDS.convert(1718111754515426000L, TimeUnit.NANOSECONDS)); + + InputStream lokiLogsQueryResponseStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("io/jenkins/plugins/opentelemetry/backend/grafana/loki_query_response.json"); + assertNotNull(lokiLogsQueryResponseStream); + LokiGetJenkinsBuildLogsQueryParameters lokiQueryParameters = new LokiGetJenkinsBuildLogsQueryParametersBuilder() + .setJobFullName("my-war/master").setRunNumber(384) + .setTraceId("69a627b7bc02241b6029bed20f4ff8d8") + .setStartTime(pipelineStartTime.minusSeconds(600)) + .setEndTime(pipelineStartTime.plusSeconds(600)) + .setServiceName("jenkins") + .setServiceNamespace("jenkins") + .build(); + try (LokiBuildLogsLineIterator lokiBuildLogsLineIterator = new LokiBuildLogsLineIterator( + lokiQueryParameters, httpClient, + new BasicHttpContext(), + "http://localhost:3100", + Optional.of(new UsernamePasswordCredentials("jenkins", "jenkins")), + Optional.empty(), + OpenTelemetry.noop().getTracer("io.jenkins") + )) { + Iterator> logLines = lokiBuildLogsLineIterator.loadLogLines(lokiLogsQueryResponseStream); + while (logLines.hasNext()) { + LogLine logLine = logLines.next(); + System.out.println(logLine); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetrieverIT.java b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetrieverIT.java new file mode 100644 index 00000000..ec9ed0bc --- /dev/null +++ b/src/test/java/io/jenkins/plugins/opentelemetry/backend/grafana/LokiLogStorageRetrieverIT.java @@ -0,0 +1,55 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.backend.grafana; + +import groovy.text.GStringTemplateEngine; +import hudson.util.FormValidation; +import io.jenkins.plugins.opentelemetry.TemplateBindingsProvider; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.internal.JavaVersionSpecific; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.junit.Test; + +import java.io.InputStream; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +public class LokiLogStorageRetrieverIT { + + @Test + public void test_checkLokiSetup() throws Exception { + System.out.println("OTel Java Specific Version: " + JavaVersionSpecific.get()); + + InputStream env = Thread.currentThread().getContextClassLoader().getResourceAsStream(".env"); + Properties properties = new Properties(); + properties.load(env); + String lokiUser = properties.getProperty("loki.user"); + String lokiPassword = properties.getProperty("loki.apiKey"); + Optional lokiCredentials = Optional.of(new UsernamePasswordCredentials(lokiUser, lokiPassword)); + + String lokiUrl = properties.getProperty("loki.url"); + + System.out.println(lokiUrl); + System.out.println(lokiUser); + + try (LokiLogStorageRetriever lokiLogStorageRetriever = new LokiLogStorageRetriever( + lokiUrl, + false, + lokiCredentials, + Optional.empty(), + new GStringTemplateEngine().createTemplate("mock"), + TemplateBindingsProvider.empty(), + "jenkins", + Optional.of("jenkins"), + OpenTelemetry.noop() + )) { + List formValidations = lokiLogStorageRetriever.checkLokiSetup(); + System.out.println(formValidations); + } + } +} \ No newline at end of file diff --git a/src/test/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/MonotonicClockTest.java b/src/test/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/MonotonicClockTest.java new file mode 100644 index 00000000..7a903086 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/opentelemetry/opentelemetry/common/MonotonicClockTest.java @@ -0,0 +1,32 @@ +/* + * Copyright The Original Author or Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.jenkins.plugins.opentelemetry.opentelemetry.common; + +import io.opentelemetry.sdk.common.Clock; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class MonotonicClockTest { + + @Test + public void nowHighPrecision() { + + long previousTimestamp = 0; + Clock clock = Clocks.monotonicClock(); + long singleIncrements = 0; + for (int i = 0; i < 10_000; i++) { + long timestamp = clock.now(true); + if (previousTimestamp >= timestamp) { + fail("Timestamps are not monotonic"); + } else if (previousTimestamp + 1 == timestamp) { + singleIncrements++; + } + previousTimestamp = timestamp; + } + System.out.println("Single increments: " + singleIncrements); + } +} \ No newline at end of file diff --git a/src/test/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilterTest.java b/src/test/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilterTest.java index db9805d6..8dfe9fda 100644 --- a/src/test/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilterTest.java +++ b/src/test/java/io/jenkins/plugins/opentelemetry/servlet/StaplerInstrumentationServletFilterTest.java @@ -82,7 +82,7 @@ private void verifyJobUrlParsing(StaplerInstrumentationServletFilter.ParsedJobUr .filter(t -> !t.isEmpty()) .collect(Collectors.toList()); - StaplerInstrumentationServletFilter.ParsedJobUrl actual = new StaplerInstrumentationServletFilter(OpenTelemetry.noop().getTracer("test")).parseJobUrl(pathInfoTokens); + StaplerInstrumentationServletFilter.ParsedJobUrl actual = new StaplerInstrumentationServletFilter(Collections.emptyList(), OpenTelemetry.noop().getTracer("test")).parseJobUrl(pathInfoTokens); System.out.println(actual); Assert.assertEquals(expected, actual); } @@ -237,7 +237,7 @@ private void verifyBlueOceanRestPipelineUrlParsing(StaplerInstrumentationServlet .collect(Collectors.toList()); try { - StaplerInstrumentationServletFilter.ParsedJobUrl actual = new StaplerInstrumentationServletFilter(OpenTelemetry.noop().getTracer("test")).parseBlueOceanRestPipelineUrl(pathInfoTokens); + StaplerInstrumentationServletFilter.ParsedJobUrl actual = new StaplerInstrumentationServletFilter(Collections.emptyList(), OpenTelemetry.noop().getTracer("test")).parseBlueOceanRestPipelineUrl(pathInfoTokens); System.out.println(actual); Assert.assertEquals(expected, actual); diff --git a/src/test/resources/io/jenkins/plugins/opentelemetry/backend/grafana/loki_query_response.json b/src/test/resources/io/jenkins/plugins/opentelemetry/backend/grafana/loki_query_response.json new file mode 100644 index 00000000..3324de28 --- /dev/null +++ b/src/test/resources/io/jenkins/plugins/opentelemetry/backend/grafana/loki_query_response.json @@ -0,0 +1,536 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": {}, + "values": [ + [ + "1718111754515426000", + "Started by user unknown or anonymous" + ], + [ + "1718111755283666000", + "Connecting to https://api.github.com using cyrille-leclerc/******" + ], + [ + "1718111756417529000", + "Obtained Jenkinsfile from 8af211e392acba2eba4648dad5e5174fd9504346" + ], + [ + "1718111757236311000", + "[Pipeline] Start of Pipeline" + ], + [ + "1718111758034978000", + "[Pipeline] node" + ], + [ + "1718111758115323000", + "Running on macos in /Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master" + ], + [ + "1718111758185011000", + "[Pipeline] {" + ], + [ + "1718111758240007000", + "[Pipeline] stage" + ], + [ + "1718111758272139000", + "[Pipeline] { (Declarative: Checkout SCM)" + ], + [ + "1718111758322139000", + "[Pipeline] checkout" + ], + [ + "1718111758353795000", + "The recommended git tool is: NONE" + ], + [ + "1718111761934560000", + "Cloning repository https://github.com/cyrille-leclerc/my-war.git" + ], + [ + "1718111761969560000", + " \u003e git init /Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master # timeout=10" + ], + [ + "1718111761995548000", + "using credential github" + ], + [ + "1718111762010560000", + "Fetching upstream changes from https://github.com/cyrille-leclerc/my-war.git" + ], + [ + "1718111762010560001", + " \u003e git --version # timeout=10" + ], + [ + "1718111762018409000", + "Cloning the remote Git repository" + ], + [ + "1718111762020046000", + "Cloning with configured refspecs honoured and without tags" + ], + [ + "1718111762023560000", + " \u003e git --version # 'git version 2.39.3 (Apple Git-146)'" + ], + [ + "1718111762023560001", + "using GIT_ASKPASS to set credentials " + ], + [ + "1718111762024560000", + " \u003e git fetch --no-tags --force --progress -- https://github.com/cyrille-leclerc/my-war.git +refs/heads/master:refs/remotes/origin/master # timeout=10" + ], + [ + "1718111762915560000", + " \u003e git config remote.origin.url https://github.com/cyrille-leclerc/my-war.git # timeout=10" + ], + [ + "1718111762927560000", + " \u003e git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master # timeout=10" + ], + [ + "1718111763264560000", + " \u003e git config core.sparsecheckout # timeout=10" + ], + [ + "1718111763278560000", + " \u003e git checkout -f 8af211e392acba2eba4648dad5e5174fd9504346 # timeout=10" + ], + [ + "1718111763389943000", + "Avoid second fetch" + ], + [ + "1718111763393007000", + "Checking out Revision 8af211e392acba2eba4648dad5e5174fd9504346 (master)" + ], + [ + "1718111763459560000", + " \u003e git rev-list --no-walk 8af211e392acba2eba4648dad5e5174fd9504346 # timeout=10" + ], + [ + "1718111763588135000", + "Commit message: \"Add Maven extension\"" + ], + [ + "1718111764182470000", + "[Pipeline] }" + ], + [ + "1718111764217237000", + "[Pipeline] // stage" + ], + [ + "1718111764263372000", + "[Pipeline] withEnv" + ], + [ + "1718111764268144000", + "[Pipeline] {" + ], + [ + "1718111764312333000", + "[Pipeline] stage" + ], + [ + "1718111764317185000", + "[Pipeline] { (Build)" + ], + [ + "1718111764376496000", + "[Pipeline] sh" + ], + [ + "1718111764499638000", + "+ ./mvnw verify" + ], + [ + "1718111764723638000", + "--2024-06-11 15:16:04-- https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar" + ], + [ + "1718111764829638000", + "Resolving repo.maven.apache.org (repo.maven.apache.org)... 151.101.0.215, 151.101.64.215, 151.101.128.215, ..." + ], + [ + "1718111764829638001", + "Connecting to repo.maven.apache.org (repo.maven.apache.org)|151.101.0.215|:443... connected." + ], + [ + "1718111764830638000", + "HTTP request sent, awaiting response... 200 OK" + ], + [ + "1718111764830638001", + "Length: 59925 (59K) [application/java-archive]" + ], + [ + "1718111764830638002", + "Saving to: ‘/Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/.mvn/wrapper/maven-wrapper.jar’" + ], + [ + "1718111764830638003", + " 0K .......... .......... .......... .......... .......... 85% 4.78M 0s" + ], + [ + "1718111764830638004", + " 50K ........ 100% 6.71M=0.01s" + ], + [ + "1718111764830638005", + "2024-06-11 15:16:04 (4.99 MB/s) - ‘/Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/.mvn/wrapper/maven-wrapper.jar’ saved [59925/59925]" + ], + [ + "1718111765662638000", + "[INFO] Scanning for projects..." + ], + [ + "1718111765662638001", + "[INFO] " + ], + [ + "1718111765662638002", + "[INFO] -------------------------\u003c com.example:my-war \u003e-------------------------" + ], + [ + "1718111765662638003", + "[INFO] Building my-warWebapp 1.0-SNAPSHOT" + ], + [ + "1718111765662638004", + "[INFO] --------------------------------[ war ]---------------------------------" + ], + [ + "1718111765768638000", + "[INFO] " + ], + [ + "1718111765768638001", + "[INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ my-war ---" + ], + [ + "1718111765873638000", + "[INFO] Using 'UTF-8' encoding to copy filtered resources." + ], + [ + "1718111765873638001", + "[INFO] skip non existing resourceDirectory /Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/src/main/resources" + ], + [ + "1718111765873638002", + "[INFO] " + ], + [ + "1718111765873638003", + "[INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ my-war ---" + ], + [ + "1718111765979638000", + "[INFO] No sources to compile" + ], + [ + "1718111765979638001", + "[INFO] " + ], + [ + "1718111765979638002", + "[INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ my-war ---" + ], + [ + "1718111765979638003", + "[INFO] Using 'UTF-8' encoding to copy filtered resources." + ], + [ + "1718111765979638004", + "[INFO] skip non existing resourceDirectory /Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/src/test/resources" + ], + [ + "1718111765979638005", + "[INFO] " + ], + [ + "1718111765979638006", + "[INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ my-war ---" + ], + [ + "1718111765979638007", + "[INFO] No sources to compile" + ], + [ + "1718111765979638008", + "[INFO] " + ], + [ + "1718111765979638009", + "[INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ my-war ---" + ], + [ + "1718111765979638010", + "[INFO] No tests to run." + ], + [ + "1718111765979638011", + "[INFO] " + ], + [ + "1718111765979638012", + "[INFO] --- maven-war-plugin:3.3.2:war (default-war) @ my-war ---" + ], + [ + "1718111766085638000", + "[INFO] Packaging webapp" + ], + [ + "1718111766085638001", + "[INFO] Assembling webapp [my-war] in [/Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/target/my-war-1.0-SNAPSHOT]" + ], + [ + "1718111766085638002", + "[INFO] Processing war project" + ], + [ + "1718111766085638003", + "[INFO] Copying webapp resources [/Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/src/main/webapp]" + ], + [ + "1718111766190638000", + "[INFO] Building war: /Users/cyrilleleclerc/jenkins-agent/workspace/my-war_master/target/my-war-1.0-SNAPSHOT.war" + ], + [ + "1718111766191638000", + "[INFO] ------------------------------------------------------------------------" + ], + [ + "1718111766191638001", + "[INFO] BUILD SUCCESS" + ], + [ + "1718111766191638002", + "[INFO] ------------------------------------------------------------------------" + ], + [ + "1718111766191638003", + "[INFO] Total time: 0.599 s" + ], + [ + "1718111766191638004", + "[INFO] Finished at: 2024-06-11T15:16:06+02:00" + ], + [ + "1718111766191638005", + "[INFO] ------------------------------------------------------------------------" + ], + [ + "1718111766268113000", + "[Pipeline] archiveArtifacts" + ], + [ + "1718111766283865000", + "Archiving artifacts" + ], + [ + "1718111766384841000", + "Recording fingerprints" + ], + [ + "1718111766407176000", + "[Pipeline] }" + ], + [ + "1718111766433199000", + "[Pipeline] // stage" + ], + [ + "1718111766442109000", + "[Pipeline] }" + ], + [ + "1718111766465175000", + "[Pipeline] // withEnv" + ], + [ + "1718111766474122000", + "[Pipeline] }" + ], + [ + "1718111766502617000", + "[Pipeline] // node" + ], + [ + "1718111766513272000", + "[Pipeline] End of Pipeline" + ], + [ + "1718111766990600000", + "GitHub has been notified of this commit’s build result" + ], + [ + "1718111766991483000", + "Finished: SUCCESS" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 1436113, + "linesProcessedPerSecond": 2298, + "totalBytesProcessed": 60602, + "totalLinesProcessed": 97, + "execTime": 0.042199, + "queueTime": 0.035091, + "subqueries": 0, + "totalEntriesReturned": 93, + "splits": 2, + "shards": 1, + "totalPostFilterLines": 93, + "totalStructuredMetadataBytesProcessed": 53650 + }, + "querier": { + "store": { + "totalChunksRef": 1, + "totalChunksDownloaded": 1, + "chunksDownloadTime": 948433, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 60602, + "decompressedLines": 97, + "compressedBytes": 3675, + "totalDuplicates": 0, + "postFilterLines": 93, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 53650 + }, + "chunkRefsFetchTime": 1079437, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 0, + "totalChunksMatched": 0, + "totalBatches": 0, + "totalLinesSent": 0, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 1, + "entriesRequested": 1, + "entriesStored": 0, + "bytesReceived": 6413, + "bytesSent": 0, + "requests": 3, + "downloadTime": 804381, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 1, + "entriesRequested": 1, + "entriesStored": 0, + "bytesReceived": 460, + "bytesSent": 0, + "requests": 1, + "downloadTime": 319081, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "entriesStored": 0, + "bytesReceived": 278, + "bytesSent": 0, + "requests": 1, + "downloadTime": 265372, + "queryLengthServed": 641000000000 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0 + } + } + } +} \ No newline at end of file