From b5748bc19308240c00f966af2804833229261caf Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 3 Jul 2024 10:55:08 -0700 Subject: [PATCH 01/22] sending metrics to openTel exporter --- .../implementation/MetricDataMapper.java | 29 +++++++++++++++++++ .../quickpulse/QuickPulseDataCollector.java | 1 + 2 files changed, 30 insertions(+) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java index 84fa1b6869b51..ad959b571b893 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java @@ -72,6 +72,35 @@ public MetricDataMapper( this.captureHttpServer4xxAsError = captureHttpServer4xxAsError; } + public void mapMetrics(MetricData metricData, Consumer breezeConsumer, Consumer quickPulseConsumer) { + MetricDataType type = metricData.getType(); + if (type == DOUBLE_SUM + || type == DOUBLE_GAUGE + || type == LONG_SUM + || type == LONG_GAUGE + || type == HISTOGRAM) { + boolean isPreAggregatedStandardMetric = + OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); + if (isPreAggregatedStandardMetric) { + List preAggregatedStandardMetrics = + convertOtelMetricToAzureMonitorMetric(metricData, true); + preAggregatedStandardMetrics.forEach(breezeConsumer::accept); + } + + // DO NOT emit unstable metrics from the OpenTelemetry auto instrumentation libraries + // custom metrics are always emitted + if (OTEL_UNSTABLE_METRICS_TO_EXCLUDE.contains(metricData.getName()) + && metricData.getInstrumentationScopeInfo().getName().startsWith(OTEL_INSTRUMENTATION_NAME_PREFIX)) { + return; + } + List stableOtelMetrics = convertOtelMetricToAzureMonitorMetric(metricData, false); + stableOtelMetrics.forEach(quickPulseConsumer::accept); + stableOtelMetrics.forEach(breezeConsumer::accept); + } else { + logger.warning("metric data type {} is not supported yet.", metricData.getType()); + } + } + public void map(MetricData metricData, Consumer consumer) { MetricDataType type = metricData.getType(); if (type == DOUBLE_SUM diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 6568b62878cb1..c35b8df5db328 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -122,6 +122,7 @@ void add(TelemetryItem telemetryItem) { } else if (data instanceof TelemetryExceptionData) { addException((TelemetryExceptionData) data, itemCount); } + System.out.println("Testing if this will show up on terminal"); } boolean isEnabled() { From e072720403b5aa74e35b403a2c5624a3fd372fd2 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 3 Jul 2024 12:06:32 -0700 Subject: [PATCH 02/22] working base version --- .../quickpulse/QuickPulseDataCollector.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index c35b8df5db328..3c5238b7e49ad 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -3,19 +3,13 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; -import com.azure.monitor.opentelemetry.exporter.implementation.models.ContextTagKeys; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; -import com.azure.monitor.opentelemetry.exporter.implementation.models.RemoteDependencyData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.RequestData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.StackFrame; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryExceptionData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryExceptionDetails; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; +import com.azure.monitor.opentelemetry.exporter.implementation.models.*; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDependencyDocument; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDocument; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseExceptionDocument; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseRequestDocument; import com.azure.monitor.opentelemetry.exporter.implementation.utils.CpuPerformanceCounterCalculator; +import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; import java.lang.management.ManagementFactory; @@ -23,10 +17,7 @@ import java.lang.management.MemoryUsage; import java.lang.management.OperatingSystemMXBean; import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -121,8 +112,20 @@ void add(TelemetryItem telemetryItem) { addDependency((RemoteDependencyData) data, itemCount); } else if (data instanceof TelemetryExceptionData) { addException((TelemetryExceptionData) data, itemCount); + }else { + if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { + MonitorDomain data2 = telemetryItem.getData().getBaseData(); + MetricsData metricsData = (MetricsData) data2; + MetricDataPoint point = metricsData.getMetrics().get(0); + System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); + System.out.println("SDK Metric Name: " + point.getName()); + System.out.println("SDK Metric Value: " + point.getValue()); + System.out.println("SDK Metric attributes: " + metricsData.getProperties()); + //System.out.println("SDK Metric Type: " + point.getType()); + } + + } - System.out.println("Testing if this will show up on terminal"); } boolean isEnabled() { From 59db9dff6f815e16859f873bb72cb43d21888b1f Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 3 Jul 2024 14:26:42 -0700 Subject: [PATCH 03/22] testing metric interval --- .../implementation/quickpulse/QuickPulse.java | 18 +++++++++++++ .../quickpulse/QuickPulseDataCollector.java | 27 ++++++++++--------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index c7260acd54b10..f43bbcb4d4b29 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -5,13 +5,18 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpRequest; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; import com.azure.monitor.opentelemetry.exporter.implementation.utils.HostName; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; import com.azure.monitor.opentelemetry.exporter.implementation.utils.ThreadPoolUtils; +import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; import java.net.URL; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; @@ -77,6 +82,19 @@ public void add(TelemetryItem telemetryItem) { } } + public void addTest(TelemetryItem telemetryItem) { + if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { + MonitorDomain data2 = telemetryItem.getData().getBaseData(); + MetricsData metricsData = (MetricsData) data2; + MetricDataPoint point = metricsData.getMetrics().get(0); + System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); + System.out.println("SDK Metric Name: " + point.getName()); + System.out.println("SDK Metric Value: " + point.getValue()); + System.out.println("SDK Metric attributes: " + metricsData.getProperties()); + //System.out.println("SDK Metric Type: " + point.getType()); + } + } + private void initialize( HttpPipeline httpPipeline, Supplier endpointUrl, diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 3c5238b7e49ad..8be2105368bdf 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -112,20 +112,21 @@ void add(TelemetryItem telemetryItem) { addDependency((RemoteDependencyData) data, itemCount); } else if (data instanceof TelemetryExceptionData) { addException((TelemetryExceptionData) data, itemCount); - }else { - if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { - MonitorDomain data2 = telemetryItem.getData().getBaseData(); - MetricsData metricsData = (MetricsData) data2; - MetricDataPoint point = metricsData.getMetrics().get(0); - System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); - System.out.println("SDK Metric Name: " + point.getName()); - System.out.println("SDK Metric Value: " + point.getValue()); - System.out.println("SDK Metric attributes: " + metricsData.getProperties()); - //System.out.println("SDK Metric Type: " + point.getType()); - } - - } +// else { +// if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { +// MonitorDomain data2 = telemetryItem.getData().getBaseData(); +// MetricsData metricsData = (MetricsData) data2; +// MetricDataPoint point = metricsData.getMetrics().get(0); +// System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); +// System.out.println("SDK Metric Name: " + point.getName()); +// System.out.println("SDK Metric Value: " + point.getValue()); +// System.out.println("SDK Metric attributes: " + metricsData.getProperties()); +// //System.out.println("SDK Metric Type: " + point.getType()); +// } +// +// +// } } boolean isEnabled() { From 7280ddf013cc13286d1a70473f8bd43ac5d1fa5b Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Fri, 5 Jul 2024 14:57:49 -0700 Subject: [PATCH 04/22] added etag variable --- .../implementation/quickpulse/QuickPulse.java | 18 +++++------ .../quickpulse/QuickPulseDataCollector.java | 31 +++++++++++++++++++ .../quickpulse/QuickPulseDataSender.java | 2 ++ .../quickpulse/QuickPulseNetworkHelper.java | 9 ++++++ .../QuickPulseDataFetcherTests.java | 1 + .../quickpulse/QuickPulsePingSenderTests.java | 1 + 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index f43bbcb4d4b29..df20f2b464d80 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -82,16 +82,14 @@ public void add(TelemetryItem telemetryItem) { } } - public void addTest(TelemetryItem telemetryItem) { - if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { - MonitorDomain data2 = telemetryItem.getData().getBaseData(); - MetricsData metricsData = (MetricsData) data2; - MetricDataPoint point = metricsData.getMetrics().get(0); - System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); - System.out.println("SDK Metric Name: " + point.getName()); - System.out.println("SDK Metric Value: " + point.getValue()); - System.out.println("SDK Metric attributes: " + metricsData.getProperties()); - //System.out.println("SDK Metric Type: " + point.getType()); + public void add(TelemetryItem telemetryItem, Boolean isOtelMetric) { + if (collector != null){ + if(isOtelMetric){ + collector.addOtelMetric(telemetryItem); + + }else{ + collector.add(telemetryItem); + } } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 8be2105368bdf..4e7bf08f202b6 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -87,6 +87,37 @@ synchronized FinalCounters peek() { return null; } + + void addOtelMetric(TelemetryItem telemetryItem){ + if (!isEnabled()) { + // quick pulse is not enabled or quick pulse data sender is not enabled + return; + } + + if (!telemetryItem.getInstrumentationKey().equals(instrumentationKeySupplier.get())) { + return; + } + + Float sampleRate = telemetryItem.getSampleRate(); + if (sampleRate != null && sampleRate == 0) { + // sampleRate should never be zero (how could it be captured if sampling set to zero percent?) + return; + } + int itemCount = sampleRate == null ? 1 : Math.round(100 / sampleRate); + + if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { + MonitorDomain data2 = telemetryItem.getData().getBaseData(); + MetricsData metricsData = (MetricsData) data2; + MetricDataPoint point = metricsData.getMetrics().get(0); + System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); + System.out.println("SDK Metric Name: " + point.getName()); + System.out.println("SDK Metric Value: " + point.getValue()); + System.out.println("SDK Metric attributes: " + metricsData.getProperties()); + //System.out.println("SDK Metric Type: " + point.getType()); + } + + } + void add(TelemetryItem telemetryItem) { if (!isEnabled()) { // quick pulse is not enabled or quick pulse data sender is not enabled diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 03addfda2347d..4f2b121655045 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -6,6 +6,8 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.concurrent.ArrayBlockingQueue; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index 400f10e79e25f..6a3341d836f94 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -26,11 +26,13 @@ final class QuickPulseNetworkHelper { private static final String QPS_MACHINE_NAME_HEADER = "x-ms-qps-machine-name"; private static final String QPS_ROLE_NAME_HEADER = "x-ms-qps-role-name"; private static final String QPS_INVARIANT_VERSION_HEADER = "x-ms-qps-invariant-version"; + private static final String QPS_CONFIGURATION_ETAG_HEADER = "x-ms-qps-configuration-etag"; private static final HttpHeaderName QPS_ROLE_NAME_HEADER_NAME = HttpHeaderName.fromString(QPS_ROLE_NAME_HEADER); private static final HttpHeaderName QPS_MACHINE_NAME_HEADER_NAME = HttpHeaderName.fromString(QPS_MACHINE_NAME_HEADER); private static final HttpHeaderName QPS_STREAM_ID_HEADER_NAME = HttpHeaderName.fromString(QPS_STREAM_ID_HEADER); private static final HttpHeaderName QPS_INSTANCE_NAME_HEADER_NAME = HttpHeaderName.fromString(QPS_INSTANCE_NAME_HEADER); private static final HttpHeaderName QPS_INVARIANT_VERSION_HEADER_NAME = HttpHeaderName.fromString(QPS_INVARIANT_VERSION_HEADER); + private static final HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER_NAME = HttpHeaderName.fromString(QPS_CONFIGURATION_ETAG_HEADER); HttpRequest buildPingRequest( Date currentDate, @@ -46,6 +48,7 @@ HttpRequest buildPingRequest( request.setHeader(QPS_STREAM_ID_HEADER_NAME, quickPulseId); request.setHeader(QPS_INSTANCE_NAME_HEADER_NAME, instanceName); request.setHeader(QPS_INVARIANT_VERSION_HEADER_NAME, Integer.toString(QuickPulse.QP_INVARIANT_VERSION)); + request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, ""); return request; } @@ -54,6 +57,7 @@ HttpRequest buildRequest(Date currentDate, String address) { HttpRequest request = new HttpRequest(HttpMethod.POST, address); request.setHeader(HttpHeaderName.fromString(HEADER_TRANSMISSION_TIME), String.valueOf(ticks)); + request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, ""); return request; } @@ -72,6 +76,11 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { String qpStatus = header.getValue(); if ("true".equalsIgnoreCase(qpStatus)) { status = QuickPulseStatus.QP_IS_ON; + String headName = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getName(); + String headValue = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); + System.out.println("************************"); + System.out.println(headName + ": " + headValue); + System.out.println("************************"); } else { status = QuickPulseStatus.QP_IS_OFF; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index 7c886a55351a2..1e5de33599079 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -89,6 +89,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); + headers.put("x-ms-qps-configuration-etag", "sample"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); HttpPipeline httpPipeline = diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 2fff1cb8f10a3..54dd00f4e15b9 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -76,6 +76,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); + headers.put("x-ms-qps-configuration-etag", "sample"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); HttpPipeline httpPipeline = From 26ddfbacaf3b82e9ea104391b2ca819a922640f1 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Tue, 9 Jul 2024 13:47:01 -0700 Subject: [PATCH 05/22] code for sending opentel metrics --- .../quickpulse/QuickPulseConfiguration.java | 148 ++++++++++++++++++ .../quickpulse/QuickPulseDataCollector.java | 105 ++++++++++++- .../quickpulse/QuickPulseDataFetcher.java | 10 +- .../quickpulse/QuickPulseDataSender.java | 20 +++ .../quickpulse/QuickPulseNetworkHelper.java | 39 +++-- .../quickpulse/QuickPulsePingSender.java | 15 ++ .../quickpulse/model/OpenTelDataPoint.java | 26 +++ .../quickpulse/model/OpenTelMetric.java | 52 ++++++ 8 files changed, 394 insertions(+), 21 deletions(-) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java new file mode 100644 index 0000000000000..fd83cee7afaaf --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -0,0 +1,148 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.logging.ClientLogger; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + + +public class QuickPulseConfiguration { + private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); + private static QuickPulseConfiguration instance = new QuickPulseConfiguration(); + private AtomicReference etag = new AtomicReference<>(); + private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + private static final ObjectMapper objectMapper = new ObjectMapper(); + + + private QuickPulseConfiguration() { + } + + public static synchronized QuickPulseConfiguration getInstance() { + if (instance == null) { + instance = new QuickPulseConfiguration(); + } + return instance; + } + + public synchronized String getEtag() { + return this.etag.get(); + } + + public synchronized void setEtag(String etag) { + this.etag.set(etag); + } + + public synchronized ConcurrentHashMap getMetrics() { + + if (this.metrics.isEmpty()) { + OpenTelMetricInfo metric = new OpenTelMetricInfo(); + metric.setId("my_gauge"); + metric.setAggregation("Avg"); + metric.setTelemetryType("Metric"); + metric.setProjection("my_gauge"); + + OpenTelMetricInfo metric2 = new OpenTelMetricInfo(); + metric2.setId("MyFruitCounter"); + metric2.setAggregation("Avg"); + metric2.setTelemetryType("Metric"); + metric2.setProjection("MyFruitCounter"); + + ConcurrentHashMap sampleMetrics = new ConcurrentHashMap<>(); + sampleMetrics.put("my_gauge", metric); + sampleMetrics.put("MyFruitCounter", metric2); + return sampleMetrics; + } + return this.metrics; + } + + public synchronized void setMetrics(ConcurrentHashMap metrics) { + this.metrics = metrics; + } + + public synchronized void updateConfig(String etagValue, ConcurrentHashMap otelMetrics) { + if (!Objects.equals(this.getEtag(), etagValue)){ + this.setEtag(etagValue); + this.setMetrics(otelMetrics); + } + + } + + public ConcurrentHashMap parseMetrics(HttpResponse response) { + + HashSet metricsSet = new HashSet<>(); + ConcurrentHashMap requestedMetrics = new ConcurrentHashMap<>(); + try { + + String responseBody = response.getBodyAsString().block(); + JsonNode rootNode = objectMapper.readTree(responseBody); + System.out.println("Metrics :" + rootNode.get("Metrics")); + JsonNode metricsNode = rootNode.get("Metrics"); + if (metricsNode instanceof ArrayNode) { + ArrayNode metricsArray = (ArrayNode) metricsNode; + for (JsonNode metricNode : metricsArray) { + OpenTelMetricInfo metric = new OpenTelMetricInfo(); + metric.setId(metricNode.get("Id").asText()); + metric.setAggregation(metricNode.get("Aggregation").asText()); + metric.setTelemetryType(metricNode.get("TelemetryType").asText()); + metric.setProjection(metricNode.get("Projection").asText()); + requestedMetrics.put(metricNode.get("Projection").asText(), metric); + + } + } + return requestedMetrics; + } catch (Exception e) { + logger.verbose("Failed to parse metrics from response: %s", e.getMessage()); + } + return new ConcurrentHashMap(); + + } + + class OpenTelMetricInfo { + private String id; + private String projection; + private String telemetryType; + private String aggregation; + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + public String getProjection() { + return projection; + } + + public void setTelemetryType(String telemetryType) { + this.telemetryType = telemetryType; + } + + public String getTelemetryType() { + return this.telemetryType; + } + + + public void setProjection(String projection) { + this.projection = projection; + } + + public String getAggregation() { + return this.aggregation; + } + + public void setAggregation(String aggregation) { + this.aggregation = aggregation; + } + } +} + + diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 4e7bf08f202b6..22282bd948f8b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -4,10 +4,7 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; import com.azure.monitor.opentelemetry.exporter.implementation.models.*; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDependencyDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseExceptionDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseRequestDocument; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.*; import com.azure.monitor.opentelemetry.exporter.implementation.utils.CpuPerformanceCounterCalculator; import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; @@ -18,10 +15,14 @@ import java.lang.management.OperatingSystemMXBean; import java.time.Duration; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + final class QuickPulseDataCollector { @@ -31,6 +32,11 @@ final class QuickPulseDataCollector { ManagementFactory.getOperatingSystemMXBean(); private final AtomicReference counters = new AtomicReference<>(null); + + private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + + private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); + private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator = getCpuPerformanceCounterCalculator(); private final boolean useNormalizedValueForNonNormalizedCpuPercentage; @@ -109,11 +115,13 @@ void addOtelMetric(TelemetryItem telemetryItem){ MonitorDomain data2 = telemetryItem.getData().getBaseData(); MetricsData metricsData = (MetricsData) data2; MetricDataPoint point = metricsData.getMetrics().get(0); - System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); - System.out.println("SDK Metric Name: " + point.getName()); - System.out.println("SDK Metric Value: " + point.getValue()); - System.out.println("SDK Metric attributes: " + metricsData.getProperties()); +// System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); +// System.out.println("SDK Metric Name: " + point.getName()); +// System.out.println("SDK Metric Value: " + point.getValue()); +// System.out.println("SDK Metric attributes: " + metricsData.getProperties()); + this.metricsStorage.addMetric(point.getName(), point.getValue()); //System.out.println("SDK Metric Type: " + point.getType()); + // metricsStorage.addMetric(point.getName(), point.getValue()); } } @@ -349,6 +357,10 @@ private static int charToInt(char c) { return x; } + public ArrayList retrieveOpenTelMetrics() { + return metricsStorage.processMetrics(); + } + class FinalCounters { final int exceptions; @@ -449,4 +461,81 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { return new CountAndDuration(countAndDuration >> 44, countAndDuration & MAX_DURATION); } } + + class OpenTelMetricsStorage { + private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + + public void addMetric(String metricName, double value) { + OpenTelMetric metric = metrics.get(metricName); + if (metric == null) { + metric = new OpenTelMetric(metricName); + metric.addDataPoint(value); + metrics.putIfAbsent(metricName, metric); + } else { + metric.addDataPoint(value); + } + } + + public ArrayList processMetrics() { + ConcurrentHashMap requestedMetrics = quickPulseConfiguration.getMetrics(); + ArrayList processedMetrics = new ArrayList<>(); + + Iterator> iterator = this.metrics.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String key = entry.getKey(); + OpenTelMetric value = entry.getValue(); + + if (requestedMetrics.containsKey(key)) { + QuickPulseMetrics processedMetric = processMetric(value, requestedMetrics.get(key)); + processedMetrics.add(processedMetric); + } + + //change to desired time limit for metrics + System.out.println("***************"); + System.out.println("METRIC: " + key + " LAST TIMESTAMP: " + value.getLastTimestamp() + " CURRENT TIME: " + LocalDateTime.now() + " DIFFERENCE: " + ChronoUnit.SECONDS.between(LocalDateTime.now(), value.getLastTimestamp())); + if (ChronoUnit.SECONDS.between(value.getLastTimestamp(), LocalDateTime.now()) > 5) { + System.out.println("***************"); + System.out.println("REMOVING METRIC: " + key); + iterator.remove(); + + } + else { + value.clearDataPoints(); + } + + } + return processedMetrics; + + } + + public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfiguration.OpenTelMetricInfo metricInfo) { + + if (metric.getDataPoints().isEmpty()) { + return new QuickPulseMetrics(metricInfo.getId(), 0, 1); + } + + String aggregation = metricInfo.getAggregation(); + ArrayList dataValues = metric.getDataValues(); + + switch (aggregation) { + case "Sum": + double sum = dataValues.stream().mapToDouble(Double::doubleValue).sum(); + return new QuickPulseMetrics(metricInfo.getId(), sum, 1); + case "Avg": + double avg = dataValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); + return new QuickPulseMetrics(metricInfo.getId(), avg, dataValues.size()); + case "Min": + double min = dataValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); + return new QuickPulseMetrics(metricInfo.getId(), min, 1); + case "Max": + double max = dataValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); + return new QuickPulseMetrics(metricInfo.getId(), max, 1); + default: + throw new IllegalArgumentException("Aggregation type not supported: " + aggregation); + } + + } + + } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index faa2f72246a34..54865c594fc4c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -39,6 +39,7 @@ class QuickPulseDataFetcher { private final ArrayBlockingQueue sendQueue; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); + private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); private final Supplier endpointUrl; private final Supplier instrumentationKey; @@ -92,7 +93,7 @@ public void prepareQuickPulseDataForSend(String redirectedEndpoint) { String endpointPrefix = Strings.isNullOrEmpty(redirectedEndpoint) ? getQuickPulseEndpoint() : redirectedEndpoint; HttpRequest request = - networkHelper.buildRequest(currentDate, this.getEndpointUrl(endpointPrefix)); + networkHelper.buildRequest(currentDate, this.getEndpointUrl(endpointPrefix), quickPulseConfiguration.getEtag()); request.setBody(buildPostEntity(counters)); if (!sendQueue.offer(request)) { @@ -140,13 +141,14 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) postEnvelope.setStreamId(quickPulseId); postEnvelope.setVersion(sdkVersion); postEnvelope.setTimeStamp("/Date(" + System.currentTimeMillis() + ")/"); - postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters)); + postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters, collector.retrieveOpenTelMetrics())); envelopes.add(postEnvelope); return mapper.writeValueAsString(envelopes); } private static List addMetricsToQuickPulseEnvelope( - QuickPulseDataCollector.FinalCounters counters) { + QuickPulseDataCollector.FinalCounters counters, + List openTelemetryMetrics) { List metricsList = new ArrayList<>(); metricsList.add( new QuickPulseMetrics("\\ApplicationInsights\\Requests/Sec", counters.requests, 1)); @@ -188,7 +190,7 @@ private static List addMetricsToQuickPulseEnvelope( new QuickPulseMetrics("\\Memory\\Committed Bytes", counters.memoryCommitted, 1)); metricsList.add( new QuickPulseMetrics("\\Processor(_Total)\\% Processor Time", counters.cpuUsage, 1)); - + metricsList.addAll(openTelemetryMetrics); return metricsList; } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 4f2b121655045..a956be2909e56 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -9,11 +9,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; class QuickPulseDataSender implements Runnable { private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); + private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); private final HttpPipeline httpPipeline; private volatile QuickPulseHeaderInfo quickPulseHeaderInfo; private long lastValidTransmission = 0; @@ -54,6 +59,11 @@ public void run() { case QP_IS_ON: lastValidTransmission = sendTime; this.quickPulseHeaderInfo = quickPulseHeaderInfo; + String etagValue = networkHelper.getEtagHeaderValue(response); + if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { + ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + quickPulseConfiguration.updateConfig(etagValue, otelMetrics); + } break; case ERROR: @@ -61,7 +71,17 @@ public void run() { break; } } + +// String etagValue = networkHelper.getEtagHeaderValue(response); +// ArrayList otelMetrics = quickPulseConfiguration.parseMetrics(response); +// quickPulseConfiguration.updateConfig(etagValue, otelMetrics); + + + } + System.out.println("POST*********************"); + System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); + System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index 6a3341d836f94..50d064e2ae154 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -10,6 +10,10 @@ import com.azure.core.http.HttpResponse; import com.azure.core.http.HttpHeaderName; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.util.Date; @@ -33,6 +37,7 @@ final class QuickPulseNetworkHelper { private static final HttpHeaderName QPS_INSTANCE_NAME_HEADER_NAME = HttpHeaderName.fromString(QPS_INSTANCE_NAME_HEADER); private static final HttpHeaderName QPS_INVARIANT_VERSION_HEADER_NAME = HttpHeaderName.fromString(QPS_INVARIANT_VERSION_HEADER); private static final HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER_NAME = HttpHeaderName.fromString(QPS_CONFIGURATION_ETAG_HEADER); + private static final ObjectMapper objectMapper = new ObjectMapper(); HttpRequest buildPingRequest( Date currentDate, @@ -42,22 +47,21 @@ HttpRequest buildPingRequest( String roleName, String instanceName) { - HttpRequest request = buildRequest(currentDate, address); + HttpRequest request = buildRequest(currentDate, address, ""); request.setHeader(QPS_ROLE_NAME_HEADER_NAME, roleName); request.setHeader(QPS_MACHINE_NAME_HEADER_NAME, machineName); request.setHeader(QPS_STREAM_ID_HEADER_NAME, quickPulseId); request.setHeader(QPS_INSTANCE_NAME_HEADER_NAME, instanceName); request.setHeader(QPS_INVARIANT_VERSION_HEADER_NAME, Integer.toString(QuickPulse.QP_INVARIANT_VERSION)); - request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, ""); return request; } - HttpRequest buildRequest(Date currentDate, String address) { + HttpRequest buildRequest(Date currentDate, String address, String etag) { long ticks = currentDate.getTime() * 10000 + TICKS_AT_EPOCH; HttpRequest request = new HttpRequest(HttpMethod.POST, address); request.setHeader(HttpHeaderName.fromString(HEADER_TRANSMISSION_TIME), String.valueOf(ticks)); - request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, ""); + request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, etag); return request; } @@ -76,11 +80,22 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { String qpStatus = header.getValue(); if ("true".equalsIgnoreCase(qpStatus)) { status = QuickPulseStatus.QP_IS_ON; - String headName = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getName(); - String headValue = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); - System.out.println("************************"); - System.out.println(headName + ": " + headValue); - System.out.println("************************"); +// String headName = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getName(); +// String headValue = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); +// System.out.println("************************"); +// System.out.println(headName + ": " + headValue); +// System.out.println("************************"); +// System.out.println("Destructuring Body: "); +// String responseBody = response.getBodyAsString().block(); +// +// try { +// JsonNode rootNode = objectMapper.readTree(responseBody); +// System.out.println("Metrics :" + rootNode.get("Metrics")); +// } +// catch (Exception e) { +// System.out.println("Error: " + e); +// } + } else { status = QuickPulseStatus.QP_IS_OFF; } @@ -95,4 +110,10 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { } return new QuickPulseHeaderInfo(status, serviceEndpointRedirect, servicePollingIntervalHint); } + + String getEtagHeaderValue(HttpResponse response){ + HttpHeaders headers = response.getHeaders(); + return headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); + } + } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 7bfded430a98f..edd41ca02d2e1 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -18,7 +18,11 @@ import reactor.util.annotation.Nullable; import java.net.URL; +import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -45,6 +49,7 @@ class QuickPulsePingSender { private final HttpPipeline httpPipeline; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); + private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); private volatile QuickPulseEnvelope pingEnvelope; // cached for performance private final Supplier endpointUrl; @@ -76,6 +81,9 @@ class QuickPulsePingSender { } QuickPulseHeaderInfo ping(String redirectedEndpoint) { + System.out.println("PING*********************"); + System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); + System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); String instrumentationKey = getInstrumentationKey(); if (Strings.isNullOrEmpty(instrumentationKey)) { // Quick Pulse Ping uri will be null when the instrumentation key is null. When that happens, @@ -111,6 +119,11 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { case QP_IS_OFF: case QP_IS_ON: lastValidTransmission = sendTime; + String etagValue = networkHelper.getEtagHeaderValue(response); + if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { + ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + quickPulseConfiguration.updateConfig(etagValue, otelMetrics); + } operationLogger.recordSuccess(); return quickPulseHeaderInfo; @@ -126,6 +139,8 @@ t, getQuickPulseEndpoint(), friendlyExceptionThrown, logger)) { } } finally { if (response != null) { + + // need to consume the body or close the response, otherwise get netty ByteBuf leak // warnings: // io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java new file mode 100644 index 0000000000000..6c70bc5d36323 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java @@ -0,0 +1,26 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashMap; + +public class OpenTelDataPoint { + private double value; + //private HashMap dimensions; + + public OpenTelDataPoint(double value) { + this.value = value; + //this.dimensions = new HashMap<>(); + } + + public double getValue() { + return value; + } + + public void setValue(double value) { + this.value = value; + } + + + +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java new file mode 100644 index 0000000000000..32307b0e35369 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java @@ -0,0 +1,52 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; + +import java.util.ArrayList; +import java.time.LocalDateTime; + +public class OpenTelMetric { + + private String name; + private ArrayList dataPoints; + private LocalDateTime lastTimestamp; + + public OpenTelMetric(String name) { + this.name = name; + this.dataPoints = new ArrayList<>(); + this.lastTimestamp = LocalDateTime.now(); + } + + public void addDataPoint(double value) { + OpenTelDataPoint dataPoint = new OpenTelDataPoint(value); + this.dataPoints.add(dataPoint); + this.updateLastTimestamp(); + } + + public String getName() { + return name; + } + + public ArrayList getDataPoints() { + return dataPoints; + } + + public ArrayList getDataValues() { + ArrayList values = new ArrayList<>(); + for (OpenTelDataPoint dataPoint : dataPoints) { + values.add(dataPoint.getValue()); + } + return values; + } + + public LocalDateTime getLastTimestamp() { + return lastTimestamp; + } + + public void updateLastTimestamp() { + this.lastTimestamp = LocalDateTime.now(); + } + + public void clearDataPoints() { + this.dataPoints.clear(); + } + +} From 26f29349d8bbbcd4f77a8942dd78ee6e6a124fe6 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Thu, 11 Jul 2024 11:41:25 -0700 Subject: [PATCH 06/22] fixed custom metrics to display on Live --- .../implementation/quickpulse/QuickPulse.java | 3 +- .../quickpulse/QuickPulseConfiguration.java | 42 +++--- .../quickpulse/QuickPulseDataCollector.java | 25 ---- .../quickpulse/QuickPulseDataSender.java | 8 +- .../quickpulse/QuickPulseNetworkHelper.java | 16 --- .../quickpulse/QuickPulsePingSender.java | 2 + .../QuickPulseDataFetcherTests.java | 2 +- .../QuickPulseNetworkHelperTest.java | 16 +++ .../quickpulse/QuickPulsePingSenderTests.java | 123 +++++++++++++++++- 9 files changed, 165 insertions(+), 72 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index df20f2b464d80..d12f04ce22daf 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -25,7 +25,8 @@ public class QuickPulse { - static final int QP_INVARIANT_VERSION = 1; + //change this back to 1 + static final int QP_INVARIANT_VERSION = 5; private volatile QuickPulseDataCollector collector; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index fd83cee7afaaf..3c9a00d295a55 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -40,25 +40,6 @@ public synchronized void setEtag(String etag) { } public synchronized ConcurrentHashMap getMetrics() { - - if (this.metrics.isEmpty()) { - OpenTelMetricInfo metric = new OpenTelMetricInfo(); - metric.setId("my_gauge"); - metric.setAggregation("Avg"); - metric.setTelemetryType("Metric"); - metric.setProjection("my_gauge"); - - OpenTelMetricInfo metric2 = new OpenTelMetricInfo(); - metric2.setId("MyFruitCounter"); - metric2.setAggregation("Avg"); - metric2.setTelemetryType("Metric"); - metric2.setProjection("MyFruitCounter"); - - ConcurrentHashMap sampleMetrics = new ConcurrentHashMap<>(); - sampleMetrics.put("my_gauge", metric); - sampleMetrics.put("MyFruitCounter", metric2); - return sampleMetrics; - } return this.metrics; } @@ -81,9 +62,13 @@ public ConcurrentHashMap parseMetrics(HttpResponse re try { String responseBody = response.getBodyAsString().block(); + if (responseBody == null || responseBody.isEmpty()) { + return new ConcurrentHashMap(); + } JsonNode rootNode = objectMapper.readTree(responseBody); - System.out.println("Metrics :" + rootNode.get("Metrics")); + //System.out.println("Metrics :" + rootNode.get("Metrics")); Debugging purposes JsonNode metricsNode = rootNode.get("Metrics"); + if (metricsNode instanceof ArrayNode) { ArrayNode metricsArray = (ArrayNode) metricsNode; for (JsonNode metricNode : metricsArray) { @@ -91,8 +76,16 @@ public ConcurrentHashMap parseMetrics(HttpResponse re metric.setId(metricNode.get("Id").asText()); metric.setAggregation(metricNode.get("Aggregation").asText()); metric.setTelemetryType(metricNode.get("TelemetryType").asText()); - metric.setProjection(metricNode.get("Projection").asText()); - requestedMetrics.put(metricNode.get("Projection").asText(), metric); + String projection = metricNode.get("Projection").asText(); + metric.setProjection(projection); + if (Objects.equals(metricNode.get("TelemetryType").asText(), "Event")) { + int dotIndex = projection.indexOf("."); + if (dotIndex != -1) { + projection = projection.substring(dotIndex + 1); + } + } + metric.setProjection(projection); + requestedMetrics.put(projection, metric); } } @@ -104,6 +97,11 @@ public ConcurrentHashMap parseMetrics(HttpResponse re } + public synchronized void reset() { + this.setEtag(null); + this.metrics.clear(); + } + class OpenTelMetricInfo { private String id; private String projection; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 22282bd948f8b..f0a10af1522d5 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -115,13 +115,7 @@ void addOtelMetric(TelemetryItem telemetryItem){ MonitorDomain data2 = telemetryItem.getData().getBaseData(); MetricsData metricsData = (MetricsData) data2; MetricDataPoint point = metricsData.getMetrics().get(0); -// System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); -// System.out.println("SDK Metric Name: " + point.getName()); -// System.out.println("SDK Metric Value: " + point.getValue()); -// System.out.println("SDK Metric attributes: " + metricsData.getProperties()); this.metricsStorage.addMetric(point.getName(), point.getValue()); - //System.out.println("SDK Metric Type: " + point.getType()); - // metricsStorage.addMetric(point.getName(), point.getValue()); } } @@ -152,20 +146,6 @@ void add(TelemetryItem telemetryItem) { } else if (data instanceof TelemetryExceptionData) { addException((TelemetryExceptionData) data, itemCount); } -// else { -// if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { -// MonitorDomain data2 = telemetryItem.getData().getBaseData(); -// MetricsData metricsData = (MetricsData) data2; -// MetricDataPoint point = metricsData.getMetrics().get(0); -// System.out.println("SDK Metric Data Length: " + metricsData.getMetrics().size()); -// System.out.println("SDK Metric Name: " + point.getName()); -// System.out.println("SDK Metric Value: " + point.getValue()); -// System.out.println("SDK Metric attributes: " + metricsData.getProperties()); -// //System.out.println("SDK Metric Type: " + point.getType()); -// } -// -// -// } } boolean isEnabled() { @@ -491,12 +471,7 @@ public ArrayList processMetrics() { processedMetrics.add(processedMetric); } - //change to desired time limit for metrics - System.out.println("***************"); - System.out.println("METRIC: " + key + " LAST TIMESTAMP: " + value.getLastTimestamp() + " CURRENT TIME: " + LocalDateTime.now() + " DIFFERENCE: " + ChronoUnit.SECONDS.between(LocalDateTime.now(), value.getLastTimestamp())); if (ChronoUnit.SECONDS.between(value.getLastTimestamp(), LocalDateTime.now()) > 5) { - System.out.println("***************"); - System.out.println("REMOVING METRIC: " + key); iterator.remove(); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index a956be2909e56..2b32a6306b75e 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -72,16 +72,12 @@ public void run() { } } -// String etagValue = networkHelper.getEtagHeaderValue(response); -// ArrayList otelMetrics = quickPulseConfiguration.parseMetrics(response); -// quickPulseConfiguration.updateConfig(etagValue, otelMetrics); - - - } + /* Debugging purposes System.out.println("POST*********************"); System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); + */ } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index 50d064e2ae154..8926d3cc37ebc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -80,22 +80,6 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { String qpStatus = header.getValue(); if ("true".equalsIgnoreCase(qpStatus)) { status = QuickPulseStatus.QP_IS_ON; -// String headName = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getName(); -// String headValue = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); -// System.out.println("************************"); -// System.out.println(headName + ": " + headValue); -// System.out.println("************************"); -// System.out.println("Destructuring Body: "); -// String responseBody = response.getBodyAsString().block(); -// -// try { -// JsonNode rootNode = objectMapper.readTree(responseBody); -// System.out.println("Metrics :" + rootNode.get("Metrics")); -// } -// catch (Exception e) { -// System.out.println("Error: " + e); -// } - } else { status = QuickPulseStatus.QP_IS_OFF; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index edd41ca02d2e1..39b01ad5a16a2 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -81,9 +81,11 @@ class QuickPulsePingSender { } QuickPulseHeaderInfo ping(String redirectedEndpoint) { + /* Debugging purposes System.out.println("PING*********************"); System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); + */ String instrumentationKey = getInstrumentationKey(); if (Strings.isNullOrEmpty(instrumentationKey)) { // Quick Pulse Ping uri will be null when the instrumentation key is null. When that happens, diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index 1e5de33599079..d78b88ad612ba 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -89,7 +89,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "sample"); + headers.put("x-ms-qps-configuration-etag", "0::randometag::2::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); HttpPipeline httpPipeline = diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java index 526deeb3518a7..f788cb2ecf445 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java @@ -3,12 +3,17 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; import org.junit.jupiter.api.Test; +import org.mockito.Mock; import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class QuickPulseNetworkHelperTest { @Test @@ -28,4 +33,15 @@ void testIsSuccessWith500() { boolean result = new QuickPulseNetworkHelper().isSuccess(response); assertThat(result).isFalse(); } + + @Test + void testGetEtagHeader() { + HttpResponse response = mock(HttpResponse.class); + HttpHeaders headers = new HttpHeaders(); + HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER_NAME = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); + headers.add(QPS_CONFIGURATION_ETAG_HEADER_NAME, "0::randometag::2::"); + Mockito.doReturn(headers).when(response).getHeaders(); + String result = new QuickPulseNetworkHelper().getEtagHeaderValue(response); + assertThat(result).isEqualTo("0::randometag::2::"); + } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 54dd00f4e15b9..2610b3af8e01f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -6,16 +6,25 @@ import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; +import com.azure.json.implementation.jackson.core.JsonProcessingException; import com.azure.monitor.opentelemetry.exporter.implementation.MockHttpResponse; import com.azure.monitor.opentelemetry.exporter.implementation.NoopTracer; import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; @@ -42,6 +51,7 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc assertThat(endpointUrl) .isEqualTo( "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=testing-123"); + QuickPulseConfiguration.getInstance().reset(); } @Test @@ -68,6 +78,7 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE assertThat(endpointUrl) .isEqualTo( "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=A-test-instrumentation-key"); + QuickPulseConfiguration.getInstance().reset(); } @Test @@ -76,7 +87,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "sample"); + headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); HttpPipeline httpPipeline = @@ -99,5 +110,115 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + QuickPulseConfiguration.getInstance().reset(); } + + @Test + void successfulPingReturnsWithEtagHeader() { + System.out.println("Etag: " + QuickPulseConfiguration.getInstance().getEtag()); + Map headers = new HashMap<>(); + headers.put("x-ms-qps-service-polling-interval-hint", "1000"); + headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); + headers.put("x-ms-qps-subscribed", "true"); + headers.put("x-ms-qps-configuration-etag", "1::randometag::2::"); + HttpHeaders httpHeaders = new HttpHeaders(headers); + ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + HttpPipeline httpPipeline = + new HttpPipelineBuilder() + .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) + .tracer(new NoopTracer()) + .build(); + QuickPulsePingSender quickPulsePingSender = + new QuickPulsePingSender( + httpPipeline, + connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, + null, + "instance1", + "machine1", + "qpid123", + "testSdkVersion"); + QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); + assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); + assertThat("https://new.endpoint.com") + .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("1::randometag::2::"); + QuickPulseConfiguration.getInstance().reset(); + } + + @Test + void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { + Map headers = new HashMap<>(); + headers.put("x-ms-qps-service-polling-interval-hint", "1000"); + headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); + headers.put("x-ms-qps-subscribed", "true"); + headers.put("x-ms-qps-configuration-etag", "2::randometag::3::"); + HttpHeaders httpHeaders = new HttpHeaders(headers); + ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + + List> metrics = new ArrayList<>(); + Map metric1 = new HashMap<>(); + metric1.put("Id", "my_gauge"); + metric1.put("Aggregation", "Avg"); + metric1.put("TelemetryType", "Metric"); + metric1.put("Projection", "my_gauge"); + metric1.put("BackendAggregation", "Min"); + metric1.put("FilterGroups", new ArrayList<>()); + Map metric2 = new HashMap<>(); + metric2.put("Id", "MyFruitCounter"); + metric2.put("Aggregation", "Sum"); + metric2.put("TelemetryType", "Metric"); + metric2.put("Projection", "MyFruitCounter"); + metric2.put("BackendAggregation", "Max"); + metric2.put("FilterGroups", new ArrayList<>()); + metrics.add(metric1); + metrics.add(metric2); + + Map metricsMap = new HashMap<>(); + metricsMap.put("Metrics", metrics); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody; + try { + jsonBody = objectMapper.writeValueAsString(metricsMap); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + jsonBody = "{}"; + } + byte[] bodyBytes = jsonBody.getBytes(StandardCharsets.UTF_8); + + HttpPipeline httpPipeline = + new HttpPipelineBuilder() + .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders, bodyBytes))) + .tracer(new NoopTracer()) + .build(); + QuickPulsePingSender quickPulsePingSender = + new QuickPulsePingSender( + httpPipeline, + connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, + null, + "instance1", + "machine1", + "qpid123", + "testSdkVersion"); + + QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); + assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); + assertThat("https://new.endpoint.com") + .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("2::randometag::3::"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().size()).isEqualTo(2); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getAggregation()) + .isEqualTo("Avg"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getTelemetryType()) + .isEqualTo("Metric"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getProjection()).isEqualTo("my_gauge"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); + System.out.println(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()).isEqualTo("Sum"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getTelemetryType()).isEqualTo("Metric"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getProjection()).isEqualTo("MyFruitCounter"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getId()).isEqualTo("MyFruitCounter"); + QuickPulseConfiguration.getInstance().reset(); + } + } From f069e2d1f6ec2fc0debfac391fa8e2d6df1a82a3 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Fri, 12 Jul 2024 09:36:58 -0700 Subject: [PATCH 07/22] Resolved intermittent display issue in Live Metrics --- .../implementation/quickpulse/QuickPulseNetworkHelper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index 8926d3cc37ebc..d33bfe862d564 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -95,9 +95,10 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { return new QuickPulseHeaderInfo(status, serviceEndpointRedirect, servicePollingIntervalHint); } - String getEtagHeaderValue(HttpResponse response){ + String getEtagHeaderValue(HttpResponse response) { HttpHeaders headers = response.getHeaders(); - return headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME).getValue(); + HttpHeader etagHeader = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME); + return etagHeader != null ? etagHeader.getValue() : null; } } From b9058f2de03a77bfd9f8a49d45a2bb73e4633d66 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Mon, 15 Jul 2024 15:39:30 -0700 Subject: [PATCH 08/22] resolved reliance on breeze interval --- .../implementation/MetricDataMapper.java | 18 ++--- .../implementation/quickpulse/QuickPulse.java | 14 ++++ .../quickpulse/QuickPulseConfiguration.java | 27 ++++--- .../quickpulse/QuickPulseDataCollector.java | 2 +- .../quickpulse/QuickPulseDataSender.java | 1 + .../quickpulse/QuickPulseMetricReader.java | 41 +++++++++++ .../quickpulse/QuickPulseMetricReceiver.java | 73 +++++++++++++++++++ .../quickpulse/QuickPulsePingSender.java | 1 + .../QuickPulseNetworkHelperTest.java | 4 +- .../quickpulse/QuickPulsePingSenderTests.java | 18 +++-- 10 files changed, 171 insertions(+), 28 deletions(-) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java index ad959b571b893..625c5dd0bc51d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java @@ -72,20 +72,20 @@ public MetricDataMapper( this.captureHttpServer4xxAsError = captureHttpServer4xxAsError; } - public void mapMetrics(MetricData metricData, Consumer breezeConsumer, Consumer quickPulseConsumer) { + public void mapMetrics(MetricData metricData, Consumer quickPulseConsumer) { MetricDataType type = metricData.getType(); if (type == DOUBLE_SUM || type == DOUBLE_GAUGE || type == LONG_SUM || type == LONG_GAUGE || type == HISTOGRAM) { - boolean isPreAggregatedStandardMetric = - OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); - if (isPreAggregatedStandardMetric) { - List preAggregatedStandardMetrics = - convertOtelMetricToAzureMonitorMetric(metricData, true); - preAggregatedStandardMetrics.forEach(breezeConsumer::accept); - } +// boolean isPreAggregatedStandardMetric = +// OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); +// if (isPreAggregatedStandardMetric) { +// List preAggregatedStandardMetrics = +// convertOtelMetricToAzureMonitorMetric(metricData, true); +// preAggregatedStandardMetrics.forEach(breezeConsumer::accept); +// } // DO NOT emit unstable metrics from the OpenTelemetry auto instrumentation libraries // custom metrics are always emitted @@ -95,7 +95,7 @@ public void mapMetrics(MetricData metricData, Consumer breezeCons } List stableOtelMetrics = convertOtelMetricToAzureMonitorMetric(metricData, false); stableOtelMetrics.forEach(quickPulseConsumer::accept); - stableOtelMetrics.forEach(breezeConsumer::accept); + //stableOtelMetrics.forEach(breezeConsumer::accept); } else { logger.warning("metric data type {} is not supported yet.", metricData.getType()); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index d12f04ce22daf..f1aadf09ffadf 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -5,6 +5,7 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpRequest; +import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; @@ -37,6 +38,8 @@ public static QuickPulse create( @Nullable String roleName, @Nullable String roleInstance, boolean useNormalizedValueForNonNormalizedCpuPercentage, + QuickPulseMetricReader quickPulseMetricReader, + MetricDataMapper metricDataMapper, String sdkVersion) { QuickPulse quickPulse = new QuickPulse(); @@ -60,6 +63,8 @@ public static QuickPulse create( roleName, roleInstance, useNormalizedValueForNonNormalizedCpuPercentage, + quickPulseMetricReader, + metricDataMapper, sdkVersion); }); // the condition below will always be false, but by referencing the executor it ensures the @@ -101,6 +106,8 @@ private void initialize( @Nullable String roleName, @Nullable String roleInstance, boolean useNormalizedValueForNonNormalizedCpuPercentage, + QuickPulseMetricReader quickPulseMetricReader, + MetricDataMapper metricDataMapper, String sdkVersion) { String quickPulseId = UUID.randomUUID().toString().replace("-", ""); @@ -152,6 +159,13 @@ private void initialize( QuickPulseCoordinator coordinator = new QuickPulseCoordinator(coordinatorInitData); + QuickPulseMetricReceiver quickPulseMetricReceiver = new QuickPulseMetricReceiver(quickPulseMetricReader, metricDataMapper, collector); + + Thread metricReceiverThread = + new Thread(quickPulseMetricReceiver, QuickPulseMetricReceiver.class.getSimpleName()); + metricReceiverThread.setDaemon(true); + metricReceiverThread.start(); + Thread senderThread = new Thread(quickPulseDataSender, QuickPulseDataSender.class.getSimpleName()); senderThread.setDaemon(true); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index 3c9a00d295a55..63d581db96e5d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -15,10 +15,11 @@ public class QuickPulseConfiguration { private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); - private static QuickPulseConfiguration instance = new QuickPulseConfiguration(); + private static volatile QuickPulseConfiguration instance = new QuickPulseConfiguration(); private AtomicReference etag = new AtomicReference<>(); private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Object lock = new Object(); private QuickPulseConfiguration() { @@ -26,31 +27,35 @@ private QuickPulseConfiguration() { public static synchronized QuickPulseConfiguration getInstance() { if (instance == null) { - instance = new QuickPulseConfiguration(); + synchronized (lock) { + if (instance == null) { + instance = new QuickPulseConfiguration(); + } + } } return instance; } public synchronized String getEtag() { - return this.etag.get(); + return instance.etag.get(); } public synchronized void setEtag(String etag) { - this.etag.set(etag); + instance.etag.set(etag); } public synchronized ConcurrentHashMap getMetrics() { - return this.metrics; + return instance.metrics; } public synchronized void setMetrics(ConcurrentHashMap metrics) { - this.metrics = metrics; + instance.metrics = metrics; } public synchronized void updateConfig(String etagValue, ConcurrentHashMap otelMetrics) { - if (!Objects.equals(this.getEtag(), etagValue)){ - this.setEtag(etagValue); - this.setMetrics(otelMetrics); + if (!Objects.equals(instance.getEtag(), etagValue)){ + instance.setEtag(etagValue); + instance.setMetrics(otelMetrics); } } @@ -98,8 +103,8 @@ public ConcurrentHashMap parseMetrics(HttpResponse re } public synchronized void reset() { - this.setEtag(null); - this.metrics.clear(); + instance.setEtag(null); + instance.setMetrics(new ConcurrentHashMap()); } class OpenTelMetricInfo { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index f0a10af1522d5..9bb7cccb23bae 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -94,7 +94,7 @@ synchronized FinalCounters peek() { } - void addOtelMetric(TelemetryItem telemetryItem){ + synchronized void addOtelMetric(TelemetryItem telemetryItem){ if (!isEnabled()) { // quick pulse is not enabled or quick pulse data sender is not enabled return; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 2b32a6306b75e..fc69508ab108d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -54,6 +54,7 @@ public void run() { if (networkHelper.isSuccess(response)) { QuickPulseHeaderInfo quickPulseHeaderInfo = networkHelper.getQuickPulseHeaderInfo(response); + QuickPulseMetricReceiver.setQuickPulseHeaderInfo(quickPulseHeaderInfo); switch (quickPulseHeaderInfo.getQuickPulseStatus()) { case QP_IS_OFF: case QP_IS_ON: diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java new file mode 100644 index 0000000000000..b8131ef179644 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java @@ -0,0 +1,41 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.CollectionRegistration; +import io.opentelemetry.sdk.metrics.export.MetricReader; + +import java.util.Collection; + +public class QuickPulseMetricReader implements MetricReader { + + private volatile CollectionRegistration collectionRegistration = CollectionRegistration.noop(); + + @Override + public void register(CollectionRegistration registration) { + // this should get (once) called when the OpenTelemetry SDK is created + collectionRegistration = registration; + } + + @Override + public CompletableResultCode forceFlush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + // this will be called on-demand from Quick Pulse code + Collection collectAllMetrics() { + return collectionRegistration.collectAllMetrics(); + } + + @Override + public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) { + return AggregationTemporality.DELTA; + } +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java new file mode 100644 index 0000000000000..1e145002a6471 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -0,0 +1,73 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; + +import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; +import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger; +import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; +import io.opentelemetry.sdk.metrics.data.MetricData; + +import java.util.Collection; +import java.util.function.Consumer; + +import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR; + +public class QuickPulseMetricReceiver implements Runnable { + + private static QuickPulseHeaderInfo quickPulseHeaderInfo; + private QuickPulseMetricReader quickPulseMetricReader; + private QuickPulseDataCollector collector; + private static final OperationLogger metricReceiverLogger = + new OperationLogger(QuickPulseMetricReceiver.class, "Exporting metric"); + private final MetricDataMapper mapper; + private final Consumer quickPulseConsumer; + + public QuickPulseMetricReceiver(QuickPulseMetricReader quickPulseMetricReader, MetricDataMapper mapper, QuickPulseDataCollector collector) { + this.quickPulseMetricReader = quickPulseMetricReader; + this.mapper = mapper; + this.collector = collector; + this.quickPulseConsumer = + telemetryItem -> { + if (collector.isEnabled()) { + collector.addOtelMetric(telemetryItem); + } + }; + } + + public static synchronized QuickPulseHeaderInfo getQuickPulseHeaderInfo() { + return quickPulseHeaderInfo; + } + + public static synchronized void setQuickPulseHeaderInfo(QuickPulseHeaderInfo info) { + quickPulseHeaderInfo = info; + } + + @Override + public void run() { + while (true) { + + Collection metrics = quickPulseMetricReader.collectAllMetrics(); + + QuickPulseHeaderInfo headerInfo = getQuickPulseHeaderInfo(); + + if (headerInfo == null || headerInfo.getQuickPulseStatus() != QuickPulseStatus.QP_IS_ON) { + continue; + } + + for (MetricData metricData : metrics) { + try { + mapper.mapMetrics(metricData, quickPulseConsumer); + metricReceiverLogger.recordSuccess(); + } catch (Throwable t) { + metricReceiverLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR); + } + + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 39b01ad5a16a2..34cc77bb59aa6 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -117,6 +117,7 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { if (networkHelper.isSuccess(response)) { QuickPulseHeaderInfo quickPulseHeaderInfo = networkHelper.getQuickPulseHeaderInfo(response); + QuickPulseMetricReceiver.setQuickPulseHeaderInfo(quickPulseHeaderInfo); switch (quickPulseHeaderInfo.getQuickPulseStatus()) { case QP_IS_OFF: case QP_IS_ON: diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java index f788cb2ecf445..8330c3cb5ff88 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java @@ -39,9 +39,9 @@ void testGetEtagHeader() { HttpResponse response = mock(HttpResponse.class); HttpHeaders headers = new HttpHeaders(); HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER_NAME = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); - headers.add(QPS_CONFIGURATION_ETAG_HEADER_NAME, "0::randometag::2::"); + headers.add(QPS_CONFIGURATION_ETAG_HEADER_NAME, "0::randometag::1::"); Mockito.doReturn(headers).when(response).getHeaders(); String result = new QuickPulseNetworkHelper().getEtagHeaderValue(response); - assertThat(result).isEqualTo("0::randometag::2::"); + assertThat(result).isEqualTo("0::randometag::1::"); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 2610b3af8e01f..733d88f3b9cfa 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -32,6 +32,7 @@ class QuickPulsePingSenderTests { @Test void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxException { + QuickPulseConfiguration.getInstance().reset(); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender( @@ -56,6 +57,7 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc @Test void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxException { + QuickPulseConfiguration.getInstance().reset(); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=A-test-instrumentation-key"); QuickPulsePingSender quickPulsePingSender = @@ -83,6 +85,7 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE @Test void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { + QuickPulseConfiguration.getInstance().reset(); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); @@ -113,14 +116,17 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { QuickPulseConfiguration.getInstance().reset(); } + @Test void successfulPingReturnsWithEtagHeader() { - System.out.println("Etag: " + QuickPulseConfiguration.getInstance().getEtag()); + QuickPulseConfiguration configInstance = QuickPulseConfiguration.getInstance(); + configInstance.reset(); + System.out.println("QuickPulseConfiguration.getInstance().getEtag() = " + QuickPulseConfiguration.getInstance().getEtag()); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "1::randometag::2::"); + headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); HttpPipeline httpPipeline = @@ -142,17 +148,18 @@ void successfulPingReturnsWithEtagHeader() { assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("1::randometag::2::"); + assertThat(configInstance.getEtag()).isEqualTo("0::randometag::1::"); QuickPulseConfiguration.getInstance().reset(); } @Test void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { + QuickPulseConfiguration.getInstance().reset(); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "2::randometag::3::"); + headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); @@ -205,7 +212,7 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("2::randometag::3::"); + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); assertThat(QuickPulseConfiguration.getInstance().getMetrics().size()).isEqualTo(2); assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getAggregation()) .isEqualTo("Avg"); @@ -221,4 +228,5 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { QuickPulseConfiguration.getInstance().reset(); } + } From a1cf9cae983bd953e33d8064f85f061c933acdfa Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Tue, 16 Jul 2024 16:34:10 -0700 Subject: [PATCH 09/22] working version --- .../quickpulse/QuickPulseDataCollector.java | 21 +++++++- .../quickpulse/QuickPulseCoordinatorTest.java | 8 ++- .../QuickPulseDataCollectorTests.java | 54 +++++++++++++++++++ .../QuickPulseDataFetcherTests.java | 3 +- .../quickpulse/QuickPulsePingSenderTests.java | 9 +--- 5 files changed, 84 insertions(+), 11 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 9bb7cccb23bae..3fdf03d97d2c4 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -56,6 +56,7 @@ private static CpuPerformanceCounterCalculator getCpuPerformanceCounterCalculato synchronized void disable() { counters.set(null); + metricsStorage.clearMetrics(); quickPulseStatus = QuickPulseStatus.QP_IS_OFF; } @@ -341,6 +342,16 @@ public ArrayList retrieveOpenTelMetrics() { return metricsStorage.processMetrics(); } + // for testing purposes + public ConcurrentHashMap getOpenTelMetrics() { + return metricsStorage.getMetrics(); + } + + //for testing purposes + public void flushOpenTelMetrics() { + metricsStorage.clearMetrics(); + } + class FinalCounters { final int exceptions; @@ -443,7 +454,7 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { } class OpenTelMetricsStorage { - private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); public void addMetric(String metricName, double value) { OpenTelMetric metric = metrics.get(metricName); @@ -512,5 +523,13 @@ public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfigur } + public void clearMetrics() { + this.metrics = new ConcurrentHashMap(); + } + + //for testing + public ConcurrentHashMap getMetrics() { + return metrics; + } } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java index c5af8634e113e..79ef365c0b32c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java @@ -3,6 +3,7 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; +import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -53,16 +54,19 @@ void testOnlyPings() throws InterruptedException { Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); // make sure QP_IS_OFF after ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); + + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isNull(); + } @Test void testOnePingAndThenOnePost() throws InterruptedException { + QuickPulseDataFetcher mockFetcher = mock(QuickPulseDataFetcher.class); QuickPulseDataSender mockSender = mock(QuickPulseDataSender.class); Mockito.doReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)) .when(mockSender) .getQuickPulseHeaderInfo(); - QuickPulsePingSender mockPingSender = mock(QuickPulsePingSender.class); Mockito.when(mockPingSender.ping(null)) .thenReturn( @@ -99,6 +103,8 @@ void testOnePingAndThenOnePost() throws InterruptedException { Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); // Make sure QP_IS_OFF after one post and ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); + + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isNull(); } @Disabled("sporadically failing on CI") diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java index aa3d0b59c29cd..60878acb5db0a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java @@ -5,11 +5,22 @@ import com.azure.monitor.opentelemetry.exporter.implementation.builders.ExceptionTelemetryBuilder; import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorBase; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.OpenTelMetric; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseMetrics; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.resources.Resource; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; import static com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulseTestBase.createRemoteDependencyTelemetry; import static com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulseTestBase.createRequestTelemetry; @@ -33,6 +44,8 @@ void emptyCountsAndDurationsAfterEnable() { collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); QuickPulseDataCollector.FinalCounters counters = collector.peek(); assertCountersReset(counters); + ArrayList storedMetrics = collector.retrieveOpenTelMetrics(); + assertThat(storedMetrics).isEmpty(); } @Test @@ -42,6 +55,7 @@ void nullCountersAfterDisable() { collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); collector.disable(); assertThat(collector.peek()).isNull(); + assertThat(collector.retrieveOpenTelMetrics()).isEmpty(); } @Test @@ -152,6 +166,46 @@ void exceptionTelemetryIsCounted() { assertCountersReset(collector.peek()); } + @Test + void openTelemetryMetricsAreCounted() { + QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + + collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); + collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); + + TelemetryItem telemetry = new TelemetryItem(); + telemetry.setConnectionString(FAKE_CONNECTION_STRING); + MonitorBase data = new MonitorBase(); + MetricDataPoint point = new MetricDataPoint(); + point.setName("TestMetric"); + point.setValue(123.456); + ArrayList metricsList = new ArrayList<>(); + metricsList.add(point); + data.setBaseData(new MetricsData().setMetrics(metricsList)); + telemetry.setData(data); + Attributes attributes = Attributes.builder() + .put("telemetry.sdk.name", "opentelemetry") + .build(); + Resource resource = Resource.create(attributes); + telemetry.setResource(resource); + collector.addOtelMetric(telemetry); + ConcurrentHashMap storedMetrics = collector.getOpenTelMetrics(); + assertThat(storedMetrics.size()).isEqualTo(1); + assertThat(storedMetrics.containsKey("TestMetric")).isTrue(); + assertThat(storedMetrics.get("TestMetric").getDataValues().get(0)).isEqualTo(123.456); + + point.setName("TestMetric2"); + point.setValue(789.012); + collector.addOtelMetric(telemetry); + storedMetrics = collector.getOpenTelMetrics(); + assertThat(storedMetrics.size()).isEqualTo(2); + assertThat(storedMetrics.containsKey("TestMetric2")).isTrue(); + assertThat(storedMetrics.get("TestMetric2").getDataValues().get(0)).isEqualTo(789.012); + + collector.flushOpenTelMetrics(); + assertThat(collector.getOpenTelMetrics().size()).isEqualTo(0); + } + @Test void encodeDecodeIsIdentity() { long count = 456L; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index d78b88ad612ba..0b6dff0108a6f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -89,7 +89,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::2::"); + headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); HttpPipeline httpPipeline = @@ -112,5 +112,6 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 733d88f3b9cfa..74a87b167a4ff 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -32,7 +32,6 @@ class QuickPulsePingSenderTests { @Test void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxException { - QuickPulseConfiguration.getInstance().reset(); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender( @@ -57,7 +56,6 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc @Test void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxException { - QuickPulseConfiguration.getInstance().reset(); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=A-test-instrumentation-key"); QuickPulsePingSender quickPulsePingSender = @@ -85,7 +83,6 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE @Test void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { - QuickPulseConfiguration.getInstance().reset(); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); @@ -113,14 +110,11 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - QuickPulseConfiguration.getInstance().reset(); } @Test void successfulPingReturnsWithEtagHeader() { - QuickPulseConfiguration configInstance = QuickPulseConfiguration.getInstance(); - configInstance.reset(); System.out.println("QuickPulseConfiguration.getInstance().getEtag() = " + QuickPulseConfiguration.getInstance().getEtag()); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); @@ -148,13 +142,12 @@ void successfulPingReturnsWithEtagHeader() { assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(configInstance.getEtag()).isEqualTo("0::randometag::1::"); + assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); QuickPulseConfiguration.getInstance().reset(); } @Test void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { - QuickPulseConfiguration.getInstance().reset(); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); From 925b4576cd9eef43d19978d4dfe85e0054affc9b Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 17 Jul 2024 18:05:53 -0700 Subject: [PATCH 10/22] refactoring QuickPulseConfiguration --- .../implementation/quickpulse/QuickPulse.java | 16 +++-- .../quickpulse/QuickPulseConfiguration.java | 24 +++---- .../quickpulse/QuickPulseDataCollector.java | 6 +- .../quickpulse/QuickPulseDataFetcher.java | 6 +- .../quickpulse/QuickPulseDataSender.java | 9 +-- .../quickpulse/QuickPulseMetricReceiver.java | 4 +- .../quickpulse/QuickPulseNetworkHelper.java | 2 - .../quickpulse/QuickPulsePingSender.java | 13 ++-- .../quickpulse/QuickPulseCoordinatorTest.java | 15 +++-- .../QuickPulseDataCollectorTests.java | 18 +++--- .../QuickPulseDataFetcherTests.java | 24 ++++--- .../QuickPulseIntegrationTests.java | 38 +++++++----- .../QuickPulseNetworkHelperTest.java | 3 - .../quickpulse/QuickPulsePingSenderTests.java | 62 +++++++++---------- 14 files changed, 123 insertions(+), 117 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index f1aadf09ffadf..e6b35032d0711 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -6,18 +6,13 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpRequest; import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; import com.azure.monitor.opentelemetry.exporter.implementation.utils.HostName; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; import com.azure.monitor.opentelemetry.exporter.implementation.utils.ThreadPoolUtils; -import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; import java.net.URL; -import java.util.Objects; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; @@ -112,8 +107,9 @@ private void initialize( String quickPulseId = UUID.randomUUID().toString().replace("-", ""); ArrayBlockingQueue sendQueue = new ArrayBlockingQueue<>(256, true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); - QuickPulseDataSender quickPulseDataSender = new QuickPulseDataSender(httpPipeline, sendQueue); + QuickPulseDataSender quickPulseDataSender = new QuickPulseDataSender(httpPipeline, sendQueue, quickPulseConfiguration); String instanceName = roleInstance; String machineName = HostName.get(); @@ -126,7 +122,7 @@ private void initialize( } QuickPulseDataCollector collector = - new QuickPulseDataCollector(useNormalizedValueForNonNormalizedCpuPercentage); + new QuickPulseDataCollector(useNormalizedValueForNonNormalizedCpuPercentage, quickPulseConfiguration); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender( @@ -137,7 +133,8 @@ private void initialize( instanceName, machineName, quickPulseId, - sdkVersion); + sdkVersion, + quickPulseConfiguration); QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( collector, @@ -147,7 +144,8 @@ private void initialize( roleName, instanceName, machineName, - quickPulseId); + quickPulseId, + quickPulseConfiguration); QuickPulseCoordinatorInitData coordinatorInitData = new QuickPulseCoordinatorInitDataBuilder() diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index 63d581db96e5d..a5e464e4a1b2a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -15,13 +15,13 @@ public class QuickPulseConfiguration { private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); - private static volatile QuickPulseConfiguration instance = new QuickPulseConfiguration(); +// private static volatile QuickPulseConfiguration instance = new QuickPulseConfiguration(); private AtomicReference etag = new AtomicReference<>(); private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); private static final ObjectMapper objectMapper = new ObjectMapper(); private static final Object lock = new Object(); - +/* private QuickPulseConfiguration() { } @@ -36,26 +36,28 @@ public static synchronized QuickPulseConfiguration getInstance() { return instance; } + */ + public synchronized String getEtag() { - return instance.etag.get(); + return this.etag.get(); } public synchronized void setEtag(String etag) { - instance.etag.set(etag); + this.etag.set(etag); } public synchronized ConcurrentHashMap getMetrics() { - return instance.metrics; + return this.metrics; } public synchronized void setMetrics(ConcurrentHashMap metrics) { - instance.metrics = metrics; + this.metrics = metrics; } public synchronized void updateConfig(String etagValue, ConcurrentHashMap otelMetrics) { - if (!Objects.equals(instance.getEtag(), etagValue)){ - instance.setEtag(etagValue); - instance.setMetrics(otelMetrics); + if (!Objects.equals(this.getEtag(), etagValue)){ + this.setEtag(etagValue); + this.setMetrics(otelMetrics); } } @@ -103,8 +105,8 @@ public ConcurrentHashMap parseMetrics(HttpResponse re } public synchronized void reset() { - instance.setEtag(null); - instance.setMetrics(new ConcurrentHashMap()); + this.setEtag(null); + this.setMetrics(new ConcurrentHashMap()); } class OpenTelMetricInfo { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 3fdf03d97d2c4..1ef937de26277 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -33,7 +33,8 @@ final class QuickPulseDataCollector { private final AtomicReference counters = new AtomicReference<>(null); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); +// private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); @@ -45,9 +46,10 @@ final class QuickPulseDataCollector { private volatile Supplier instrumentationKeySupplier; - QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage) { + QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseConfiguration quickPulseConfiguration) { this.useNormalizedValueForNonNormalizedCpuPercentage = useNormalizedValueForNonNormalizedCpuPercentage; + this.quickPulseConfiguration = quickPulseConfiguration; } private static CpuPerformanceCounterCalculator getCpuPerformanceCounterCalculator() { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index 54865c594fc4c..a0e2785836f83 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -39,7 +39,7 @@ class QuickPulseDataFetcher { private final ArrayBlockingQueue sendQueue; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private final Supplier endpointUrl; private final Supplier instrumentationKey; @@ -58,7 +58,8 @@ public QuickPulseDataFetcher( String roleName, String instanceName, String machineName, - String quickPulseId) { + String quickPulseId, + QuickPulseConfiguration quickPulseConfiguration) { this.collector = collector; this.sendQueue = sendQueue; this.endpointUrl = endpointUrl; @@ -67,6 +68,7 @@ public QuickPulseDataFetcher( this.instanceName = instanceName; this.machineName = machineName; this.quickPulseId = quickPulseId; + this.quickPulseConfiguration = quickPulseConfiguration; sdkVersion = getCurrentSdkVersion(); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 92d2d6adc70c6..968a81fddee70 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -7,10 +7,6 @@ import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.util.logging.ClientLogger; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.HashSet; import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -20,16 +16,17 @@ class QuickPulseDataSender implements Runnable { private static final ClientLogger logger = new ClientLogger(QuickPulseCoordinator.class); private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private final HttpPipeline httpPipeline; private volatile QuickPulseHeaderInfo quickPulseHeaderInfo; private long lastValidTransmission = 0; private final ArrayBlockingQueue sendQueue; - QuickPulseDataSender(HttpPipeline httpPipeline, ArrayBlockingQueue sendQueue) { + QuickPulseDataSender(HttpPipeline httpPipeline, ArrayBlockingQueue sendQueue, QuickPulseConfiguration quickPulseConfiguration) { this.httpPipeline = httpPipeline; this.sendQueue = sendQueue; + this.quickPulseConfiguration = quickPulseConfiguration; } @Override diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java index 1e145002a6471..b4ca8e283e9d1 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -26,8 +26,8 @@ public QuickPulseMetricReceiver(QuickPulseMetricReader quickPulseMetricReader, M this.collector = collector; this.quickPulseConsumer = telemetryItem -> { - if (collector.isEnabled()) { - collector.addOtelMetric(telemetryItem); + if (this.collector.isEnabled()) { + this.collector.addOtelMetric(telemetryItem); } }; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index d33bfe862d564..f917b06f93515 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -10,8 +10,6 @@ import com.azure.core.http.HttpResponse; import com.azure.core.http.HttpHeaderName; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 34cc77bb59aa6..33290801b2f04 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -18,9 +18,7 @@ import reactor.util.annotation.Nullable; import java.net.URL; -import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -49,7 +47,7 @@ class QuickPulsePingSender { private final HttpPipeline httpPipeline; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private volatile QuickPulseEnvelope pingEnvelope; // cached for performance private final Supplier endpointUrl; @@ -69,7 +67,8 @@ class QuickPulsePingSender { String instanceName, String machineName, String quickPulseId, - String sdkVersion) { + String sdkVersion, + QuickPulseConfiguration quickPulseConfiguration) { this.httpPipeline = httpPipeline; this.endpointUrl = endpointUrl; this.instrumentationKey = instrumentationKey; @@ -78,14 +77,10 @@ class QuickPulsePingSender { this.machineName = machineName; this.quickPulseId = quickPulseId; this.sdkVersion = sdkVersion; + this.quickPulseConfiguration = quickPulseConfiguration; } QuickPulseHeaderInfo ping(String redirectedEndpoint) { - /* Debugging purposes - System.out.println("PING*********************"); - System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); - System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); - */ String instrumentationKey = getInstrumentationKey(); if (Strings.isNullOrEmpty(instrumentationKey)) { // Quick Pulse Ping uri will be null when the instrumentation key is null. When that happens, diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java index 79ef365c0b32c..42302b655c2bb 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java @@ -3,7 +3,6 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; -import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -20,7 +19,8 @@ void testOnlyPings() throws InterruptedException { QuickPulseDataFetcher mockFetcher = mock(QuickPulseDataFetcher.class); QuickPulseDataSender mockSender = mock(QuickPulseDataSender.class); QuickPulsePingSender mockPingSender = mock(QuickPulsePingSender.class); - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); Mockito.doReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)) .when(mockPingSender) .ping(null); @@ -55,7 +55,7 @@ void testOnlyPings() throws InterruptedException { // make sure QP_IS_OFF after ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isNull(); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } @@ -73,7 +73,8 @@ void testOnePingAndThenOnePost() throws InterruptedException { new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_ON), new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)); - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder() .withDataFetcher(mockFetcher) @@ -104,7 +105,7 @@ void testOnePingAndThenOnePost() throws InterruptedException { // Make sure QP_IS_OFF after one post and ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isNull(); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } @Disabled("sporadically failing on CI") @@ -113,6 +114,7 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { QuickPulseDataFetcher mockFetcher = Mockito.mock(QuickPulseDataFetcher.class); QuickPulseDataSender mockSender = Mockito.mock(QuickPulseDataSender.class); QuickPulsePingSender mockPingSender = Mockito.mock(QuickPulsePingSender.class); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); Mockito.doNothing().when(mockFetcher).prepareQuickPulseDataForSend(notNull()); Mockito.doReturn( @@ -129,7 +131,7 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { .withDataFetcher(mockFetcher) .withDataSender(mockSender) .withPingSender(mockPingSender) - .withCollector(new QuickPulseDataCollector(true)) + .withCollector(new QuickPulseDataCollector(true, quickPulseConfiguration)) .withWaitBetweenPingsInMillis(10L) .withWaitBetweenPostsInMillis(10L) .withWaitOnErrorInMillis(10L) @@ -149,5 +151,6 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { .prepareQuickPulseDataForSend("https://new.endpoint.com"); Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); Mockito.verify(mockPingSender, Mockito.times(2)).ping("https://new.endpoint.com"); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java index 60878acb5db0a..73e2c9ed27177 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java @@ -11,7 +11,6 @@ import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.OpenTelMetric; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseMetrics; -import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.sdk.resources.Resource; import org.junit.jupiter.api.Test; @@ -19,7 +18,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; import static com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulseTestBase.createRemoteDependencyTelemetry; @@ -34,12 +32,12 @@ class QuickPulseDataCollectorTests { @Test void initialStateIsDisabled() { - assertThat(new QuickPulseDataCollector(true).peek()).isNull(); + assertThat(new QuickPulseDataCollector(true, new QuickPulseConfiguration()).peek()).isNull(); } @Test void emptyCountsAndDurationsAfterEnable() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); QuickPulseDataCollector.FinalCounters counters = collector.peek(); @@ -50,7 +48,7 @@ void emptyCountsAndDurationsAfterEnable() { @Test void nullCountersAfterDisable() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); collector.disable(); @@ -60,7 +58,7 @@ void nullCountersAfterDisable() { @Test void requestTelemetryIsCounted_DurationIsSum() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -103,7 +101,7 @@ void requestTelemetryIsCounted_DurationIsSum() { @Test void dependencyTelemetryIsCounted_DurationIsSum() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -146,7 +144,7 @@ void dependencyTelemetryIsCounted_DurationIsSum() { @Test void exceptionTelemetryIsCounted() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -168,7 +166,7 @@ void exceptionTelemetryIsCounted() { @Test void openTelemetryMetricsAreCounted() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -284,7 +282,7 @@ private static void assertCountersReset(QuickPulseDataCollector.FinalCounters co @Test void checkDocumentsListSize() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index 0b6dff0108a6f..c265d3ddc3e47 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -24,16 +24,18 @@ class QuickPulseDataFetcherTests { @Test void testGetCurrentSdkVersion() { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher( - new QuickPulseDataCollector(true), + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, - null); + null, + quickPulseConfiguration); String sdkVersion = dataFetcher.getCurrentSdkVersion(); assertThat(sdkVersion).isNotNull(); assertThat(sdkVersion).isNotEqualTo("java:unknown"); @@ -42,16 +44,18 @@ void testGetCurrentSdkVersion() { @Test void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( - new QuickPulseDataCollector(true), + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, - null); + null, + quickPulseConfiguration); String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint(); String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -64,16 +68,18 @@ void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException { @Test void endpointIsFormattedCorrectlyWhenConfigIsNull() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( - new QuickPulseDataCollector(true), + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, - null); + null, + quickPulseConfiguration); String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint(); String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -92,6 +98,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); HttpPipeline httpPipeline = new HttpPipelineBuilder() .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) @@ -106,12 +113,13 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("0::randometag::1::"); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java index b4536e2ec720c..13eb62ff44ce1 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java @@ -26,7 +26,8 @@ public class QuickPulseIntegrationTests extends QuickPulseTestBase { ConnectionString.parse("InstrumentationKey=ikey123"); private static final String instrumentationKey = "ikey123"; - private QuickPulsePingSender getQuickPulsePingSender() { + + private QuickPulsePingSender getQuickPulsePingSender(QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender( getHttpPipeline(), connectionString::getLiveEndpoint, @@ -35,10 +36,11 @@ private QuickPulsePingSender getQuickPulsePingSender() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); } - private QuickPulsePingSender getQuickPulsePingSenderWithAuthentication() { + private QuickPulsePingSender getQuickPulsePingSenderWithAuthentication(QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender( getHttpPipelineWithAuthentication(), connectionString::getLiveEndpoint, @@ -47,10 +49,11 @@ private QuickPulsePingSender getQuickPulsePingSenderWithAuthentication() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); } - private QuickPulsePingSender getQuickPulsePingSenderWithValidator(HttpPipelinePolicy validator) { + private QuickPulsePingSender getQuickPulsePingSenderWithValidator(HttpPipelinePolicy validator, QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender( getHttpPipeline(validator), connectionString::getLiveEndpoint, @@ -59,13 +62,14 @@ private QuickPulsePingSender getQuickPulsePingSenderWithValidator(HttpPipelinePo "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); } @Disabled @Test public void testPing() { - QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSender(); + QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSender(new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); } @@ -73,7 +77,7 @@ public void testPing() { @Disabled @Test public void testPingWithAuthentication() { - QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithAuthentication(); + QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithAuthentication(new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); } @@ -86,7 +90,7 @@ public void testPingRequestBody() throws InterruptedException { "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithValidator( - new ValidationPolicy(pingCountDown, expectedRequestBody)); + new ValidationPolicy(pingCountDown, expectedRequestBody), new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); assertTrue(pingCountDown.await(60, TimeUnit.SECONDS)); @@ -99,21 +103,24 @@ public void testPostRequest() throws InterruptedException { CountDownLatch pingCountDown = new CountDownLatch(1); CountDownLatch postCountDown = new CountDownLatch(1); Date currDate = new Date(); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); String expectedPingRequestBody = "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; String expectedPostRequestBody = "\\[\\{\"Documents\":\\[\\{\"__type\":\"RequestTelemetryDocument\",\"DocumentType\":\"Request\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"request-test\",\"Success\":true,\"Duration\":\"PT.*S\",\"ResponseCode\":\"200\",\"OperationName\":null,\"Url\":\"foo\"\\},\\{\"__type\":\"DependencyTelemetryDocument\",\"DocumentType\":\"RemoteDependency\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"dep-test\",\"Target\":null,\"Success\":true,\"Duration\":\"PT.*S\",\"ResultCode\":null,\"CommandName\":\"dep-test-cmd\",\"DependencyTypeName\":null,\"OperationName\":null\\},\\{\"__type\":\"ExceptionTelemetryDocument\",\"DocumentType\":\"Exception\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":null,\"Exception\":\"\",\"ExceptionMessage\":\"test\",\"ExceptionType\":\"java.lang.Exception\"\\}\\],\"InstrumentationKey\":\"" + instrumentationKey + "\",\"Metrics\":\\[\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Request Duration\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests Failed\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests Succeeded\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Call Duration\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls Failed\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls Succeeded\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Exceptions\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\Memory\\\\\\\\Committed Bytes\",\"Value\":[0-9.E]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\Processor\\(_Total\\)\\\\\\\\% Processor Time\",\"Value\":-?[0-9.]+,\"Weight\":\\d+\\}\\],\"InvariantVersion\":1,\"Timestamp\":\"\\\\\\/Date\\(\\d+\\)\\\\\\/\",\"Version\":\"[^\"]*\",\"StreamId\":null,\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}\\]"; - QuickPulsePingSender pingSender = - getQuickPulsePingSenderWithValidator( - new ValidationPolicy(pingCountDown, expectedPingRequestBody)); + QuickPulsePingSender pingSender = + getQuickPulsePingSenderWithValidator( + new ValidationPolicy(pingCountDown, expectedPingRequestBody), + new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = pingSender.ping(null); QuickPulseDataSender dataSender = new QuickPulseDataSender( getHttpPipeline(new ValidationPolicy(postCountDown, expectedPostRequestBody)), - sendQueue); - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + sendQueue, + quickPulseConfiguration); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher( collector, @@ -123,7 +130,8 @@ public void testPostRequest() throws InterruptedException { null, "instance1", "machine1", - null); + null, + quickPulseConfiguration); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(connectionString::getInstrumentationKey); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java index 8330c3cb5ff88..ae5d6e293d2fe 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java @@ -3,17 +3,14 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; -import com.azure.core.http.HttpHeader; import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; import org.junit.jupiter.api.Test; -import org.mockito.Mock; import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; class QuickPulseNetworkHelperTest { @Test diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 74a87b167a4ff..80f21031de678 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -6,16 +6,13 @@ import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; -import com.azure.json.implementation.jackson.core.JsonProcessingException; import com.azure.monitor.opentelemetry.exporter.implementation.MockHttpResponse; import com.azure.monitor.opentelemetry.exporter.implementation.NoopTracer; import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; -import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -23,8 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; @@ -33,6 +28,7 @@ class QuickPulsePingSenderTests { @Test void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender( null, @@ -42,7 +38,8 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc null, null, null, - null); + null, + quickPulseConfiguration); String quickPulseEndpoint = quickPulsePingSender.getQuickPulseEndpoint(); String endpointUrl = quickPulsePingSender.getQuickPulsePingUri(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -51,13 +48,13 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc assertThat(endpointUrl) .isEqualTo( "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=testing-123"); - QuickPulseConfiguration.getInstance().reset(); } @Test void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=A-test-instrumentation-key"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender( null, @@ -67,7 +64,8 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE null, null, null, - null); + null, + quickPulseConfiguration); String quickPulseEndpoint = quickPulsePingSender.getQuickPulseEndpoint(); String endpointUrl = quickPulsePingSender.getQuickPulsePingUri(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -78,7 +76,6 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE assertThat(endpointUrl) .isEqualTo( "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=A-test-instrumentation-key"); - QuickPulseConfiguration.getInstance().reset(); } @Test @@ -87,9 +84,10 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); + headers.put("x-ms-qps-configuration-etag", "1::randometag::2::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); HttpPipeline httpPipeline = new HttpPipelineBuilder() .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) @@ -104,25 +102,27 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("1::randometag::2::"); } @Test void successfulPingReturnsWithEtagHeader() { - System.out.println("QuickPulseConfiguration.getInstance().getEtag() = " + QuickPulseConfiguration.getInstance().getEtag()); Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); + headers.put("x-ms-qps-configuration-etag", "2::randometag::3::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); HttpPipeline httpPipeline = new HttpPipelineBuilder() .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) @@ -137,13 +137,13 @@ void successfulPingReturnsWithEtagHeader() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); - QuickPulseConfiguration.getInstance().reset(); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("2::randometag::3::"); } @Test @@ -152,9 +152,10 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); + headers.put("x-ms-qps-configuration-etag", "3::randometag::4::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); List> metrics = new ArrayList<>(); Map metric1 = new HashMap<>(); @@ -199,27 +200,24 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { "instance1", "machine1", "qpid123", - "testSdkVersion"); + "testSdkVersion", + quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("0::randometag::1::"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().size()).isEqualTo(2); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getAggregation()) + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("3::randometag::4::"); + assertThat(quickPulseConfiguration.getMetrics().size()).isEqualTo(2); + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getAggregation()) .isEqualTo("Avg"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getTelemetryType()) + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getTelemetryType()) .isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getProjection()).isEqualTo("my_gauge"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); - System.out.println(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()).isEqualTo("Sum"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getTelemetryType()).isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getProjection()).isEqualTo("MyFruitCounter"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getId()).isEqualTo("MyFruitCounter"); - QuickPulseConfiguration.getInstance().reset(); + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getProjection()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getAggregation()).isEqualTo("Sum"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getTelemetryType()).isEqualTo("Metric"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getProjection()).isEqualTo("MyFruitCounter"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getId()).isEqualTo("MyFruitCounter"); } - - } From 258b89cb86f806d3bd02ba1b7be333db14a9e033 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Fri, 19 Jul 2024 15:20:12 -0700 Subject: [PATCH 11/22] fixed same metric on multiple dashboards bug --- .../implementation/MetricDataMapper.java | 12 +---- .../implementation/quickpulse/QuickPulse.java | 17 ++----- .../quickpulse/QuickPulseConfiguration.java | 44 +++++-------------- .../quickpulse/QuickPulseCoordinator.java | 1 + .../quickpulse/QuickPulseDataCollector.java | 29 +++++++----- .../quickpulse/QuickPulseDataSender.java | 10 ++--- .../quickpulse/QuickPulseMetricReceiver.java | 3 -- .../quickpulse/QuickPulsePingSender.java | 8 ++-- .../quickpulse/model/OpenTelDataPoint.java | 9 ---- .../quickpulse/QuickPulsePingSenderTests.java | 16 +++---- 10 files changed, 49 insertions(+), 100 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java index 625c5dd0bc51d..659510568cec7 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java @@ -72,20 +72,13 @@ public MetricDataMapper( this.captureHttpServer4xxAsError = captureHttpServer4xxAsError; } - public void mapMetrics(MetricData metricData, Consumer quickPulseConsumer) { + public void mapMetrics(MetricData metricData, Consumer consumer) { MetricDataType type = metricData.getType(); if (type == DOUBLE_SUM || type == DOUBLE_GAUGE || type == LONG_SUM || type == LONG_GAUGE || type == HISTOGRAM) { -// boolean isPreAggregatedStandardMetric = -// OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); -// if (isPreAggregatedStandardMetric) { -// List preAggregatedStandardMetrics = -// convertOtelMetricToAzureMonitorMetric(metricData, true); -// preAggregatedStandardMetrics.forEach(breezeConsumer::accept); -// } // DO NOT emit unstable metrics from the OpenTelemetry auto instrumentation libraries // custom metrics are always emitted @@ -94,8 +87,7 @@ public void mapMetrics(MetricData metricData, Consumer quickPulse return; } List stableOtelMetrics = convertOtelMetricToAzureMonitorMetric(metricData, false); - stableOtelMetrics.forEach(quickPulseConsumer::accept); - //stableOtelMetrics.forEach(breezeConsumer::accept); + stableOtelMetrics.forEach(consumer::accept); } else { logger.warning("metric data type {} is not supported yet.", metricData.getType()); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index e6b35032d0711..169ea5178a49c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -21,8 +21,9 @@ public class QuickPulse { - //change this back to 1 - static final int QP_INVARIANT_VERSION = 5; + //supports Open Tel metrics but not live filtering + // change to 7 once live filtering is also supported + static final int QP_INVARIANT_VERSION = 6; private volatile QuickPulseDataCollector collector; @@ -83,17 +84,6 @@ public void add(TelemetryItem telemetryItem) { } } - public void add(TelemetryItem telemetryItem, Boolean isOtelMetric) { - if (collector != null){ - if(isOtelMetric){ - collector.addOtelMetric(telemetryItem); - - }else{ - collector.add(telemetryItem); - } - } - } - private void initialize( HttpPipeline httpPipeline, Supplier endpointUrl, @@ -135,6 +125,7 @@ private void initialize( quickPulseId, sdkVersion, quickPulseConfiguration); + QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( collector, diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index a5e464e4a1b2a..aa5c569e60263 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -15,28 +13,9 @@ public class QuickPulseConfiguration { private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); -// private static volatile QuickPulseConfiguration instance = new QuickPulseConfiguration(); private AtomicReference etag = new AtomicReference<>(); - private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + private ConcurrentHashMap> metrics = new ConcurrentHashMap<>(); private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final Object lock = new Object(); - -/* - private QuickPulseConfiguration() { - } - - public static synchronized QuickPulseConfiguration getInstance() { - if (instance == null) { - synchronized (lock) { - if (instance == null) { - instance = new QuickPulseConfiguration(); - } - } - } - return instance; - } - - */ public synchronized String getEtag() { return this.etag.get(); @@ -46,15 +25,15 @@ public synchronized void setEtag(String etag) { this.etag.set(etag); } - public synchronized ConcurrentHashMap getMetrics() { + public synchronized ConcurrentHashMap> getMetrics() { return this.metrics; } - public synchronized void setMetrics(ConcurrentHashMap metrics) { + public synchronized void setMetrics(ConcurrentHashMap> metrics) { this.metrics = metrics; } - public synchronized void updateConfig(String etagValue, ConcurrentHashMap otelMetrics) { + public synchronized void updateConfig(String etagValue, ConcurrentHashMap> otelMetrics) { if (!Objects.equals(this.getEtag(), etagValue)){ this.setEtag(etagValue); this.setMetrics(otelMetrics); @@ -62,18 +41,16 @@ public synchronized void updateConfig(String etagValue, ConcurrentHashMap parseMetrics(HttpResponse response) { + public ConcurrentHashMap> parseMetrics(HttpResponse response) { - HashSet metricsSet = new HashSet<>(); - ConcurrentHashMap requestedMetrics = new ConcurrentHashMap<>(); + ConcurrentHashMap> requestedMetrics = new ConcurrentHashMap<>(); try { String responseBody = response.getBodyAsString().block(); if (responseBody == null || responseBody.isEmpty()) { - return new ConcurrentHashMap(); + return new ConcurrentHashMap>(); } JsonNode rootNode = objectMapper.readTree(responseBody); - //System.out.println("Metrics :" + rootNode.get("Metrics")); Debugging purposes JsonNode metricsNode = rootNode.get("Metrics"); if (metricsNode instanceof ArrayNode) { @@ -92,21 +69,20 @@ public ConcurrentHashMap parseMetrics(HttpResponse re } } metric.setProjection(projection); - requestedMetrics.put(projection, metric); - + requestedMetrics.computeIfAbsent(metric.getProjection(), k -> new ArrayList<>()).add(metric); } } return requestedMetrics; } catch (Exception e) { logger.verbose("Failed to parse metrics from response: %s", e.getMessage()); } - return new ConcurrentHashMap(); + return new ConcurrentHashMap>(); } public synchronized void reset() { this.setEtag(null); - this.setMetrics(new ConcurrentHashMap()); + this.setMetrics(new ConcurrentHashMap>()); } class OpenTelMetricInfo { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java index d65cebdde18f9..a54e08a71994f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java @@ -78,6 +78,7 @@ private long sendData() { case QP_IS_OFF: pingMode = true; + collector.flushOpenTelMetrics(); return qpsServicePollingIntervalHintMillis > 0 ? qpsServicePollingIntervalHintMillis : waitBetweenPingsInMillis; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 1ef937de26277..828f13dfaf67c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -33,7 +33,6 @@ final class QuickPulseDataCollector { private final AtomicReference counters = new AtomicReference<>(null); -// private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); private QuickPulseConfiguration quickPulseConfiguration; private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); @@ -344,12 +343,10 @@ public ArrayList retrieveOpenTelMetrics() { return metricsStorage.processMetrics(); } - // for testing purposes public ConcurrentHashMap getOpenTelMetrics() { return metricsStorage.getMetrics(); } - //for testing purposes public void flushOpenTelMetrics() { metricsStorage.clearMetrics(); } @@ -457,20 +454,28 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { class OpenTelMetricsStorage { private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + // setting most amount of openTel metrics a user can stream to this. will review if users surpass threshold. + private final int maxMetricsLimit = 50; + // setting buffer time for receiving data for a metric before declaring it inactive. (in minutes) + private final double metricBufferTime = 5; public void addMetric(String metricName, double value) { OpenTelMetric metric = metrics.get(metricName); if (metric == null) { - metric = new OpenTelMetric(metricName); - metric.addDataPoint(value); - metrics.putIfAbsent(metricName, metric); + if (metrics.size() <= maxMetricsLimit) { + // create new metric and add to metrics map + // (if we have reached the limit, we will not create a new metric and add it to the map + metric = new OpenTelMetric(metricName); + metric.addDataPoint(value); + metrics.putIfAbsent(metricName, metric); + } } else { metric.addDataPoint(value); } } public ArrayList processMetrics() { - ConcurrentHashMap requestedMetrics = quickPulseConfiguration.getMetrics(); + ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getMetrics(); ArrayList processedMetrics = new ArrayList<>(); Iterator> iterator = this.metrics.entrySet().iterator(); @@ -480,11 +485,14 @@ public ArrayList processMetrics() { OpenTelMetric value = entry.getValue(); if (requestedMetrics.containsKey(key)) { - QuickPulseMetrics processedMetric = processMetric(value, requestedMetrics.get(key)); - processedMetrics.add(processedMetric); + for (QuickPulseConfiguration.OpenTelMetricInfo metricInfo : requestedMetrics.get(key)) { + QuickPulseMetrics processedMetric = processMetric(value, metricInfo); + processedMetrics.add(processedMetric); + } + } - if (ChronoUnit.SECONDS.between(value.getLastTimestamp(), LocalDateTime.now()) > 5) { + if (ChronoUnit.MINUTES.between(value.getLastTimestamp(), LocalDateTime.now()) > metricBufferTime) { iterator.remove(); } @@ -522,7 +530,6 @@ public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfigur default: throw new IllegalArgumentException("Aggregation type not supported: " + aggregation); } - } public void clearMetrics() { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 968a81fddee70..076a235dd8c81 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -7,6 +7,8 @@ import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.util.logging.ClientLogger; + +import java.util.ArrayList; import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -63,7 +65,7 @@ public void run() { this.quickPulseHeaderInfo = quickPulseHeaderInfo; String etagValue = networkHelper.getEtagHeaderValue(response); if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } break; @@ -73,15 +75,9 @@ public void run() { break; } } - } catch (Throwable t) { logger.error("QuickPulseDataSender failed to send a request", t); } - /* Debugging purposes - System.out.println("POST*********************"); - System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); - System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); - */ } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java index b4ca8e283e9d1..4801a987a0635 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -45,7 +45,6 @@ public void run() { while (true) { Collection metrics = quickPulseMetricReader.collectAllMetrics(); - QuickPulseHeaderInfo headerInfo = getQuickPulseHeaderInfo(); if (headerInfo == null || headerInfo.getQuickPulseStatus() != QuickPulseStatus.QP_IS_ON) { @@ -59,9 +58,7 @@ public void run() { } catch (Throwable t) { metricReceiverLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR); } - } - try { Thread.sleep(1000); } catch (InterruptedException e) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 33290801b2f04..1488270f35e9f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -18,8 +18,8 @@ import reactor.util.annotation.Nullable; import java.net.URL; +import java.util.ArrayList; import java.util.Date; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -118,8 +118,8 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { case QP_IS_ON: lastValidTransmission = sendTime; String etagValue = networkHelper.getEtagHeaderValue(response); - if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + if (etagValue != null) { + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } operationLogger.recordSuccess(); @@ -137,8 +137,6 @@ t, getQuickPulseEndpoint(), friendlyExceptionThrown, logger)) { } } finally { if (response != null) { - - // need to consume the body or close the response, otherwise get netty ByteBuf leak // warnings: // io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java index 6c70bc5d36323..e1b6db5a52376 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java @@ -1,16 +1,10 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.HashMap; - public class OpenTelDataPoint { private double value; - //private HashMap dimensions; public OpenTelDataPoint(double value) { this.value = value; - //this.dimensions = new HashMap<>(); } public double getValue() { @@ -20,7 +14,4 @@ public double getValue() { public void setValue(double value) { this.value = value; } - - - } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 80f21031de678..21ac6c0796836 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -209,15 +209,15 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); assertThat(quickPulseConfiguration.getEtag()).isEqualTo("3::randometag::4::"); assertThat(quickPulseConfiguration.getMetrics().size()).isEqualTo(2); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getAggregation()) + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getAggregation()) .isEqualTo("Avg"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getTelemetryType()) + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getTelemetryType()) .isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getProjection()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getAggregation()).isEqualTo("Sum"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getTelemetryType()).isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getProjection()).isEqualTo("MyFruitCounter"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").getId()).isEqualTo("MyFruitCounter"); + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getProjection()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getId()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getAggregation()).isEqualTo("Sum"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getTelemetryType()).isEqualTo("Metric"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getProjection()).isEqualTo("MyFruitCounter"); + assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getId()).isEqualTo("MyFruitCounter"); } } From e2113f76dd22677e81a5676c20745f599a823c44 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 24 Jul 2024 09:54:50 -0700 Subject: [PATCH 12/22] temporarily removing timestamp --- .../quickpulse/QuickPulseDataCollector.java | 25 ++++++++++++++++--- .../quickpulse/model/OpenTelMetric.java | 9 ++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 828f13dfaf67c..9613e28b3d87c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -457,14 +457,14 @@ class OpenTelMetricsStorage { // setting most amount of openTel metrics a user can stream to this. will review if users surpass threshold. private final int maxMetricsLimit = 50; // setting buffer time for receiving data for a metric before declaring it inactive. (in minutes) - private final double metricBufferTime = 5; + //private final double metricBufferTime = 5; public void addMetric(String metricName, double value) { OpenTelMetric metric = metrics.get(metricName); if (metric == null) { if (metrics.size() <= maxMetricsLimit) { // create new metric and add to metrics map - // (if we have reached the limit, we will not create a new metric and add it to the map + // (if we have reached the limit, we will block the metric from being created metric = new OpenTelMetric(metricName); metric.addDataPoint(value); metrics.putIfAbsent(metricName, metric); @@ -478,6 +478,23 @@ public ArrayList processMetrics() { ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getMetrics(); ArrayList processedMetrics = new ArrayList<>(); + for(Map.Entry entry : this.metrics.entrySet()) { + + String key = entry.getKey(); + OpenTelMetric value = entry.getValue(); + + if (requestedMetrics.containsKey(key)) { + for (QuickPulseConfiguration.OpenTelMetricInfo metricInfo : requestedMetrics.get(key)) { + QuickPulseMetrics processedMetric = processMetric(value, metricInfo); + processedMetrics.add(processedMetric); + } + } + } + + this.clearMetrics(); + return processedMetrics; + + /* Iterator> iterator = this.metrics.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -503,6 +520,8 @@ public ArrayList processMetrics() { } return processedMetrics; + */ + } public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfiguration.OpenTelMetricInfo metricInfo) { @@ -533,7 +552,7 @@ public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfigur } public void clearMetrics() { - this.metrics = new ConcurrentHashMap(); + this.metrics.clear(); } //for testing diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java index 32307b0e35369..29e0abf362e0e 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java @@ -7,18 +7,18 @@ public class OpenTelMetric { private String name; private ArrayList dataPoints; - private LocalDateTime lastTimestamp; + //private LocalDateTime lastTimestamp; public OpenTelMetric(String name) { this.name = name; this.dataPoints = new ArrayList<>(); - this.lastTimestamp = LocalDateTime.now(); + //this.lastTimestamp = LocalDateTime.now(); } public void addDataPoint(double value) { OpenTelDataPoint dataPoint = new OpenTelDataPoint(value); this.dataPoints.add(dataPoint); - this.updateLastTimestamp(); + //this.updateLastTimestamp(); } public String getName() { @@ -37,13 +37,16 @@ public ArrayList getDataValues() { return values; } + /* public LocalDateTime getLastTimestamp() { return lastTimestamp; } + public void updateLastTimestamp() { this.lastTimestamp = LocalDateTime.now(); } + */ public void clearDataPoints() { this.dataPoints.clear(); From 472287fd2979347a6538c2e6ef8414cafbe5d00f Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 24 Jul 2024 14:40:43 -0700 Subject: [PATCH 13/22] updated functionality --- ...penTelDataPoint.java => OTelDataPoint.java} | 2 +- .../{OpenTelMetric.java => OTelMetric.java} | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) rename sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/{OpenTelDataPoint.java => OTelDataPoint.java} (91%) rename sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/{OpenTelMetric.java => OTelMetric.java} (69%) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java similarity index 91% rename from sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java rename to sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java index e1b6db5a52376..515ab70bca7ff 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java @@ -1,6 +1,6 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; -public class OpenTelDataPoint { +public class OTelDataPoint { private double value; public OpenTelDataPoint(double value) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java similarity index 69% rename from sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java rename to sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java index 29e0abf362e0e..6fad6ce08b7e2 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java @@ -7,18 +7,15 @@ public class OpenTelMetric { private String name; private ArrayList dataPoints; - //private LocalDateTime lastTimestamp; public OpenTelMetric(String name) { this.name = name; this.dataPoints = new ArrayList<>(); - //this.lastTimestamp = LocalDateTime.now(); } public void addDataPoint(double value) { OpenTelDataPoint dataPoint = new OpenTelDataPoint(value); this.dataPoints.add(dataPoint); - //this.updateLastTimestamp(); } public String getName() { @@ -37,19 +34,4 @@ public ArrayList getDataValues() { return values; } - /* - public LocalDateTime getLastTimestamp() { - return lastTimestamp; - } - - - public void updateLastTimestamp() { - this.lastTimestamp = LocalDateTime.now(); - } - */ - - public void clearDataPoints() { - this.dataPoints.clear(); - } - } From 52fc3483b6de00602521e83090ca2bcfcf5a04d9 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 24 Jul 2024 14:47:08 -0700 Subject: [PATCH 14/22] updated invariant version in tests --- .../implementation/quickpulse/QuickPulse.java | 2 +- .../quickpulse/QuickPulseConfiguration.java | 40 ++++------ .../quickpulse/QuickPulseCoordinator.java | 2 +- .../quickpulse/QuickPulseDataCollector.java | 78 ++++++------------- .../quickpulse/QuickPulseDataFetcher.java | 6 +- .../quickpulse/QuickPulseDataSender.java | 2 +- .../quickpulse/QuickPulsePingSender.java | 2 +- .../quickpulse/model/OTelDataPoint.java | 2 +- .../quickpulse/model/OTelMetric.java | 13 ++-- .../QuickPulseDataCollectorTests.java | 14 ++-- .../QuickPulseDataFetcherTests.java | 4 +- .../QuickPulseIntegrationTests.java | 4 +- .../quickpulse/QuickPulsePingSenderTests.java | 55 +++---------- 13 files changed, 77 insertions(+), 147 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index 169ea5178a49c..e54309fcab919 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -21,7 +21,7 @@ public class QuickPulse { - //supports Open Tel metrics but not live filtering + //supports OTel metrics but not live filtering // change to 7 once live filtering is also supported static final int QP_INVARIANT_VERSION = 6; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index aa5c569e60263..cf053526b95d0 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -14,7 +14,7 @@ public class QuickPulseConfiguration { private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); private AtomicReference etag = new AtomicReference<>(); - private ConcurrentHashMap> metrics = new ConcurrentHashMap<>(); + private ConcurrentHashMap> derivedMetrics = new ConcurrentHashMap<>(); private static final ObjectMapper objectMapper = new ObjectMapper(); public synchronized String getEtag() { @@ -25,67 +25,61 @@ public synchronized void setEtag(String etag) { this.etag.set(etag); } - public synchronized ConcurrentHashMap> getMetrics() { - return this.metrics; + public synchronized ConcurrentHashMap> getDerivedMetrics() { + return this.derivedMetrics; } - public synchronized void setMetrics(ConcurrentHashMap> metrics) { - this.metrics = metrics; + public synchronized void setDerivedMetrics(ConcurrentHashMap> derivedMetrics) { + this.derivedMetrics = derivedMetrics; } - public synchronized void updateConfig(String etagValue, ConcurrentHashMap> otelMetrics) { + public synchronized void updateConfig(String etagValue, ConcurrentHashMap> otelMetrics) { if (!Objects.equals(this.getEtag(), etagValue)){ this.setEtag(etagValue); - this.setMetrics(otelMetrics); + this.setDerivedMetrics(otelMetrics); } } - public ConcurrentHashMap> parseMetrics(HttpResponse response) { + public ConcurrentHashMap> parseDerivedMetrics(HttpResponse response) { - ConcurrentHashMap> requestedMetrics = new ConcurrentHashMap<>(); + ConcurrentHashMap> requestedMetrics = new ConcurrentHashMap<>(); try { String responseBody = response.getBodyAsString().block(); if (responseBody == null || responseBody.isEmpty()) { - return new ConcurrentHashMap>(); + return new ConcurrentHashMap>(); } JsonNode rootNode = objectMapper.readTree(responseBody); JsonNode metricsNode = rootNode.get("Metrics"); if (metricsNode instanceof ArrayNode) { ArrayNode metricsArray = (ArrayNode) metricsNode; + for (JsonNode metricNode : metricsArray) { - OpenTelMetricInfo metric = new OpenTelMetricInfo(); + DerivedMetricInfo metric = new DerivedMetricInfo(); metric.setId(metricNode.get("Id").asText()); metric.setAggregation(metricNode.get("Aggregation").asText()); metric.setTelemetryType(metricNode.get("TelemetryType").asText()); - String projection = metricNode.get("Projection").asText(); - metric.setProjection(projection); - if (Objects.equals(metricNode.get("TelemetryType").asText(), "Event")) { - int dotIndex = projection.indexOf("."); - if (dotIndex != -1) { - projection = projection.substring(dotIndex + 1); - } - } - metric.setProjection(projection); + metric.setProjection(metricNode.get("Projection").asText()); requestedMetrics.computeIfAbsent(metric.getProjection(), k -> new ArrayList<>()).add(metric); + } } return requestedMetrics; } catch (Exception e) { logger.verbose("Failed to parse metrics from response: %s", e.getMessage()); } - return new ConcurrentHashMap>(); + return new ConcurrentHashMap>(); } public synchronized void reset() { this.setEtag(null); - this.setMetrics(new ConcurrentHashMap>()); + this.derivedMetrics.clear(); } - class OpenTelMetricInfo { + class DerivedMetricInfo { private String id; private String projection; private String telemetryType; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java index a54e08a71994f..206c4cd83593f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java @@ -78,7 +78,7 @@ private long sendData() { case QP_IS_OFF: pingMode = true; - collector.flushOpenTelMetrics(); + collector.flushOTelMetrics(); return qpsServicePollingIntervalHintMillis > 0 ? qpsServicePollingIntervalHintMillis : waitBetweenPingsInMillis; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 9613e28b3d87c..a24333211b5e3 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -20,8 +20,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; final class QuickPulseDataCollector { @@ -35,7 +33,7 @@ final class QuickPulseDataCollector { private QuickPulseConfiguration quickPulseConfiguration; - private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); + private OTelMetricsStorage metricsStorage = new OTelMetricsStorage(); private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator = getCpuPerformanceCounterCalculator(); @@ -111,11 +109,10 @@ synchronized void addOtelMetric(TelemetryItem telemetryItem){ // sampleRate should never be zero (how could it be captured if sampling set to zero percent?) return; } - int itemCount = sampleRate == null ? 1 : Math.round(100 / sampleRate); if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { - MonitorDomain data2 = telemetryItem.getData().getBaseData(); - MetricsData metricsData = (MetricsData) data2; + MonitorDomain data = telemetryItem.getData().getBaseData(); + MetricsData metricsData = (MetricsData) data; MetricDataPoint point = metricsData.getMetrics().get(0); this.metricsStorage.addMetric(point.getName(), point.getValue()); } @@ -339,15 +336,15 @@ private static int charToInt(char c) { return x; } - public ArrayList retrieveOpenTelMetrics() { + public ArrayList retrieveOTelMetrics() { return metricsStorage.processMetrics(); } - public ConcurrentHashMap getOpenTelMetrics() { + public ConcurrentHashMap getOTelMetrics() { return metricsStorage.getMetrics(); } - public void flushOpenTelMetrics() { + public void flushOTelMetrics() { metricsStorage.clearMetrics(); } @@ -452,20 +449,18 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { } } - class OpenTelMetricsStorage { - private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); - // setting most amount of openTel metrics a user can stream to this. will review if users surpass threshold. + class OTelMetricsStorage { + private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + // setting most amount of OTel metrics a user can stream to this. will review if users surpass threshold. private final int maxMetricsLimit = 50; - // setting buffer time for receiving data for a metric before declaring it inactive. (in minutes) - //private final double metricBufferTime = 5; public void addMetric(String metricName, double value) { - OpenTelMetric metric = metrics.get(metricName); + OTelMetric metric = metrics.get(metricName); if (metric == null) { + // create new metric and add to metrics map + // (if we have reached the limit, we will block the metric from being created if (metrics.size() <= maxMetricsLimit) { - // create new metric and add to metrics map - // (if we have reached the limit, we will block the metric from being created - metric = new OpenTelMetric(metricName); + metric = new OTelMetric(metricName); metric.addDataPoint(value); metrics.putIfAbsent(metricName, metric); } @@ -475,18 +470,19 @@ public void addMetric(String metricName, double value) { } public ArrayList processMetrics() { - ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getMetrics(); + ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getDerivedMetrics(); ArrayList processedMetrics = new ArrayList<>(); - for(Map.Entry entry : this.metrics.entrySet()) { - + for(Map.Entry entry : this.metrics.entrySet()) { String key = entry.getKey(); - OpenTelMetric value = entry.getValue(); + OTelMetric value = entry.getValue(); if (requestedMetrics.containsKey(key)) { - for (QuickPulseConfiguration.OpenTelMetricInfo metricInfo : requestedMetrics.get(key)) { - QuickPulseMetrics processedMetric = processMetric(value, metricInfo); - processedMetrics.add(processedMetric); + for (QuickPulseConfiguration.DerivedMetricInfo metricInfo : requestedMetrics.get(key)) { + if (Objects.equals(metricInfo.getTelemetryType(), "Metric")) { + QuickPulseMetrics processedMetric = processMetric(value, metricInfo); + processedMetrics.add(processedMetric); + } } } } @@ -494,37 +490,9 @@ public ArrayList processMetrics() { this.clearMetrics(); return processedMetrics; - /* - Iterator> iterator = this.metrics.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String key = entry.getKey(); - OpenTelMetric value = entry.getValue(); - - if (requestedMetrics.containsKey(key)) { - for (QuickPulseConfiguration.OpenTelMetricInfo metricInfo : requestedMetrics.get(key)) { - QuickPulseMetrics processedMetric = processMetric(value, metricInfo); - processedMetrics.add(processedMetric); - } - - } - - if (ChronoUnit.MINUTES.between(value.getLastTimestamp(), LocalDateTime.now()) > metricBufferTime) { - iterator.remove(); - - } - else { - value.clearDataPoints(); - } - - } - return processedMetrics; - - */ - } - public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfiguration.OpenTelMetricInfo metricInfo) { + public QuickPulseMetrics processMetric(OTelMetric metric, QuickPulseConfiguration.DerivedMetricInfo metricInfo) { if (metric.getDataPoints().isEmpty()) { return new QuickPulseMetrics(metricInfo.getId(), 0, 1); @@ -556,7 +524,7 @@ public void clearMetrics() { } //for testing - public ConcurrentHashMap getMetrics() { + public ConcurrentHashMap getMetrics() { return metrics; } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index a0e2785836f83..f379b77005c62 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -143,14 +143,14 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) postEnvelope.setStreamId(quickPulseId); postEnvelope.setVersion(sdkVersion); postEnvelope.setTimeStamp("/Date(" + System.currentTimeMillis() + ")/"); - postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters, collector.retrieveOpenTelMetrics())); + postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters, collector.retrieveOTelMetrics())); envelopes.add(postEnvelope); return mapper.writeValueAsString(envelopes); } private static List addMetricsToQuickPulseEnvelope( QuickPulseDataCollector.FinalCounters counters, - List openTelemetryMetrics) { + List OTelMetrics) { List metricsList = new ArrayList<>(); metricsList.add( new QuickPulseMetrics("\\ApplicationInsights\\Requests/Sec", counters.requests, 1)); @@ -192,7 +192,7 @@ private static List addMetricsToQuickPulseEnvelope( new QuickPulseMetrics("\\Memory\\Committed Bytes", counters.memoryCommitted, 1)); metricsList.add( new QuickPulseMetrics("\\Processor(_Total)\\% Processor Time", counters.cpuUsage, 1)); - metricsList.addAll(openTelemetryMetrics); + metricsList.addAll(OTelMetrics); return metricsList; } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 076a235dd8c81..4b0bd4e888ab6 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -65,7 +65,7 @@ public void run() { this.quickPulseHeaderInfo = quickPulseHeaderInfo; String etagValue = networkHelper.getEtagHeaderValue(response); if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseMetrics(response); + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } break; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 1488270f35e9f..ad29ad54adc10 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -119,7 +119,7 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { lastValidTransmission = sendTime; String etagValue = networkHelper.getEtagHeaderValue(response); if (etagValue != null) { - ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseMetrics(response); + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } operationLogger.recordSuccess(); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java index 515ab70bca7ff..96786a6668725 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java @@ -3,7 +3,7 @@ public class OTelDataPoint { private double value; - public OpenTelDataPoint(double value) { + public OTelDataPoint(double value) { this.value = value; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java index 6fad6ce08b7e2..57d80786b196b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java @@ -1,20 +1,19 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; import java.util.ArrayList; -import java.time.LocalDateTime; -public class OpenTelMetric { +public class OTelMetric { private String name; - private ArrayList dataPoints; + private ArrayList dataPoints; - public OpenTelMetric(String name) { + public OTelMetric(String name) { this.name = name; this.dataPoints = new ArrayList<>(); } public void addDataPoint(double value) { - OpenTelDataPoint dataPoint = new OpenTelDataPoint(value); + OTelDataPoint dataPoint = new OTelDataPoint(value); this.dataPoints.add(dataPoint); } @@ -22,13 +21,13 @@ public String getName() { return name; } - public ArrayList getDataPoints() { + public ArrayList getDataPoints() { return dataPoints; } public ArrayList getDataValues() { ArrayList values = new ArrayList<>(); - for (OpenTelDataPoint dataPoint : dataPoints) { + for (OTelDataPoint dataPoint : dataPoints) { values.add(dataPoint.getValue()); } return values; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java index 73e2c9ed27177..d07a90d7c8474 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java @@ -9,7 +9,7 @@ import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorBase; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.OpenTelMetric; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.OTelMetric; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseMetrics; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.sdk.resources.Resource; @@ -42,7 +42,7 @@ void emptyCountsAndDurationsAfterEnable() { collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); QuickPulseDataCollector.FinalCounters counters = collector.peek(); assertCountersReset(counters); - ArrayList storedMetrics = collector.retrieveOpenTelMetrics(); + ArrayList storedMetrics = collector.retrieveOTelMetrics(); assertThat(storedMetrics).isEmpty(); } @@ -53,7 +53,7 @@ void nullCountersAfterDisable() { collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); collector.disable(); assertThat(collector.peek()).isNull(); - assertThat(collector.retrieveOpenTelMetrics()).isEmpty(); + assertThat(collector.retrieveOTelMetrics()).isEmpty(); } @Test @@ -187,7 +187,7 @@ void openTelemetryMetricsAreCounted() { Resource resource = Resource.create(attributes); telemetry.setResource(resource); collector.addOtelMetric(telemetry); - ConcurrentHashMap storedMetrics = collector.getOpenTelMetrics(); + ConcurrentHashMap storedMetrics = collector.getOTelMetrics(); assertThat(storedMetrics.size()).isEqualTo(1); assertThat(storedMetrics.containsKey("TestMetric")).isTrue(); assertThat(storedMetrics.get("TestMetric").getDataValues().get(0)).isEqualTo(123.456); @@ -195,13 +195,13 @@ void openTelemetryMetricsAreCounted() { point.setName("TestMetric2"); point.setValue(789.012); collector.addOtelMetric(telemetry); - storedMetrics = collector.getOpenTelMetrics(); + storedMetrics = collector.getOTelMetrics(); assertThat(storedMetrics.size()).isEqualTo(2); assertThat(storedMetrics.containsKey("TestMetric2")).isTrue(); assertThat(storedMetrics.get("TestMetric2").getDataValues().get(0)).isEqualTo(789.012); - collector.flushOpenTelMetrics(); - assertThat(collector.getOpenTelMetrics().size()).isEqualTo(0); + collector.flushOTelMetrics(); + assertThat(collector.getOTelMetrics().size()).isEqualTo(0); } @Test diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index c265d3ddc3e47..1e9fb8e054f02 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -95,7 +95,7 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); + headers.put("x-ms-qps-configuration-etag", ""); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); @@ -120,6 +120,6 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(quickPulseConfiguration.getEtag()).isEqualTo("0::randometag::1::"); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo(""); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java index 13eb62ff44ce1..c002bc52e0968 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java @@ -87,7 +87,7 @@ public void testPingWithAuthentication() { public void testPingRequestBody() throws InterruptedException { CountDownLatch pingCountDown = new CountDownLatch(1); String expectedRequestBody = - "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; + "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":6,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithValidator( new ValidationPolicy(pingCountDown, expectedRequestBody), new QuickPulseConfiguration()); @@ -105,7 +105,7 @@ public void testPostRequest() throws InterruptedException { Date currDate = new Date(); QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); String expectedPingRequestBody = - "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; + "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":6,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; String expectedPostRequestBody = "\\[\\{\"Documents\":\\[\\{\"__type\":\"RequestTelemetryDocument\",\"DocumentType\":\"Request\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"request-test\",\"Success\":true,\"Duration\":\"PT.*S\",\"ResponseCode\":\"200\",\"OperationName\":null,\"Url\":\"foo\"\\},\\{\"__type\":\"DependencyTelemetryDocument\",\"DocumentType\":\"RemoteDependency\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"dep-test\",\"Target\":null,\"Success\":true,\"Duration\":\"PT.*S\",\"ResultCode\":null,\"CommandName\":\"dep-test-cmd\",\"DependencyTypeName\":null,\"OperationName\":null\\},\\{\"__type\":\"ExceptionTelemetryDocument\",\"DocumentType\":\"Exception\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":null,\"Exception\":\"\",\"ExceptionMessage\":\"test\",\"ExceptionType\":\"java.lang.Exception\"\\}\\],\"InstrumentationKey\":\"" + instrumentationKey diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 21ac6c0796836..06de782108d34 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -112,40 +112,6 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { assertThat(quickPulseConfiguration.getEtag()).isEqualTo("1::randometag::2::"); } - - @Test - void successfulPingReturnsWithEtagHeader() { - Map headers = new HashMap<>(); - headers.put("x-ms-qps-service-polling-interval-hint", "1000"); - headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); - headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "2::randometag::3::"); - HttpHeaders httpHeaders = new HttpHeaders(headers); - ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); - QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); - HttpPipeline httpPipeline = - new HttpPipelineBuilder() - .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) - .tracer(new NoopTracer()) - .build(); - QuickPulsePingSender quickPulsePingSender = - new QuickPulsePingSender( - httpPipeline, - connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, - null, - "instance1", - "machine1", - "qpid123", - "testSdkVersion", - quickPulseConfiguration); - QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); - assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); - assertThat("https://new.endpoint.com") - .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(quickPulseConfiguration.getEtag()).isEqualTo("2::randometag::3::"); - } - @Test void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { Map headers = new HashMap<>(); @@ -176,7 +142,10 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { metrics.add(metric2); Map metricsMap = new HashMap<>(); + metricsMap.put("DocumentStreams", null); + metricsMap.put("ETag", "3::randometag::4::"); metricsMap.put("Metrics", metrics); + metricsMap.put("QuotaInfo", null); ObjectMapper objectMapper = new ObjectMapper(); String jsonBody; try { @@ -208,16 +177,16 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); assertThat(quickPulseConfiguration.getEtag()).isEqualTo("3::randometag::4::"); - assertThat(quickPulseConfiguration.getMetrics().size()).isEqualTo(2); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getAggregation()) + assertThat(quickPulseConfiguration.getDerivedMetrics().size()).isEqualTo(2); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getAggregation()) .isEqualTo("Avg"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getTelemetryType()) + assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getTelemetryType()) .isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getProjection()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getMetrics().get("my_gauge").get(0).getId()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getAggregation()).isEqualTo("Sum"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getTelemetryType()).isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getProjection()).isEqualTo("MyFruitCounter"); - assertThat(quickPulseConfiguration.getMetrics().get("MyFruitCounter").get(0).getId()).isEqualTo("MyFruitCounter"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getProjection()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getId()).isEqualTo("my_gauge"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getAggregation()).isEqualTo("Sum"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getTelemetryType()).isEqualTo("Metric"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getProjection()).isEqualTo("MyFruitCounter"); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getId()).isEqualTo("MyFruitCounter"); } } From ac7631a7b10f05bf019f9c4b2697b584cd1058ab Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Wed, 24 Jul 2024 15:47:05 -0700 Subject: [PATCH 15/22] deleted reset function in config --- .../implementation/quickpulse/QuickPulseConfiguration.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index cf053526b95d0..dbd6f7748c4f5 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -74,11 +74,6 @@ public ConcurrentHashMap> parseDerivedMetr } - public synchronized void reset() { - this.setEtag(null); - this.derivedMetrics.clear(); - } - class DerivedMetricInfo { private String id; private String projection; From 9ec6d169051a40c59ca499a69052dd2431e222ad Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Thu, 1 Aug 2024 14:59:33 -0700 Subject: [PATCH 16/22] refactored storage of derived metric info --- .../implementation/quickpulse/QuickPulse.java | 2 +- .../quickpulse/QuickPulseConfiguration.java | 2 +- .../quickpulse/QuickPulseDataCollector.java | 24 ++++++++++--------- .../quickpulse/QuickPulseMetricReceiver.java | 1 + .../quickpulse/QuickPulsePingSenderTests.java | 21 +++++++++------- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index e54309fcab919..6268aa953ae11 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -22,9 +22,9 @@ public class QuickPulse { //supports OTel metrics but not live filtering - // change to 7 once live filtering is also supported static final int QP_INVARIANT_VERSION = 6; + private volatile QuickPulseDataCollector collector; public static QuickPulse create( diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index dbd6f7748c4f5..c472c722c2433 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -62,7 +62,7 @@ public ConcurrentHashMap> parseDerivedMetr metric.setAggregation(metricNode.get("Aggregation").asText()); metric.setTelemetryType(metricNode.get("TelemetryType").asText()); metric.setProjection(metricNode.get("Projection").asText()); - requestedMetrics.computeIfAbsent(metric.getProjection(), k -> new ArrayList<>()).add(metric); + requestedMetrics.computeIfAbsent(metric.getTelemetryType(), k -> new ArrayList<>()).add(metric); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index a24333211b5e3..6c0deb1dd956b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -113,6 +113,10 @@ synchronized void addOtelMetric(TelemetryItem telemetryItem){ if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { MonitorDomain data = telemetryItem.getData().getBaseData(); MetricsData metricsData = (MetricsData) data; + System.out.println("Properties: " + metricsData.getProperties()); + if (metricsData.getProperties() != null && metricsData.getProperties().get("testing") != null) { + System.out.println("Type: " + metricsData.getProperties().get("testing").length()); + } MetricDataPoint point = metricsData.getMetrics().get(0); this.metricsStorage.addMetric(point.getName(), point.getValue()); } @@ -473,23 +477,17 @@ public ArrayList processMetrics() { ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getDerivedMetrics(); ArrayList processedMetrics = new ArrayList<>(); - for(Map.Entry entry : this.metrics.entrySet()) { - String key = entry.getKey(); - OTelMetric value = entry.getValue(); - - if (requestedMetrics.containsKey(key)) { - for (QuickPulseConfiguration.DerivedMetricInfo metricInfo : requestedMetrics.get(key)) { - if (Objects.equals(metricInfo.getTelemetryType(), "Metric")) { - QuickPulseMetrics processedMetric = processMetric(value, metricInfo); - processedMetrics.add(processedMetric); - } + if (requestedMetrics.get("Metric") != null) { + for (QuickPulseConfiguration.DerivedMetricInfo metricInfo : requestedMetrics.get("Metric")) { + if (this.getMetrics().get(metricInfo.getProjection()) != null) { + QuickPulseMetrics processedMetric = processMetric(this.getMetrics().get(metricInfo.getProjection()), metricInfo); + processedMetrics.add(processedMetric); } } } this.clearMetrics(); return processedMetrics; - } public QuickPulseMetrics processMetric(OTelMetric metric, QuickPulseConfiguration.DerivedMetricInfo metricInfo) { @@ -501,6 +499,10 @@ public QuickPulseMetrics processMetric(OTelMetric metric, QuickPulseConfiguratio String aggregation = metricInfo.getAggregation(); ArrayList dataValues = metric.getDataValues(); + if (Objects.equals(metricInfo.getProjection(), "UserEngagement")) { + System.out.println("User Engagement: " + dataValues); + } + switch (aggregation) { case "Sum": double sum = dataValues.stream().mapToDouble(Double::doubleValue).sum(); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java index 4801a987a0635..f988b9bcf5753 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -53,6 +53,7 @@ public void run() { for (MetricData metricData : metrics) { try { + System.out.println("MetricData: " + metricData.getData().getPoints()); mapper.mapMetrics(metricData, quickPulseConsumer); metricReceiverLogger.recordSuccess(); } catch (Throwable t) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 06de782108d34..70fad123d3323 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -177,16 +177,19 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { assertThat("https://new.endpoint.com") .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); assertThat(quickPulseConfiguration.getEtag()).isEqualTo("3::randometag::4::"); - assertThat(quickPulseConfiguration.getDerivedMetrics().size()).isEqualTo(2); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getAggregation()) + assertThat(quickPulseConfiguration.getDerivedMetrics().size()).isEqualTo(1); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("Metric").size()).isEqualTo(2); + ArrayList metricCategory = + quickPulseConfiguration.getDerivedMetrics().get("Metric"); + assertThat(metricCategory.get(0).getAggregation()) .isEqualTo("Avg"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getTelemetryType()) + assertThat(metricCategory.get(0).getTelemetryType()) .isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getProjection()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("my_gauge").get(0).getId()).isEqualTo("my_gauge"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getAggregation()).isEqualTo("Sum"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getTelemetryType()).isEqualTo("Metric"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getProjection()).isEqualTo("MyFruitCounter"); - assertThat(quickPulseConfiguration.getDerivedMetrics().get("MyFruitCounter").get(0).getId()).isEqualTo("MyFruitCounter"); + assertThat(metricCategory.get(0).getProjection()).isEqualTo("my_gauge"); + assertThat(metricCategory.get(0).getId()).isEqualTo("my_gauge"); + assertThat(metricCategory.get(1).getAggregation()).isEqualTo("Sum"); + assertThat(metricCategory.get(1).getTelemetryType()).isEqualTo("Metric"); + assertThat(metricCategory.get(1).getProjection()).isEqualTo("MyFruitCounter"); + assertThat(metricCategory.get(1).getId()).isEqualTo("MyFruitCounter"); } } From 7cbc3714c7746545ce124a790c1fd0718b87773f Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Fri, 9 Aug 2024 16:24:39 -0700 Subject: [PATCH 17/22] added filtering support --- .../quickpulse/QuickPulseConfiguration.java | 67 +++++++++++++++++++ .../quickpulse/QuickPulseDataCollector.java | 41 ++++++++---- .../quickpulse/model/OTelDataPoint.java | 14 +++- .../quickpulse/model/OTelMetric.java | 5 +- .../quickpulse/QuickPulsePingSenderTests.java | 19 +++++- 5 files changed, 129 insertions(+), 17 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index c472c722c2433..3273e1b58b75b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -64,6 +64,28 @@ public ConcurrentHashMap> parseDerivedMetr metric.setProjection(metricNode.get("Projection").asText()); requestedMetrics.computeIfAbsent(metric.getTelemetryType(), k -> new ArrayList<>()).add(metric); + + JsonNode filterGroupsNode = metricNode.get("FilterGroups"); + if (filterGroupsNode instanceof ArrayNode) { + ArrayNode filterGroupsArray = (ArrayNode) filterGroupsNode; + for (JsonNode filterGroupNode : filterGroupsArray) { + JsonNode filtersNode = filterGroupNode.get("Filters"); + if (filtersNode instanceof ArrayNode) { + ArrayNode filtersArray = (ArrayNode) filtersNode; + for (JsonNode filterNode : filtersArray) { + String fieldName = filterNode.get("FieldName").asText(); + if (fieldName.contains(".")) { + fieldName = fieldName.split("\\.")[1]; + } + String predicate = filterNode.get("Predicate").asText(); + String comparand = filterNode.get("Comparand").asText(); + if (!fieldName.isEmpty() && !fieldName.equals("undefined") && !predicate.isEmpty() && !comparand.isEmpty()) { + metric.addFilterGroup(fieldName, predicate, comparand); + } + } + } + } + } } } return requestedMetrics; @@ -79,6 +101,7 @@ class DerivedMetricInfo { private String projection; private String telemetryType; private String aggregation; + private ArrayList filterGroups = new ArrayList<>(); public String getId() { return this.id; @@ -112,6 +135,50 @@ public String getAggregation() { public void setAggregation(String aggregation) { this.aggregation = aggregation; } + + public ArrayList getFilterGroups() { + return this.filterGroups; + } + + public void addFilterGroup(String fieldName, String predicate, String comparand) { + this.filterGroups.add(new FilterGroup(fieldName, predicate, comparand)); + } + } + + class FilterGroup { + private String fieldName; + private String operator; + private String comparand; + + public FilterGroup(String fieldName, String predicate, String comparand) { + this.fieldName = fieldName; + this.operator = predicate; + this.comparand = comparand; + } + + public String getFieldName() { + return this.fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String getOperator() { + return this.operator; + } + + public void setOperator(String operator) { + this.operator = operator; + } + + public String getComparand() { + return this.comparand; + } + + public void setComparand(String comparand) { + this.comparand = comparand; + } } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 6c0deb1dd956b..a74505610491c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -118,7 +118,7 @@ synchronized void addOtelMetric(TelemetryItem telemetryItem){ System.out.println("Type: " + metricsData.getProperties().get("testing").length()); } MetricDataPoint point = metricsData.getMetrics().get(0); - this.metricsStorage.addMetric(point.getName(), point.getValue()); + this.metricsStorage.addMetric(point.getName(), point.getValue(), metricsData.getProperties()); } } @@ -458,18 +458,18 @@ class OTelMetricsStorage { // setting most amount of OTel metrics a user can stream to this. will review if users surpass threshold. private final int maxMetricsLimit = 50; - public void addMetric(String metricName, double value) { + public void addMetric(String metricName, double value, Map dimensions) { OTelMetric metric = metrics.get(metricName); if (metric == null) { // create new metric and add to metrics map // (if we have reached the limit, we will block the metric from being created if (metrics.size() <= maxMetricsLimit) { metric = new OTelMetric(metricName); - metric.addDataPoint(value); + metric.addDataPoint(value, dimensions == null ? new HashMap<>() : new HashMap<>(dimensions)); metrics.putIfAbsent(metricName, metric); } } else { - metric.addDataPoint(value); + metric.addDataPoint(value, dimensions == null ? new HashMap<>() : new HashMap<>(dimensions)); } } @@ -497,24 +497,39 @@ public QuickPulseMetrics processMetric(OTelMetric metric, QuickPulseConfiguratio } String aggregation = metricInfo.getAggregation(); - ArrayList dataValues = metric.getDataValues(); - - if (Objects.equals(metricInfo.getProjection(), "UserEngagement")) { - System.out.println("User Engagement: " + dataValues); + ArrayList filteredValues = new ArrayList<>(); + for (OTelDataPoint dataPoint : metric.getDataPoints()) { + boolean passedFilter = true; + for (QuickPulseConfiguration.FilterGroup filterGroup : metricInfo.getFilterGroups()) { + String fieldName = filterGroup.getFieldName(); + String operator = filterGroup.getOperator(); + String comparand = filterGroup.getComparand(); + if (Objects.equals(operator, "Equal")) { + String fieldValue = dataPoint.getDimensions().get(fieldName); + if (fieldValue == null || !fieldValue.equals(comparand)) { + passedFilter = false; + break; + // Handle the case where the value is null or does not equal the operator + } + } + } + if (passedFilter) { + filteredValues.add(dataPoint.getValue()); + } } switch (aggregation) { case "Sum": - double sum = dataValues.stream().mapToDouble(Double::doubleValue).sum(); + double sum = filteredValues.stream().mapToDouble(Double::doubleValue).sum(); return new QuickPulseMetrics(metricInfo.getId(), sum, 1); case "Avg": - double avg = dataValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); - return new QuickPulseMetrics(metricInfo.getId(), avg, dataValues.size()); + double avg = filteredValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); + return new QuickPulseMetrics(metricInfo.getId(), avg, filteredValues.size()); case "Min": - double min = dataValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); + double min = filteredValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), min, 1); case "Max": - double max = dataValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); + double max = filteredValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), max, 1); default: throw new IllegalArgumentException("Aggregation type not supported: " + aggregation); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java index 96786a6668725..d63ada52c8b71 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java @@ -1,10 +1,14 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; +import java.util.HashMap; + public class OTelDataPoint { private double value; + private HashMap dimensions; - public OTelDataPoint(double value) { + public OTelDataPoint(double value, HashMap dimensions) { this.value = value; + this.dimensions = dimensions; } public double getValue() { @@ -14,4 +18,12 @@ public double getValue() { public void setValue(double value) { this.value = value; } + + public HashMap getDimensions() { + return dimensions; + } + + public void setDimensions(HashMap dimensions) { + this.dimensions = dimensions; + } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java index 57d80786b196b..d51a1b3db9da3 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java @@ -1,6 +1,7 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; import java.util.ArrayList; +import java.util.HashMap; public class OTelMetric { @@ -12,8 +13,8 @@ public OTelMetric(String name) { this.dataPoints = new ArrayList<>(); } - public void addDataPoint(double value) { - OTelDataPoint dataPoint = new OTelDataPoint(value); + public void addDataPoint(double value, HashMap dimensions) { + OTelDataPoint dataPoint = new OTelDataPoint(value, dimensions); this.dataPoints.add(dataPoint); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index 70fad123d3323..d710701c524fd 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -130,7 +130,19 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { metric1.put("TelemetryType", "Metric"); metric1.put("Projection", "my_gauge"); metric1.put("BackendAggregation", "Min"); - metric1.put("FilterGroups", new ArrayList<>()); + + ArrayList>>> filterGroups = new ArrayList<>(); + HashMap>> filterGroup = new HashMap<>(); + ArrayList> filters = new ArrayList<>(); + HashMap filterOne = new HashMap<>(); + filterOne.put("FieldName", "Test"); + filterOne.put("Predicate", "Equals"); + filterOne.put("Comparand", "Value"); + filters.add(filterOne); + filterGroup.put("Filters", filters); + filterGroups.add(filterGroup); + metric1.put("FilterGroups", filterGroups); + Map metric2 = new HashMap<>(); metric2.put("Id", "MyFruitCounter"); metric2.put("Aggregation", "Sum"); @@ -185,6 +197,11 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { .isEqualTo("Avg"); assertThat(metricCategory.get(0).getTelemetryType()) .isEqualTo("Metric"); + assertThat(metricCategory.get(0).getFilterGroups().size() == 1); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getFieldName()) + .isEqualTo("Test"); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getOperator()).isEqualTo("Equals"); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getComparand()).isEqualTo("Value"); assertThat(metricCategory.get(0).getProjection()).isEqualTo("my_gauge"); assertThat(metricCategory.get(0).getId()).isEqualTo("my_gauge"); assertThat(metricCategory.get(1).getAggregation()).isEqualTo("Sum"); From dcc0e672a70f4cf502d77c66dd5d0240a55fba80 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Mon, 12 Aug 2024 10:57:14 -0700 Subject: [PATCH 18/22] working on fixing implementation --- .../implementation/MetricDataMapper.java | 17 ++- .../implementation/quickpulse/QuickPulse.java | 29 ++--- .../quickpulse/QuickPulseConfiguration.java | 114 +++++++++--------- .../quickpulse/QuickPulseDataCollector.java | 45 +++---- .../quickpulse/QuickPulseDataFetcher.java | 10 +- .../quickpulse/QuickPulseDataSender.java | 8 +- .../quickpulse/QuickPulseNetworkHelper.java | 9 +- .../quickpulse/QuickPulsePingSender.java | 6 +- .../quickpulse/model/OpenTelDataPoint.java | 2 - .../quickpulse/QuickPulsePingSenderTests.java | 71 +++++------ 10 files changed, 138 insertions(+), 173 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java index 646cdc6fdd301..5cfdbd6c9d700 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java @@ -72,18 +72,15 @@ public MetricDataMapper(BiConsumer telemetry this.captureHttpServer4xxAsError = captureHttpServer4xxAsError; } - public void mapMetrics(MetricData metricData, Consumer breezeConsumer, Consumer quickPulseConsumer) { + public void mapMetrics(MetricData metricData, Consumer breezeConsumer, + Consumer quickPulseConsumer) { MetricDataType type = metricData.getType(); - if (type == DOUBLE_SUM - || type == DOUBLE_GAUGE - || type == LONG_SUM - || type == LONG_GAUGE - || type == HISTOGRAM) { - boolean isPreAggregatedStandardMetric = - OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); + if (type == DOUBLE_SUM || type == DOUBLE_GAUGE || type == LONG_SUM || type == LONG_GAUGE || type == HISTOGRAM) { + boolean isPreAggregatedStandardMetric + = OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); if (isPreAggregatedStandardMetric) { - List preAggregatedStandardMetrics = - convertOtelMetricToAzureMonitorMetric(metricData, true); + List preAggregatedStandardMetrics + = convertOtelMetricToAzureMonitorMetric(metricData, true); preAggregatedStandardMetrics.forEach(breezeConsumer::accept); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index b38e1a3057f94..f58cfee9c6b8a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -25,13 +25,15 @@ public class QuickPulse { - static final int QP_INVARIANT_VERSION = 1; + // 6 represents filtering support for Otel metrics only is enabled + static final int QP_INVARIANT_VERSION = 6; private volatile QuickPulseDataCollector collector; public static QuickPulse create(HttpPipeline httpPipeline, Supplier endpointUrl, Supplier instrumentationKey, @Nullable String roleName, @Nullable String roleInstance, - boolean useNormalizedValueForNonNormalizedCpuPercentage, String sdkVersion) { + boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseMetricReader quickPulseMetricReader, + String sdkVersion) { QuickPulse quickPulse = new QuickPulse(); @@ -46,7 +48,7 @@ public static QuickPulse create(HttpPipeline httpPipeline, Supplier endpoin Thread.currentThread().interrupt(); } quickPulse.initialize(httpPipeline, endpointUrl, instrumentationKey, roleName, roleInstance, - useNormalizedValueForNonNormalizedCpuPercentage, sdkVersion); + useNormalizedValueForNonNormalizedCpuPercentage, quickPulseMetricReader, sdkVersion); }); // the condition below will always be false, but by referencing the executor it ensures the // executor can't become unreachable in the middle of the execute() method execution above @@ -69,23 +71,14 @@ public void add(TelemetryItem telemetryItem) { } } - public void add(TelemetryItem telemetryItem, Boolean isOtelMetric) { - if (collector != null){ - if(isOtelMetric){ - collector.addOtelMetric(telemetryItem); - - }else{ - collector.add(telemetryItem); - } - } - } - private void initialize(HttpPipeline httpPipeline, Supplier endpointUrl, Supplier instrumentationKey, @Nullable String roleName, @Nullable String roleInstance, - boolean useNormalizedValueForNonNormalizedCpuPercentage, String sdkVersion) { + boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseMetricReader quickPulseMetricReader, + String sdkVersion) { String quickPulseId = UUID.randomUUID().toString().replace("-", ""); ArrayBlockingQueue sendQueue = new ArrayBlockingQueue<>(256, true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulseDataSender quickPulseDataSender = new QuickPulseDataSender(httpPipeline, sendQueue); @@ -100,12 +93,12 @@ private void initialize(HttpPipeline httpPipeline, Supplier endpointUrl, Su } QuickPulseDataCollector collector - = new QuickPulseDataCollector(useNormalizedValueForNonNormalizedCpuPercentage); + = new QuickPulseDataCollector(useNormalizedValueForNonNormalizedCpuPercentage, quickPulseConfiguration); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(httpPipeline, endpointUrl, - instrumentationKey, roleName, instanceName, machineName, quickPulseId, sdkVersion); + instrumentationKey, roleName, instanceName, machineName, quickPulseId, sdkVersion, quickPulseConfiguration); QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(collector, sendQueue, endpointUrl, - instrumentationKey, roleName, instanceName, machineName, quickPulseId); + instrumentationKey, roleName, instanceName, machineName, quickPulseId, quickPulseConfiguration); QuickPulseCoordinatorInitData coordinatorInitData = new QuickPulseCoordinatorInitDataBuilder().withPingSender(quickPulsePingSender) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index 3c9a00d295a55..2f2533f1c6dbc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -1,35 +1,19 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; + import com.azure.core.http.HttpResponse; import com.azure.core.util.logging.ClientLogger; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; +import com.azure.json.*; +import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; - public class QuickPulseConfiguration { private static final ClientLogger logger = new ClientLogger(QuickPulseDataFetcher.class); - private static QuickPulseConfiguration instance = new QuickPulseConfiguration(); - private AtomicReference etag = new AtomicReference<>(); - private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); - private static final ObjectMapper objectMapper = new ObjectMapper(); - - - private QuickPulseConfiguration() { - } - - public static synchronized QuickPulseConfiguration getInstance() { - if (instance == null) { - instance = new QuickPulseConfiguration(); - } - return instance; - } + private AtomicReference etag = new AtomicReference<>(); + private ConcurrentHashMap> derivedMetrics = new ConcurrentHashMap<>(); public synchronized String getEtag() { return this.etag.get(); @@ -39,70 +23,85 @@ public synchronized void setEtag(String etag) { this.etag.set(etag); } - public synchronized ConcurrentHashMap getMetrics() { - return this.metrics; + public synchronized ConcurrentHashMap> getDerivedMetrics() { + return this.derivedMetrics; } - public synchronized void setMetrics(ConcurrentHashMap metrics) { - this.metrics = metrics; + public synchronized void setDerivedMetrics(ConcurrentHashMap> metrics) { + this.derivedMetrics = metrics; } - public synchronized void updateConfig(String etagValue, ConcurrentHashMap otelMetrics) { - if (!Objects.equals(this.getEtag(), etagValue)){ + public synchronized void updateConfig(String etagValue, ConcurrentHashMap> otelMetrics) { + if (!Objects.equals(this.getEtag(), etagValue)) { this.setEtag(etagValue); - this.setMetrics(otelMetrics); + this.setDerivedMetrics(otelMetrics); } } - public ConcurrentHashMap parseMetrics(HttpResponse response) { + public ConcurrentHashMap> parseDerivedMetrics(HttpResponse response) throws IOException { - HashSet metricsSet = new HashSet<>(); - ConcurrentHashMap requestedMetrics = new ConcurrentHashMap<>(); + ConcurrentHashMap> requestedMetrics = new ConcurrentHashMap<>(); try { String responseBody = response.getBodyAsString().block(); if (responseBody == null || responseBody.isEmpty()) { - return new ConcurrentHashMap(); + return new ConcurrentHashMap>(); } - JsonNode rootNode = objectMapper.readTree(responseBody); - //System.out.println("Metrics :" + rootNode.get("Metrics")); Debugging purposes - JsonNode metricsNode = rootNode.get("Metrics"); - - if (metricsNode instanceof ArrayNode) { - ArrayNode metricsArray = (ArrayNode) metricsNode; - for (JsonNode metricNode : metricsArray) { - OpenTelMetricInfo metric = new OpenTelMetricInfo(); - metric.setId(metricNode.get("Id").asText()); - metric.setAggregation(metricNode.get("Aggregation").asText()); - metric.setTelemetryType(metricNode.get("TelemetryType").asText()); - String projection = metricNode.get("Projection").asText(); - metric.setProjection(projection); - if (Objects.equals(metricNode.get("TelemetryType").asText(), "Event")) { - int dotIndex = projection.indexOf("."); - if (dotIndex != -1) { - projection = projection.substring(dotIndex + 1); + + try (JsonReader jsonReader = JsonProviders.createReader(responseBody)) { + jsonReader.nextToken(); + while (jsonReader.nextToken() != JsonToken.END_OBJECT) { + if ("Metrics".equals(jsonReader.getFieldName())) { + jsonReader.nextToken(); + + while (jsonReader.nextToken() != JsonToken.END_ARRAY) { + DerivedMetricInfo metric = new DerivedMetricInfo(); + + while (jsonReader.nextToken() != JsonToken.END_OBJECT) { + + String fieldName = jsonReader.getFieldName(); + jsonReader.nextToken(); + + switch (fieldName) { + case "Id": + metric.setId(jsonReader.getString()); + break; + case "Aggregation": + metric.setAggregation(jsonReader.getString()); + break; + case "TelemetryType": + metric.setTelemetryType(jsonReader.getString()); + break; + case "Projection": + metric.setProjection(jsonReader.getString()); + break; + default: + jsonReader.skipChildren(); + break; + } + } + requestedMetrics.computeIfAbsent(metric.getTelemetryType(), k -> new ArrayList<>()).add(metric); } - } - metric.setProjection(projection); - requestedMetrics.put(projection, metric); + } else { + jsonReader.skipChildren(); + } } } - return requestedMetrics; } catch (Exception e) { logger.verbose("Failed to parse metrics from response: %s", e.getMessage()); } - return new ConcurrentHashMap(); + return requestedMetrics; } public synchronized void reset() { this.setEtag(null); - this.metrics.clear(); + this.derivedMetrics.clear(); } - class OpenTelMetricInfo { + public class DerivedMetricInfo { private String id; private String projection; private String telemetryType; @@ -128,7 +127,6 @@ public String getTelemetryType() { return this.telemetryType; } - public void setProjection(String projection) { this.projection = projection; } @@ -142,5 +140,3 @@ public void setAggregation(String aggregation) { } } } - - diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 84319daba5c8b..517fc5884cbf4 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -3,18 +3,8 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; -import com.azure.monitor.opentelemetry.exporter.implementation.models.ContextTagKeys; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; -import com.azure.monitor.opentelemetry.exporter.implementation.models.RemoteDependencyData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.RequestData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.StackFrame; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryExceptionData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryExceptionDetails; -import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDependencyDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseExceptionDocument; -import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseRequestDocument; +import com.azure.monitor.opentelemetry.exporter.implementation.models.*; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.*; import com.azure.monitor.opentelemetry.exporter.implementation.utils.CpuPerformanceCounterCalculator; import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; @@ -33,7 +23,6 @@ import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; - final class QuickPulseDataCollector { private static final MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); @@ -44,20 +33,19 @@ final class QuickPulseDataCollector { private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator = getCpuPerformanceCounterCalculator(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); - private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator = - getCpuPerformanceCounterCalculator(); private final boolean useNormalizedValueForNonNormalizedCpuPercentage; private volatile QuickPulseStatus quickPulseStatus = QuickPulseStatus.QP_IS_OFF; private volatile Supplier instrumentationKeySupplier; - QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage) { + QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseConfiguration quickPulseConfiguration) { this.useNormalizedValueForNonNormalizedCpuPercentage = useNormalizedValueForNonNormalizedCpuPercentage; + this.quickPulseConfiguration = quickPulseConfiguration; } private static CpuPerformanceCounterCalculator getCpuPerformanceCounterCalculator() { @@ -103,8 +91,7 @@ synchronized FinalCounters peek() { return null; } - - void addOtelMetric(TelemetryItem telemetryItem){ + void addOtelMetric(TelemetryItem telemetryItem) { if (!isEnabled()) { // quick pulse is not enabled or quick pulse data sender is not enabled return; @@ -121,7 +108,8 @@ void addOtelMetric(TelemetryItem telemetryItem){ } int itemCount = sampleRate == null ? 1 : Math.round(100 / sampleRate); - if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { + if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), + "opentelemetry")) { MonitorDomain data2 = telemetryItem.getData().getBaseData(); MetricsData metricsData = (MetricsData) data2; MetricDataPoint point = metricsData.getMetrics().get(0); @@ -350,6 +338,8 @@ public ArrayList retrieveOpenTelMetrics() { return metricsStorage.processMetrics(); } + public ConcurrentHashMap + class FinalCounters { final int exceptions; @@ -452,7 +442,7 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { } class OpenTelMetricsStorage { - private ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + private ConcurrentHashMap metrics = new ConcurrentHashMap(); public void addMetric(String metricName, double value) { OpenTelMetric metric = metrics.get(metricName); @@ -466,7 +456,8 @@ public void addMetric(String metricName, double value) { } public ArrayList processMetrics() { - ConcurrentHashMap requestedMetrics = quickPulseConfiguration.getMetrics(); + ConcurrentHashMap requestedMetrics + = quickPulseConfiguration.getMetrics(); ArrayList processedMetrics = new ArrayList<>(); Iterator> iterator = this.metrics.entrySet().iterator(); @@ -483,8 +474,7 @@ public ArrayList processMetrics() { if (ChronoUnit.SECONDS.between(value.getLastTimestamp(), LocalDateTime.now()) > 5) { iterator.remove(); - } - else { + } else { value.clearDataPoints(); } @@ -493,7 +483,8 @@ public ArrayList processMetrics() { } - public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfiguration.OpenTelMetricInfo metricInfo) { + public QuickPulseMetrics processMetric(OpenTelMetric metric, + QuickPulseConfiguration.DerivedMetricInfo metricInfo) { if (metric.getDataPoints().isEmpty()) { return new QuickPulseMetrics(metricInfo.getId(), 0, 1); @@ -506,15 +497,19 @@ public QuickPulseMetrics processMetric( OpenTelMetric metric, QuickPulseConfigur case "Sum": double sum = dataValues.stream().mapToDouble(Double::doubleValue).sum(); return new QuickPulseMetrics(metricInfo.getId(), sum, 1); + case "Avg": double avg = dataValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), avg, dataValues.size()); + case "Min": double min = dataValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), min, 1); + case "Max": double max = dataValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), max, 1); + default: throw new IllegalArgumentException("Aggregation type not supported: " + aggregation); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index 0a01e17b8afb1..ecdfd4947da6d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -75,11 +75,8 @@ public void prepareQuickPulseDataForSend(String redirectedEndpoint) { Date currentDate = new Date(); String endpointPrefix = Strings.isNullOrEmpty(redirectedEndpoint) ? getQuickPulseEndpoint() : redirectedEndpoint; - HttpRequest request = networkHelper.buildRequest(currentDate, this.getEndpointUrl(endpointPrefix)); - String endpointPrefix = - Strings.isNullOrEmpty(redirectedEndpoint) ? getQuickPulseEndpoint() : redirectedEndpoint; - HttpRequest request = - networkHelper.buildRequest(currentDate, this.getEndpointUrl(endpointPrefix), quickPulseConfiguration.getEtag()); + HttpRequest request = networkHelper.buildRequest(currentDate, this.getEndpointUrl(endpointPrefix), + quickPulseConfiguration.getEtag()); request.setBody(buildPostEntity(counters)); if (!sendQueue.offer(request)) { @@ -134,8 +131,7 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) t } private static List addMetricsToQuickPulseEnvelope( - QuickPulseDataCollector.FinalCounters counters, - List openTelemetryMetrics) { + QuickPulseDataCollector.FinalCounters counters, ArrayList openTelemetryMetrics) { List metricsList = new ArrayList<>(); metricsList.add(new QuickPulseMetrics("\\ApplicationInsights\\Requests/Sec", counters.requests, 1)); if (counters.requests != 0) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index f3ba31904bdf8..70d60775b2f0c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -8,10 +8,7 @@ import com.azure.core.http.HttpResponse; import com.azure.core.util.Context; import com.azure.core.util.logging.ClientLogger; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.HashSet; + import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -66,7 +63,8 @@ public void run() { this.quickPulseHeaderInfo = quickPulseHeaderInfo; String etagValue = networkHelper.getEtagHeaderValue(response); if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + ConcurrentHashMap otelMetrics + = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } break; diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index f77f2cd26c440..9df9cdc33c9b6 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -5,6 +5,7 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpHeader; import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; @@ -13,7 +14,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.Date; final class QuickPulseNetworkHelper { @@ -30,7 +30,8 @@ final class QuickPulseNetworkHelper { private static final HttpHeaderName QPS_STREAM_ID = HttpHeaderName.fromString("x-ms-qps-stream-id"); private static final HttpHeaderName QPS_INSTANCE_NAME = HttpHeaderName.fromString("x-ms-qps-instance-name"); private static final HttpHeaderName QPS_INVARIANT_VERSION = HttpHeaderName.fromString("x-ms-qps-invariant-version"); - private static final HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); + private static final HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER + = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); private static final ObjectMapper objectMapper = new ObjectMapper(); HttpRequest buildPingRequest(Date currentDate, String address, String quickPulseId, String machineName, @@ -50,7 +51,7 @@ HttpRequest buildRequest(Date currentDate, String address, String etag) { HttpRequest request = new HttpRequest(HttpMethod.POST, address); request.setHeader(HEADER_TRANSMISSION_TIME, String.valueOf(ticks)); - request.setHeader(QPS_CONFIGURATION_ETAG_HEADER_NAME, etag); + request.setHeader(QPS_CONFIGURATION_ETAG_HEADER, etag); return request; } @@ -83,7 +84,7 @@ QuickPulseHeaderInfo getQuickPulseHeaderInfo(HttpResponse response) { String getEtagHeaderValue(HttpResponse response) { HttpHeaders headers = response.getHeaders(); - HttpHeader etagHeader = headers.get(QPS_CONFIGURATION_ETAG_HEADER_NAME); + HttpHeader etagHeader = headers.get(QPS_CONFIGURATION_ETAG_HEADER); return etagHeader != null ? etagHeader.getValue() : null; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 31180668e1f1d..5a44d8b2dcb65 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -16,9 +16,7 @@ import java.io.IOException; import java.net.URL; -import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -100,7 +98,8 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { lastValidTransmission = sendTime; String etagValue = networkHelper.getEtagHeaderValue(response); if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics = quickPulseConfiguration.parseMetrics(response); + ConcurrentHashMap otelMetrics + = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } operationLogger.recordSuccess(); @@ -118,7 +117,6 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { } finally { if (response != null) { - // need to consume the body or close the response, otherwise get netty ByteBuf leak // warnings: // io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java index 6c70bc5d36323..adc9a3578c8b3 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java @@ -21,6 +21,4 @@ public void setValue(double value) { this.value = value; } - - } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index e6644e47cf8e1..b52f1fdb4db8e 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -11,10 +11,16 @@ import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -85,25 +91,16 @@ void successfulPingReturnsWithEtagHeader() { headers.put("x-ms-qps-configuration-etag", "1::randometag::2::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); - HttpPipeline httpPipeline = - new HttpPipelineBuilder() - .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) - .tracer(new NoopTracer()) - .build(); - QuickPulsePingSender quickPulsePingSender = - new QuickPulsePingSender( - httpPipeline, - connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, - null, - "instance1", - "machine1", - "qpid123", - "testSdkVersion"); + HttpPipeline httpPipeline = new HttpPipelineBuilder() + .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) + .tracer(new NoopTracer()) + .build(); + QuickPulsePingSender quickPulsePingSender + = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); - assertThat("https://new.endpoint.com") - .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("1::randometag::2::"); QuickPulseConfiguration.getInstance().reset(); } @@ -147,39 +144,35 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { } byte[] bodyBytes = jsonBody.getBytes(StandardCharsets.UTF_8); - HttpPipeline httpPipeline = - new HttpPipelineBuilder() - .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders, bodyBytes))) - .tracer(new NoopTracer()) - .build(); - QuickPulsePingSender quickPulsePingSender = - new QuickPulsePingSender( - httpPipeline, - connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, - null, - "instance1", - "machine1", - "qpid123", - "testSdkVersion"); + HttpPipeline httpPipeline = new HttpPipelineBuilder() + .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders, bodyBytes))) + .tracer(new NoopTracer()) + .build(); + QuickPulsePingSender quickPulsePingSender + = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); - assertThat("https://new.endpoint.com") - .isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("2::randometag::3::"); assertThat(QuickPulseConfiguration.getInstance().getMetrics().size()).isEqualTo(2); assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getAggregation()) .isEqualTo("Avg"); assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getTelemetryType()) .isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getProjection()).isEqualTo("my_gauge"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getProjection()) + .isEqualTo("my_gauge"); assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); System.out.println(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()).isEqualTo("Sum"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getTelemetryType()).isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getProjection()).isEqualTo("MyFruitCounter"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getId()).isEqualTo("MyFruitCounter"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()) + .isEqualTo("Sum"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getTelemetryType()) + .isEqualTo("Metric"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getProjection()) + .isEqualTo("MyFruitCounter"); + assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getId()) + .isEqualTo("MyFruitCounter"); QuickPulseConfiguration.getInstance().reset(); } From 640a32ece9aed028bd053bde8a32f478922b38e5 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Tue, 13 Aug 2024 12:08:12 -0700 Subject: [PATCH 19/22] adapted to use azure-json --- .../implementation/MetricDataMapper.java | 13 +- .../implementation/quickpulse/QuickPulse.java | 23 ++-- .../quickpulse/QuickPulseConfiguration.java | 130 ++++++++++++++++-- .../quickpulse/QuickPulseDataCollector.java | 120 ++++++++++------ .../quickpulse/QuickPulseDataFetcher.java | 13 +- .../quickpulse/QuickPulseDataSender.java | 14 +- .../quickpulse/QuickPulseMetricReader.java | 41 ++++++ .../quickpulse/QuickPulseMetricReceiver.java | 70 ++++++++++ .../quickpulse/QuickPulseNetworkHelper.java | 5 - .../quickpulse/QuickPulsePingSender.java | 16 +-- .../quickpulse/model/OTelDataPoint.java | 30 ++++ .../quickpulse/model/OTelMetric.java | 37 +++++ .../quickpulse/model/OpenTelDataPoint.java | 24 ---- .../quickpulse/model/OpenTelMetric.java | 52 ------- .../quickpulse/QuickPulseCoordinatorTest.java | 13 +- .../QuickPulseDataCollectorTests.java | 64 ++++++++- .../QuickPulseDataFetcherTests.java | 28 ++-- .../QuickPulseIntegrationTests.java | 40 +++--- .../QuickPulseNetworkHelperTest.java | 7 +- .../quickpulse/QuickPulsePingSenderTests.java | 110 +++++++-------- 20 files changed, 571 insertions(+), 279 deletions(-) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java delete mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java delete mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java index 5cfdbd6c9d700..5f5705337eb3a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/MetricDataMapper.java @@ -72,17 +72,9 @@ public MetricDataMapper(BiConsumer telemetry this.captureHttpServer4xxAsError = captureHttpServer4xxAsError; } - public void mapMetrics(MetricData metricData, Consumer breezeConsumer, - Consumer quickPulseConsumer) { + public void mapMetrics(MetricData metricData, Consumer consumer) { MetricDataType type = metricData.getType(); if (type == DOUBLE_SUM || type == DOUBLE_GAUGE || type == LONG_SUM || type == LONG_GAUGE || type == HISTOGRAM) { - boolean isPreAggregatedStandardMetric - = OTEL_PRE_AGGREGATED_STANDARD_METRIC_NAMES.contains(metricData.getName()); - if (isPreAggregatedStandardMetric) { - List preAggregatedStandardMetrics - = convertOtelMetricToAzureMonitorMetric(metricData, true); - preAggregatedStandardMetrics.forEach(breezeConsumer::accept); - } // DO NOT emit unstable metrics from the OpenTelemetry auto instrumentation libraries // custom metrics are always emitted @@ -91,8 +83,7 @@ public void mapMetrics(MetricData metricData, Consumer breezeCons return; } List stableOtelMetrics = convertOtelMetricToAzureMonitorMetric(metricData, false); - stableOtelMetrics.forEach(quickPulseConsumer::accept); - stableOtelMetrics.forEach(breezeConsumer::accept); + stableOtelMetrics.forEach(consumer::accept); } else { logger.warning("metric data type {} is not supported yet.", metricData.getType()); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java index f58cfee9c6b8a..89eaa93fa5182 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulse.java @@ -5,18 +5,14 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpRequest; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; -import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorDomain; +import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; import com.azure.monitor.opentelemetry.exporter.implementation.utils.HostName; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; import com.azure.monitor.opentelemetry.exporter.implementation.utils.ThreadPoolUtils; -import io.opentelemetry.api.common.AttributeKey; import reactor.util.annotation.Nullable; import java.net.URL; -import java.util.Objects; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; @@ -33,7 +29,7 @@ public class QuickPulse { public static QuickPulse create(HttpPipeline httpPipeline, Supplier endpointUrl, Supplier instrumentationKey, @Nullable String roleName, @Nullable String roleInstance, boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseMetricReader quickPulseMetricReader, - String sdkVersion) { + MetricDataMapper metricDataMapper, String sdkVersion) { QuickPulse quickPulse = new QuickPulse(); @@ -48,7 +44,7 @@ public static QuickPulse create(HttpPipeline httpPipeline, Supplier endpoin Thread.currentThread().interrupt(); } quickPulse.initialize(httpPipeline, endpointUrl, instrumentationKey, roleName, roleInstance, - useNormalizedValueForNonNormalizedCpuPercentage, quickPulseMetricReader, sdkVersion); + useNormalizedValueForNonNormalizedCpuPercentage, quickPulseMetricReader, metricDataMapper, sdkVersion); }); // the condition below will always be false, but by referencing the executor it ensures the // executor can't become unreachable in the middle of the execute() method execution above @@ -74,13 +70,14 @@ public void add(TelemetryItem telemetryItem) { private void initialize(HttpPipeline httpPipeline, Supplier endpointUrl, Supplier instrumentationKey, @Nullable String roleName, @Nullable String roleInstance, boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseMetricReader quickPulseMetricReader, - String sdkVersion) { + MetricDataMapper metricDataMapper, String sdkVersion) { String quickPulseId = UUID.randomUUID().toString().replace("-", ""); ArrayBlockingQueue sendQueue = new ArrayBlockingQueue<>(256, true); QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); - QuickPulseDataSender quickPulseDataSender = new QuickPulseDataSender(httpPipeline, sendQueue); + QuickPulseDataSender quickPulseDataSender + = new QuickPulseDataSender(httpPipeline, sendQueue, quickPulseConfiguration); String instanceName = roleInstance; String machineName = HostName.get(); @@ -109,6 +106,14 @@ private void initialize(HttpPipeline httpPipeline, Supplier endpointUrl, Su QuickPulseCoordinator coordinator = new QuickPulseCoordinator(coordinatorInitData); + QuickPulseMetricReceiver quickPulseMetricReceiver + = new QuickPulseMetricReceiver(quickPulseMetricReader, metricDataMapper, collector); + + Thread metricReceiverThread + = new Thread(quickPulseMetricReceiver, QuickPulseMetricReceiver.class.getSimpleName()); + metricReceiverThread.setDaemon(true); + metricReceiverThread.start(); + Thread senderThread = new Thread(quickPulseDataSender, QuickPulseDataSender.class.getSimpleName()); senderThread.setDaemon(true); senderThread.start(); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java index 2f2533f1c6dbc..7c5ab87006371 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseConfiguration.java @@ -31,7 +31,8 @@ public synchronized void setDerivedMetrics(ConcurrentHashMap> otelMetrics) { + public synchronized void updateConfig(String etagValue, + ConcurrentHashMap> otelMetrics) { if (!Objects.equals(this.getEtag(), etagValue)) { this.setEtag(etagValue); this.setDerivedMetrics(otelMetrics); @@ -39,14 +40,15 @@ public synchronized void updateConfig(String etagValue, ConcurrentHashMap> parseDerivedMetrics(HttpResponse response) throws IOException { + public ConcurrentHashMap> parseDerivedMetrics(HttpResponse response) + throws IOException { ConcurrentHashMap> requestedMetrics = new ConcurrentHashMap<>(); try { String responseBody = response.getBodyAsString().block(); if (responseBody == null || responseBody.isEmpty()) { - return new ConcurrentHashMap>(); + return new ConcurrentHashMap>(); } try (JsonReader jsonReader = JsonProviders.createReader(responseBody)) { @@ -67,21 +69,86 @@ public ConcurrentHashMap> parseDerivedMetri case "Id": metric.setId(jsonReader.getString()); break; + case "Aggregation": metric.setAggregation(jsonReader.getString()); break; + case "TelemetryType": metric.setTelemetryType(jsonReader.getString()); break; + case "Projection": metric.setProjection(jsonReader.getString()); break; + + case "FilterGroups": + // Handle "FilterGroups" field + if (jsonReader.currentToken() == JsonToken.START_ARRAY) { + while (jsonReader.nextToken() != JsonToken.END_ARRAY) { + if (jsonReader.currentToken() == JsonToken.START_OBJECT) { + while (jsonReader.nextToken() != JsonToken.END_OBJECT) { + if (jsonReader.currentToken() == JsonToken.FIELD_NAME + && jsonReader.getFieldName().equals("Filters")) { + jsonReader.nextToken(); + if (jsonReader.currentToken() == JsonToken.START_ARRAY) { + while (jsonReader.nextToken() != JsonToken.END_ARRAY) { + if (jsonReader.currentToken() + == JsonToken.START_OBJECT) { + String innerFieldName = ""; + String predicate = ""; + String comparand = ""; + + while (jsonReader.nextToken() + != JsonToken.END_OBJECT) { + String filterFieldName + = jsonReader.getFieldName(); + jsonReader.nextToken(); + + switch (filterFieldName) { + case "FieldName": + innerFieldName + = jsonReader.getString(); + if (innerFieldName.contains(".")) { + innerFieldName = innerFieldName + .split("\\.")[1]; + } + break; + + case "Predicate": + predicate = jsonReader.getString(); + break; + + case "Comparand": + comparand = jsonReader.getString(); + break; + } + } + + if (!innerFieldName.isEmpty() + && !innerFieldName.equals("undefined") + && !predicate.isEmpty() + && !comparand.isEmpty()) { + metric.addFilterGroup(innerFieldName, + predicate, comparand); + } + } + } + } + } + } + } + } + } + break; + default: jsonReader.skipChildren(); break; } } - requestedMetrics.computeIfAbsent(metric.getTelemetryType(), k -> new ArrayList<>()).add(metric); + requestedMetrics.computeIfAbsent(metric.getTelemetryType(), k -> new ArrayList<>()) + .add(metric); } } else { jsonReader.skipChildren(); @@ -89,16 +156,11 @@ public ConcurrentHashMap> parseDerivedMetri } } } + return requestedMetrics; } catch (Exception e) { logger.verbose("Failed to parse metrics from response: %s", e.getMessage()); } - - return requestedMetrics; - } - - public synchronized void reset() { - this.setEtag(null); - this.derivedMetrics.clear(); + return new ConcurrentHashMap>(); } public class DerivedMetricInfo { @@ -106,6 +168,7 @@ public class DerivedMetricInfo { private String projection; private String telemetryType; private String aggregation; + private ArrayList filterGroups = new ArrayList(); public String getId() { return this.id; @@ -138,5 +201,50 @@ public String getAggregation() { public void setAggregation(String aggregation) { this.aggregation = aggregation; } + + public ArrayList getFilterGroups() { + return this.filterGroups; + } + + public void addFilterGroup(String fieldName, String predicate, String comparand) { + this.filterGroups.add(new FilterGroup(fieldName, predicate, comparand)); + } + } + + class FilterGroup { + private String fieldName; + private String operator; + private String comparand; + + public FilterGroup(String fieldName, String predicate, String comparand) { + this.setFieldName(fieldName); + this.setOperator(predicate); + this.setComparand(comparand); + } + + public String getFieldName() { + return this.fieldName; + } + + private void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String getOperator() { + return this.operator; + } + + private void setOperator(String operator) { + this.operator = operator; + } + + public String getComparand() { + return this.comparand; + } + + public void setComparand(String comparand) { + this.comparand = comparand; + } } + } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 517fc5884cbf4..8990118a38ebc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -20,8 +20,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; final class QuickPulseDataCollector { @@ -30,12 +28,13 @@ final class QuickPulseDataCollector { private static final OperatingSystemMXBean operatingSystemMxBean = ManagementFactory.getOperatingSystemMXBean(); private final AtomicReference counters = new AtomicReference<>(null); + private final CpuPerformanceCounterCalculator cpuPerformanceCounterCalculator = getCpuPerformanceCounterCalculator(); private QuickPulseConfiguration quickPulseConfiguration; - private OpenTelMetricsStorage metricsStorage = new OpenTelMetricsStorage(); + private OtelMetricsStorage metricsStorage = new OtelMetricsStorage(); private final boolean useNormalizedValueForNonNormalizedCpuPercentage; @@ -43,7 +42,8 @@ final class QuickPulseDataCollector { private volatile Supplier instrumentationKeySupplier; - QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage, QuickPulseConfiguration quickPulseConfiguration) { + QuickPulseDataCollector(boolean useNormalizedValueForNonNormalizedCpuPercentage, + QuickPulseConfiguration quickPulseConfiguration) { this.useNormalizedValueForNonNormalizedCpuPercentage = useNormalizedValueForNonNormalizedCpuPercentage; this.quickPulseConfiguration = quickPulseConfiguration; } @@ -54,6 +54,7 @@ private static CpuPerformanceCounterCalculator getCpuPerformanceCounterCalculato synchronized void disable() { counters.set(null); + metricsStorage.clearMetrics(); quickPulseStatus = QuickPulseStatus.QP_IS_OFF; } @@ -106,14 +107,13 @@ void addOtelMetric(TelemetryItem telemetryItem) { // sampleRate should never be zero (how could it be captured if sampling set to zero percent?) return; } - int itemCount = sampleRate == null ? 1 : Math.round(100 / sampleRate); if (Objects.equals(telemetryItem.getResource().getAttribute(AttributeKey.stringKey("telemetry.sdk.name")), "opentelemetry")) { - MonitorDomain data2 = telemetryItem.getData().getBaseData(); - MetricsData metricsData = (MetricsData) data2; + MonitorDomain data = telemetryItem.getData().getBaseData(); + MetricsData metricsData = (MetricsData) data; MetricDataPoint point = metricsData.getMetrics().get(0); - this.metricsStorage.addMetric(point.getName(), point.getValue()); + this.metricsStorage.addMetric(point.getName(), point.getValue(), metricsData.getProperties()); } } @@ -334,11 +334,17 @@ private static int charToInt(char c) { return x; } - public ArrayList retrieveOpenTelMetrics() { + public ArrayList retrieveOtelMetrics() { return metricsStorage.processMetrics(); } - public ConcurrentHashMap + public ConcurrentHashMap getOtelMetrics() { + return metricsStorage.getMetrics(); + } + + public void flushOtelMetrics() { + metricsStorage.clearMetrics(); + } class FinalCounters { @@ -441,49 +447,46 @@ static CountAndDuration decodeCountAndDuration(long countAndDuration) { } } - class OpenTelMetricsStorage { - private ConcurrentHashMap metrics = new ConcurrentHashMap(); + class OtelMetricsStorage { + private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); + // setting most amount of OTel metrics a user can stream to this. will review if users surpass threshold. + private final int maxMetricsLimit = 50; - public void addMetric(String metricName, double value) { - OpenTelMetric metric = metrics.get(metricName); + public void addMetric(String metricName, double value, Map dimensions) { + OTelMetric metric = metrics.get(metricName); if (metric == null) { - metric = new OpenTelMetric(metricName); - metric.addDataPoint(value); - metrics.putIfAbsent(metricName, metric); + // create new metric and add to metrics map + // (if we have reached the limit, we will block the metric from being created + if (metrics.size() <= maxMetricsLimit) { + metric = new OTelMetric(metricName); + metric.addDataPoint(value, dimensions == null ? new HashMap<>() : new HashMap<>(dimensions)); + metrics.putIfAbsent(metricName, metric); + } } else { - metric.addDataPoint(value); + metric.addDataPoint(value, dimensions == null ? new HashMap<>() : new HashMap<>(dimensions)); } } public ArrayList processMetrics() { - ConcurrentHashMap requestedMetrics - = quickPulseConfiguration.getMetrics(); + ConcurrentHashMap> requestedMetrics + = quickPulseConfiguration.getDerivedMetrics(); ArrayList processedMetrics = new ArrayList<>(); - Iterator> iterator = this.metrics.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String key = entry.getKey(); - OpenTelMetric value = entry.getValue(); - - if (requestedMetrics.containsKey(key)) { - QuickPulseMetrics processedMetric = processMetric(value, requestedMetrics.get(key)); - processedMetrics.add(processedMetric); - } - - if (ChronoUnit.SECONDS.between(value.getLastTimestamp(), LocalDateTime.now()) > 5) { - iterator.remove(); - - } else { - value.clearDataPoints(); + if (requestedMetrics.get("Metric") != null) { + for (QuickPulseConfiguration.DerivedMetricInfo metricInfo : requestedMetrics.get("Metric")) { + if (this.getMetrics().get(metricInfo.getProjection()) != null) { + QuickPulseMetrics processedMetric + = processMetric(this.getMetrics().get(metricInfo.getProjection()), metricInfo); + processedMetrics.add(processedMetric); + } } - } - return processedMetrics; + this.clearMetrics(); + return processedMetrics; } - public QuickPulseMetrics processMetric(OpenTelMetric metric, + public QuickPulseMetrics processMetric(OTelMetric metric, QuickPulseConfiguration.DerivedMetricInfo metricInfo) { if (metric.getDataPoints().isEmpty()) { @@ -491,29 +494,56 @@ public QuickPulseMetrics processMetric(OpenTelMetric metric, } String aggregation = metricInfo.getAggregation(); - ArrayList dataValues = metric.getDataValues(); + ArrayList filteredValues = new ArrayList<>(); + for (OTelDataPoint dataPoint : metric.getDataPoints()) { + boolean passedFilter = true; + for (QuickPulseConfiguration.FilterGroup filterGroup : metricInfo.getFilterGroups()) { + String fieldName = filterGroup.getFieldName(); + String operator = filterGroup.getOperator(); + String comparand = filterGroup.getComparand(); + if (Objects.equals(operator, "Equal")) { + String fieldValue = dataPoint.getDimensions().get(fieldName); + if (fieldValue == null || !fieldValue.equals(comparand)) { + passedFilter = false; + break; + // Handle the case where the value is null or does not equal the operator + } + } + } + if (passedFilter) { + filteredValues.add(dataPoint.getValue()); + } + } switch (aggregation) { case "Sum": - double sum = dataValues.stream().mapToDouble(Double::doubleValue).sum(); + double sum = filteredValues.stream().mapToDouble(Double::doubleValue).sum(); return new QuickPulseMetrics(metricInfo.getId(), sum, 1); case "Avg": - double avg = dataValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); - return new QuickPulseMetrics(metricInfo.getId(), avg, dataValues.size()); + double avg = filteredValues.stream().mapToDouble(Double::doubleValue).average().orElse(0); + return new QuickPulseMetrics(metricInfo.getId(), avg, filteredValues.size()); case "Min": - double min = dataValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); + double min = filteredValues.stream().mapToDouble(Double::doubleValue).min().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), min, 1); case "Max": - double max = dataValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); + double max = filteredValues.stream().mapToDouble(Double::doubleValue).max().orElse(0); return new QuickPulseMetrics(metricInfo.getId(), max, 1); default: throw new IllegalArgumentException("Aggregation type not supported: " + aggregation); } + } + + public void clearMetrics() { + this.metrics.clear(); + } + //for testing + public ConcurrentHashMap getMetrics() { + return metrics; } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index ecdfd4947da6d..6f39597972b2f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -28,7 +28,7 @@ class QuickPulseDataFetcher { private final ArrayBlockingQueue sendQueue; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private final Supplier endpointUrl; private final Supplier instrumentationKey; @@ -41,7 +41,7 @@ class QuickPulseDataFetcher { public QuickPulseDataFetcher(QuickPulseDataCollector collector, ArrayBlockingQueue sendQueue, Supplier endpointUrl, Supplier instrumentationKey, String roleName, String instanceName, - String machineName, String quickPulseId) { + String machineName, String quickPulseId, QuickPulseConfiguration quickPulseConfiguration) { this.collector = collector; this.sendQueue = sendQueue; this.endpointUrl = endpointUrl; @@ -50,6 +50,7 @@ public QuickPulseDataFetcher(QuickPulseDataCollector collector, ArrayBlockingQue this.instanceName = instanceName; this.machineName = machineName; this.quickPulseId = quickPulseId; + this.quickPulseConfiguration = quickPulseConfiguration; sdkVersion = getCurrentSdkVersion(); } @@ -123,15 +124,15 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) t postEnvelope.setStreamId(quickPulseId); postEnvelope.setVersion(sdkVersion); postEnvelope.setTimeStamp("/Date(" + System.currentTimeMillis() + ")/"); - postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters, collector.retrieveOpenTelMetrics())); + postEnvelope.setMetrics(addMetricsToQuickPulseEnvelope(counters, collector.retrieveOtelMetrics())); envelopes.add(postEnvelope); - // By default '/' is not escaped in JSON, so we need to escape it manually as the backend requires it. + //By default, '/' is not escaped in JSON, so we need to escape it manually as the backend requires it. return postEnvelope.toJsonString().replace("/", "\\/"); } private static List addMetricsToQuickPulseEnvelope( - QuickPulseDataCollector.FinalCounters counters, ArrayList openTelemetryMetrics) { + QuickPulseDataCollector.FinalCounters counters, ArrayList otelMetrics) { List metricsList = new ArrayList<>(); metricsList.add(new QuickPulseMetrics("\\ApplicationInsights\\Requests/Sec", counters.requests, 1)); if (counters.requests != 0) { @@ -155,7 +156,7 @@ private static List addMetricsToQuickPulseEnvelope( metricsList.add(new QuickPulseMetrics("\\Memory\\Committed Bytes", counters.memoryCommitted, 1)); metricsList.add(new QuickPulseMetrics("\\Processor(_Total)\\% Processor Time", counters.cpuUsage, 1)); - metricsList.addAll(openTelemetryMetrics); + metricsList.addAll(otelMetrics); return metricsList; } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java index 70d60775b2f0c..02c661bc5d5b1 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataSender.java @@ -9,6 +9,7 @@ import com.azure.core.util.Context; import com.azure.core.util.logging.ClientLogger; +import java.util.ArrayList; import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -18,16 +19,18 @@ class QuickPulseDataSender implements Runnable { private static final ClientLogger logger = new ClientLogger(QuickPulseCoordinator.class); private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private final HttpPipeline httpPipeline; private volatile QuickPulseHeaderInfo quickPulseHeaderInfo; private long lastValidTransmission = 0; private final ArrayBlockingQueue sendQueue; - QuickPulseDataSender(HttpPipeline httpPipeline, ArrayBlockingQueue sendQueue) { + QuickPulseDataSender(HttpPipeline httpPipeline, ArrayBlockingQueue sendQueue, + QuickPulseConfiguration quickPulseConfiguration) { this.httpPipeline = httpPipeline; this.sendQueue = sendQueue; + this.quickPulseConfiguration = quickPulseConfiguration; } @Override @@ -63,7 +66,7 @@ public void run() { this.quickPulseHeaderInfo = quickPulseHeaderInfo; String etagValue = networkHelper.getEtagHeaderValue(response); if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } @@ -78,11 +81,6 @@ public void run() { } catch (Throwable t) { logger.error("QuickPulseDataSender failed to send a request", t); } - /* Debugging purposes - System.out.println("POST*********************"); - System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); - System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); - */ } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java new file mode 100644 index 0000000000000..b8131ef179644 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReader.java @@ -0,0 +1,41 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.CollectionRegistration; +import io.opentelemetry.sdk.metrics.export.MetricReader; + +import java.util.Collection; + +public class QuickPulseMetricReader implements MetricReader { + + private volatile CollectionRegistration collectionRegistration = CollectionRegistration.noop(); + + @Override + public void register(CollectionRegistration registration) { + // this should get (once) called when the OpenTelemetry SDK is created + collectionRegistration = registration; + } + + @Override + public CompletableResultCode forceFlush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + // this will be called on-demand from Quick Pulse code + Collection collectAllMetrics() { + return collectionRegistration.collectAllMetrics(); + } + + @Override + public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) { + return AggregationTemporality.DELTA; + } +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java new file mode 100644 index 0000000000000..7d199f337b701 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -0,0 +1,70 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; + +import com.azure.monitor.opentelemetry.exporter.implementation.MetricDataMapper; +import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger; +import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; +import io.opentelemetry.sdk.metrics.data.MetricData; + +import java.util.Collection; +import java.util.function.Consumer; + +import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR; + +public class QuickPulseMetricReceiver implements Runnable { + + private static QuickPulseHeaderInfo quickPulseHeaderInfo; + private QuickPulseMetricReader quickPulseMetricReader; + private QuickPulseDataCollector collector; + private static final OperationLogger metricReceiverLogger + = new OperationLogger(QuickPulseMetricReceiver.class, "Exporting metric"); + private final MetricDataMapper mapper; + private final Consumer quickPulseConsumer; + + public QuickPulseMetricReceiver(QuickPulseMetricReader quickPulseMetricReader, MetricDataMapper metricDataMapper, + QuickPulseDataCollector collector) { + this.quickPulseMetricReader = quickPulseMetricReader; + this.mapper = metricDataMapper; + this.collector = collector; + this.quickPulseConsumer = telemetryItem -> { + if (this.collector.isEnabled()) { + this.collector.addOtelMetric(telemetryItem); + } + }; + } + + public static synchronized QuickPulseHeaderInfo getQuickPulseHeaderInfo() { + return quickPulseHeaderInfo; + } + + public static synchronized void setQuickPulseHeaderInfo(QuickPulseHeaderInfo info) { + quickPulseHeaderInfo = info; + } + + @Override + public void run() { + while (true) { + + Collection metrics = quickPulseMetricReader.collectAllMetrics(); + QuickPulseHeaderInfo headerInfo = getQuickPulseHeaderInfo(); + + if (headerInfo == null || headerInfo.getQuickPulseStatus() != QuickPulseStatus.QP_IS_ON) { + continue; + } + + for (MetricData metricData : metrics) { + try { + mapper.mapMetrics(metricData, quickPulseConsumer); + metricReceiverLogger.recordSuccess(); + } catch (Throwable t) { + metricReceiverLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR); + } + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java index 9df9cdc33c9b6..f41b7006541a2 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelper.java @@ -10,10 +10,6 @@ import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.Date; final class QuickPulseNetworkHelper { @@ -32,7 +28,6 @@ final class QuickPulseNetworkHelper { private static final HttpHeaderName QPS_INVARIANT_VERSION = HttpHeaderName.fromString("x-ms-qps-invariant-version"); private static final HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); - private static final ObjectMapper objectMapper = new ObjectMapper(); HttpRequest buildPingRequest(Date currentDate, String address, String quickPulseId, String machineName, String roleName, String instanceName) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index 5a44d8b2dcb65..fc1a034824e72 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.Date; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -37,7 +38,7 @@ class QuickPulsePingSender { private final HttpPipeline httpPipeline; private final QuickPulseNetworkHelper networkHelper = new QuickPulseNetworkHelper(); - private QuickPulseConfiguration quickPulseConfiguration = QuickPulseConfiguration.getInstance(); + private QuickPulseConfiguration quickPulseConfiguration; private volatile QuickPulseEnvelope pingEnvelope; // cached for performance private final Supplier endpointUrl; @@ -50,7 +51,8 @@ class QuickPulsePingSender { private final String sdkVersion; QuickPulsePingSender(HttpPipeline httpPipeline, Supplier endpointUrl, Supplier instrumentationKey, - String roleName, String instanceName, String machineName, String quickPulseId, String sdkVersion) { + String roleName, String instanceName, String machineName, String quickPulseId, String sdkVersion, + QuickPulseConfiguration quickPulseConfiguration) { this.httpPipeline = httpPipeline; this.endpointUrl = endpointUrl; this.instrumentationKey = instrumentationKey; @@ -59,14 +61,10 @@ class QuickPulsePingSender { this.machineName = machineName; this.quickPulseId = quickPulseId; this.sdkVersion = sdkVersion; + this.quickPulseConfiguration = quickPulseConfiguration; } QuickPulseHeaderInfo ping(String redirectedEndpoint) { - /* Debugging purposes - System.out.println("PING*********************"); - System.out.println("ETAG: " + quickPulseConfiguration.getEtag()); - System.out.println("METRICS: " + quickPulseConfiguration.getMetrics()); - */ String instrumentationKey = getInstrumentationKey(); if (Strings.isNullOrEmpty(instrumentationKey)) { // Quick Pulse Ping uri will be null when the instrumentation key is null. When that happens, @@ -97,8 +95,8 @@ QuickPulseHeaderInfo ping(String redirectedEndpoint) { case QP_IS_ON: lastValidTransmission = sendTime; String etagValue = networkHelper.getEtagHeaderValue(response); - if (!Objects.equals(etagValue, quickPulseConfiguration.getEtag())) { - ConcurrentHashMap otelMetrics + if (etagValue != null) { + ConcurrentHashMap> otelMetrics = quickPulseConfiguration.parseDerivedMetrics(response); quickPulseConfiguration.updateConfig(etagValue, otelMetrics); } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java new file mode 100644 index 0000000000000..24fbecc708bbf --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelDataPoint.java @@ -0,0 +1,30 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; + +import java.util.HashMap; + +public class OTelDataPoint { + private double value; + private HashMap dimensions; + + public OTelDataPoint(double value, HashMap dimensions) { + this.value = value; + this.dimensions = dimensions; + } + + public double getValue() { + return value; + } + + public void setValue(double value) { + this.value = value; + } + + public HashMap getDimensions() { + return dimensions; + } + + public void setDimensions(HashMap dimensions) { + this.dimensions = dimensions; + } + +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java new file mode 100644 index 0000000000000..d51a1b3db9da3 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OTelMetric.java @@ -0,0 +1,37 @@ +package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; + +import java.util.ArrayList; +import java.util.HashMap; + +public class OTelMetric { + + private String name; + private ArrayList dataPoints; + + public OTelMetric(String name) { + this.name = name; + this.dataPoints = new ArrayList<>(); + } + + public void addDataPoint(double value, HashMap dimensions) { + OTelDataPoint dataPoint = new OTelDataPoint(value, dimensions); + this.dataPoints.add(dataPoint); + } + + public String getName() { + return name; + } + + public ArrayList getDataPoints() { + return dataPoints; + } + + public ArrayList getDataValues() { + ArrayList values = new ArrayList<>(); + for (OTelDataPoint dataPoint : dataPoints) { + values.add(dataPoint.getValue()); + } + return values; + } + +} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java deleted file mode 100644 index adc9a3578c8b3..0000000000000 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelDataPoint.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.HashMap; - -public class OpenTelDataPoint { - private double value; - //private HashMap dimensions; - - public OpenTelDataPoint(double value) { - this.value = value; - //this.dimensions = new HashMap<>(); - } - - public double getValue() { - return value; - } - - public void setValue(double value) { - this.value = value; - } - -} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java deleted file mode 100644 index 32307b0e35369..0000000000000 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/model/OpenTelMetric.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model; - -import java.util.ArrayList; -import java.time.LocalDateTime; - -public class OpenTelMetric { - - private String name; - private ArrayList dataPoints; - private LocalDateTime lastTimestamp; - - public OpenTelMetric(String name) { - this.name = name; - this.dataPoints = new ArrayList<>(); - this.lastTimestamp = LocalDateTime.now(); - } - - public void addDataPoint(double value) { - OpenTelDataPoint dataPoint = new OpenTelDataPoint(value); - this.dataPoints.add(dataPoint); - this.updateLastTimestamp(); - } - - public String getName() { - return name; - } - - public ArrayList getDataPoints() { - return dataPoints; - } - - public ArrayList getDataValues() { - ArrayList values = new ArrayList<>(); - for (OpenTelDataPoint dataPoint : dataPoints) { - values.add(dataPoint.getValue()); - } - return values; - } - - public LocalDateTime getLastTimestamp() { - return lastTimestamp; - } - - public void updateLastTimestamp() { - this.lastTimestamp = LocalDateTime.now(); - } - - public void clearDataPoints() { - this.dataPoints.clear(); - } - -} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java index 101bde0c6f985..d3d997b40dd6d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinatorTest.java @@ -19,7 +19,8 @@ void testOnlyPings() throws InterruptedException { QuickPulseDataFetcher mockFetcher = mock(QuickPulseDataFetcher.class); QuickPulseDataSender mockSender = mock(QuickPulseDataSender.class); QuickPulsePingSender mockPingSender = mock(QuickPulsePingSender.class); - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); Mockito.doReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)).when(mockPingSender).ping(null); QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher) @@ -49,6 +50,7 @@ void testOnlyPings() throws InterruptedException { Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); // make sure QP_IS_OFF after ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } @Test @@ -63,8 +65,8 @@ void testOnePingAndThenOnePost() throws InterruptedException { Mockito.when(mockPingSender.ping(null)) .thenReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_ON), new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_OFF)); - - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher) .withDataSender(mockSender) .withPingSender(mockPingSender) @@ -92,6 +94,7 @@ void testOnePingAndThenOnePost() throws InterruptedException { Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); // Make sure QP_IS_OFF after one post and ping assertThat(collector.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_OFF); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } @Disabled("sporadically failing on CI") @@ -100,6 +103,7 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { QuickPulseDataFetcher mockFetcher = Mockito.mock(QuickPulseDataFetcher.class); QuickPulseDataSender mockSender = Mockito.mock(QuickPulseDataSender.class); QuickPulsePingSender mockPingSender = Mockito.mock(QuickPulsePingSender.class); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); Mockito.doNothing().when(mockFetcher).prepareQuickPulseDataForSend(notNull()); Mockito.doReturn(new QuickPulseHeaderInfo(QuickPulseStatus.QP_IS_ON, "https://new.endpoint.com", 100)) @@ -112,7 +116,7 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { QuickPulseCoordinatorInitData initData = new QuickPulseCoordinatorInitDataBuilder().withDataFetcher(mockFetcher) .withDataSender(mockSender) .withPingSender(mockPingSender) - .withCollector(new QuickPulseDataCollector(true)) + .withCollector(new QuickPulseDataCollector(true, quickPulseConfiguration)) .withWaitBetweenPingsInMillis(10L) .withWaitBetweenPostsInMillis(10L) .withWaitOnErrorInMillis(10L) @@ -131,5 +135,6 @@ void testOnePingAndThenOnePostWithRedirectedLink() throws InterruptedException { Mockito.verify(mockFetcher, Mockito.atLeast(1)).prepareQuickPulseDataForSend("https://new.endpoint.com"); Mockito.verify(mockPingSender, Mockito.atLeast(1)).ping(null); Mockito.verify(mockPingSender, Mockito.times(2)).ping("https://new.endpoint.com"); + assertThat(quickPulseConfiguration.getEtag()).isNull(); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java index c9d5bdf7905c5..86325d57e8eb8 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollectorTests.java @@ -5,11 +5,20 @@ import com.azure.monitor.opentelemetry.exporter.implementation.builders.ExceptionTelemetryBuilder; import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricDataPoint; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MetricsData; +import com.azure.monitor.opentelemetry.exporter.implementation.models.MonitorBase; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.OTelMetric; +import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.model.QuickPulseMetrics; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.resources.Resource; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.util.ArrayList; import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; import static com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulseTestBase.createRemoteDependencyTelemetry; import static com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulseTestBase.createRequestTelemetry; @@ -23,30 +32,33 @@ class QuickPulseDataCollectorTests { @Test void initialStateIsDisabled() { - assertThat(new QuickPulseDataCollector(true).peek()).isNull(); + assertThat(new QuickPulseDataCollector(true, new QuickPulseConfiguration()).peek()).isNull(); } @Test void emptyCountsAndDurationsAfterEnable() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); QuickPulseDataCollector.FinalCounters counters = collector.peek(); assertCountersReset(counters); + ArrayList storedMetrics = collector.retrieveOtelMetrics(); + assertThat(storedMetrics).isEmpty(); } @Test void nullCountersAfterDisable() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); collector.disable(); assertThat(collector.peek()).isNull(); + assertThat(collector.retrieveOtelMetrics()).isEmpty(); } @Test void requestTelemetryIsCounted_DurationIsSum() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -88,7 +100,7 @@ void requestTelemetryIsCounted_DurationIsSum() { @Test void dependencyTelemetryIsCounted_DurationIsSum() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -130,7 +142,7 @@ void dependencyTelemetryIsCounted_DurationIsSum() { @Test void exceptionTelemetryIsCounted() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); @@ -150,6 +162,44 @@ void exceptionTelemetryIsCounted() { assertCountersReset(collector.peek()); } + @Test + void openTelemetryMetricsAreCounted() { + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); + + collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); + collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); + + TelemetryItem telemetry = new TelemetryItem(); + telemetry.setConnectionString(FAKE_CONNECTION_STRING); + MonitorBase data = new MonitorBase(); + MetricDataPoint point = new MetricDataPoint(); + point.setName("TestMetric"); + point.setValue(123.456); + ArrayList metricsList = new ArrayList<>(); + metricsList.add(point); + data.setBaseData(new MetricsData().setMetrics(metricsList)); + telemetry.setData(data); + Attributes attributes = Attributes.builder().put("telemetry.sdk.name", "opentelemetry").build(); + Resource resource = Resource.create(attributes); + telemetry.setResource(resource); + collector.addOtelMetric(telemetry); + ConcurrentHashMap storedMetrics = collector.getOtelMetrics(); + assertThat(storedMetrics.size()).isEqualTo(1); + assertThat(storedMetrics.containsKey("TestMetric")).isTrue(); + assertThat(storedMetrics.get("TestMetric").getDataValues().get(0)).isEqualTo(123.456); + + point.setName("TestMetric2"); + point.setValue(789.012); + collector.addOtelMetric(telemetry); + storedMetrics = collector.getOtelMetrics(); + assertThat(storedMetrics.size()).isEqualTo(2); + assertThat(storedMetrics.containsKey("TestMetric2")).isTrue(); + assertThat(storedMetrics.get("TestMetric2").getDataValues().get(0)).isEqualTo(789.012); + + collector.flushOtelMetrics(); + assertThat(collector.getOtelMetrics().size()).isEqualTo(0); + } + @Test void encodeDecodeIsIdentity() { long count = 456L; @@ -204,7 +254,7 @@ private static void assertCountersReset(QuickPulseDataCollector.FinalCounters co @Test void checkDocumentsListSize() { - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, new QuickPulseConfiguration()); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(FAKE_CONNECTION_STRING::getInstrumentationKey); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java index 71d9c61834e7a..6c45d54044c18 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcherTests.java @@ -24,8 +24,10 @@ class QuickPulseDataFetcherTests { @Test void testGetCurrentSdkVersion() { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); - QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null, - connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher( + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, null, null, null, null, quickPulseConfiguration); String sdkVersion = dataFetcher.getCurrentSdkVersion(); assertThat(sdkVersion).isNotNull(); assertThat(sdkVersion).isNotEqualTo("java:unknown"); @@ -34,8 +36,10 @@ void testGetCurrentSdkVersion() { @Test void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); - QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null, - connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, null, null, null, null, quickPulseConfiguration); String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint(); String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -47,8 +51,10 @@ void endpointIsFormattedCorrectlyWhenUsingConfig() throws URISyntaxException { @Test void endpointIsFormattedCorrectlyWhenConfigIsNull() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); - QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher(new QuickPulseDataCollector(true), null, - connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, null, null, null); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); + QuickPulseDataFetcher quickPulseDataFetcher = new QuickPulseDataFetcher( + new QuickPulseDataCollector(true, quickPulseConfiguration), null, connectionString::getLiveEndpoint, + connectionString::getInstrumentationKey, null, null, null, null, quickPulseConfiguration); String quickPulseEndpoint = quickPulseDataFetcher.getQuickPulseEndpoint(); String endpointUrl = quickPulseDataFetcher.getEndpointUrl(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -63,19 +69,21 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "0::randometag::2::"); + headers.put("x-ms-qps-configuration-etag", ""); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); HttpPipeline httpPipeline = new HttpPipelineBuilder() .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) .tracer(new NoopTracer()) .build(); - QuickPulsePingSender quickPulsePingSender - = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(httpPipeline, + connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, "instance1", "machine1", + "qpid123", "testSdkVersion", quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo(""); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java index 073fd544dd2d4..eb4daa1e70797 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseIntegrationTests.java @@ -25,25 +25,30 @@ public class QuickPulseIntegrationTests extends QuickPulseTestBase { private static final ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=ikey123"); private static final String instrumentationKey = "ikey123"; - private QuickPulsePingSender getQuickPulsePingSender() { + private QuickPulsePingSender getQuickPulsePingSender(QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender(getHttpPipeline(), connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion", + quickPulseConfiguration); } - private QuickPulsePingSender getQuickPulsePingSenderWithAuthentication() { + private QuickPulsePingSender + getQuickPulsePingSenderWithAuthentication(QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender(getHttpPipelineWithAuthentication(), connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion", + quickPulseConfiguration); } - private QuickPulsePingSender getQuickPulsePingSenderWithValidator(HttpPipelinePolicy validator) { + private QuickPulsePingSender getQuickPulsePingSenderWithValidator(HttpPipelinePolicy validator, + QuickPulseConfiguration quickPulseConfiguration) { return new QuickPulsePingSender(getHttpPipeline(validator), connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion", + quickPulseConfiguration); } @Disabled @Test public void testPing() { - QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSender(); + QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSender(new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); } @@ -51,7 +56,8 @@ public void testPing() { @Disabled @Test public void testPingWithAuthentication() { - QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithAuthentication(); + QuickPulsePingSender quickPulsePingSender + = getQuickPulsePingSenderWithAuthentication(new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); } @@ -62,8 +68,8 @@ public void testPingRequestBody() throws InterruptedException { CountDownLatch pingCountDown = new CountDownLatch(1); String expectedRequestBody = "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; - QuickPulsePingSender quickPulsePingSender - = getQuickPulsePingSenderWithValidator(new ValidationPolicy(pingCountDown, expectedRequestBody)); + QuickPulsePingSender quickPulsePingSender = getQuickPulsePingSenderWithValidator( + new ValidationPolicy(pingCountDown, expectedRequestBody), new QuickPulseConfiguration()); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(quickPulseHeaderInfo.getQuickPulseStatus()).isEqualTo(QuickPulseStatus.QP_IS_ON); assertTrue(pingCountDown.await(60, TimeUnit.SECONDS)); @@ -76,21 +82,23 @@ public void testPostRequest() throws InterruptedException { CountDownLatch pingCountDown = new CountDownLatch(1); CountDownLatch postCountDown = new CountDownLatch(1); Date currDate = new Date(); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); String expectedPingRequestBody = "\\{\"Documents\":null,\"InstrumentationKey\":null,\"Metrics\":null,\"InvariantVersion\":1,\"Timestamp\":\"\\\\/Date\\(\\d+\\)\\\\/\",\"Version\":\"testSdkVersion\",\"StreamId\":\"qpid123\",\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}"; String expectedPostRequestBody = "\\[\\{\"Documents\":\\[\\{\"__type\":\"RequestTelemetryDocument\",\"DocumentType\":\"Request\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"request-test\",\"Success\":true,\"Duration\":\"PT.*S\",\"ResponseCode\":\"200\",\"OperationName\":null,\"Url\":\"foo\"\\},\\{\"__type\":\"DependencyTelemetryDocument\",\"DocumentType\":\"RemoteDependency\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":\\{\"customProperty\":\"customValue\"\\},\"Name\":\"dep-test\",\"Target\":null,\"Success\":true,\"Duration\":\"PT.*S\",\"ResultCode\":null,\"CommandName\":\"dep-test-cmd\",\"DependencyTypeName\":null,\"OperationName\":null\\},\\{\"__type\":\"ExceptionTelemetryDocument\",\"DocumentType\":\"Exception\",\"Version\":\"1.0\",\"OperationId\":null,\"Properties\":null,\"Exception\":\"\",\"ExceptionMessage\":\"test\",\"ExceptionType\":\"java.lang.Exception\"\\}\\],\"InstrumentationKey\":\"" + instrumentationKey + "\",\"Metrics\":\\[\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Request Duration\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests Failed\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Requests Succeeded\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Call Duration\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls Failed\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Dependency Calls Succeeded\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\ApplicationInsights\\\\\\\\Exceptions\\\\\\/Sec\",\"Value\":[0-9.]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\Memory\\\\\\\\Committed Bytes\",\"Value\":[0-9.E]+,\"Weight\":\\d+\\},\\{\"Name\":\"\\\\\\\\Processor\\(_Total\\)\\\\\\\\% Processor Time\",\"Value\":-?[0-9.]+,\"Weight\":\\d+\\}\\],\"InvariantVersion\":1,\"Timestamp\":\"\\\\\\/Date\\(\\d+\\)\\\\\\/\",\"Version\":\"[^\"]*\",\"StreamId\":null,\"MachineName\":\"machine1\",\"Instance\":\"instance1\",\"RoleName\":null\\}\\]"; - QuickPulsePingSender pingSender - = getQuickPulsePingSenderWithValidator(new ValidationPolicy(pingCountDown, expectedPingRequestBody)); + QuickPulsePingSender pingSender = getQuickPulsePingSenderWithValidator( + new ValidationPolicy(pingCountDown, expectedPingRequestBody), quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = pingSender.ping(null); - QuickPulseDataSender dataSender = new QuickPulseDataSender( - getHttpPipeline(new ValidationPolicy(postCountDown, expectedPostRequestBody)), sendQueue); - QuickPulseDataCollector collector = new QuickPulseDataCollector(true); + QuickPulseDataSender dataSender + = new QuickPulseDataSender(getHttpPipeline(new ValidationPolicy(postCountDown, expectedPostRequestBody)), + sendQueue, quickPulseConfiguration); + QuickPulseDataCollector collector = new QuickPulseDataCollector(true, quickPulseConfiguration); QuickPulseDataFetcher dataFetcher = new QuickPulseDataFetcher(collector, sendQueue, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", null); + connectionString::getInstrumentationKey, null, "instance1", "machine1", null, quickPulseConfiguration); collector.setQuickPulseStatus(QuickPulseStatus.QP_IS_ON); collector.enable(connectionString::getInstrumentationKey); diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java index f788cb2ecf445..ae5d6e293d2fe 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseNetworkHelperTest.java @@ -3,17 +3,14 @@ package com.azure.monitor.opentelemetry.exporter.implementation.quickpulse; -import com.azure.core.http.HttpHeader; import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; import org.junit.jupiter.api.Test; -import org.mockito.Mock; import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; class QuickPulseNetworkHelperTest { @Test @@ -39,9 +36,9 @@ void testGetEtagHeader() { HttpResponse response = mock(HttpResponse.class); HttpHeaders headers = new HttpHeaders(); HttpHeaderName QPS_CONFIGURATION_ETAG_HEADER_NAME = HttpHeaderName.fromString("x-ms-qps-configuration-etag"); - headers.add(QPS_CONFIGURATION_ETAG_HEADER_NAME, "0::randometag::2::"); + headers.add(QPS_CONFIGURATION_ETAG_HEADER_NAME, "0::randometag::1::"); Mockito.doReturn(headers).when(response).getHeaders(); String result = new QuickPulseNetworkHelper().getEtagHeaderValue(response); - assertThat(result).isEqualTo("0::randometag::2::"); + assertThat(result).isEqualTo("0::randometag::1::"); } } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java index b52f1fdb4db8e..9085435421243 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/test/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSenderTests.java @@ -6,15 +6,20 @@ import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; +import com.azure.json.JsonProviders; +import com.azure.json.JsonWriter; import com.azure.monitor.opentelemetry.exporter.implementation.MockHttpResponse; import com.azure.monitor.opentelemetry.exporter.implementation.NoopTracer; import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import java.io.IOException; +import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -30,8 +35,9 @@ class QuickPulsePingSenderTests { @Test void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=testing-123"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(null, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, null, null, null, null); + connectionString::getInstrumentationKey, null, null, null, null, null, quickPulseConfiguration); String quickPulseEndpoint = quickPulsePingSender.getQuickPulseEndpoint(); String endpointUrl = quickPulsePingSender.getQuickPulsePingUri(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -39,14 +45,14 @@ void endpointIsFormattedCorrectlyWhenUsingConnectionString() throws URISyntaxExc assertThat(endpointUrl).endsWith("/ping?ikey=testing-123"); assertThat(endpointUrl) .isEqualTo("https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=testing-123"); - QuickPulseConfiguration.getInstance().reset(); } @Test void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxException { ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=A-test-instrumentation-key"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(null, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, null, null, null, null); + connectionString::getInstrumentationKey, null, null, null, null, null, quickPulseConfiguration); String quickPulseEndpoint = quickPulsePingSender.getQuickPulseEndpoint(); String endpointUrl = quickPulsePingSender.getQuickPulsePingUri(quickPulseEndpoint); URI uri = new URI(endpointUrl); @@ -54,7 +60,6 @@ void endpointIsFormattedCorrectlyWhenUsingInstrumentationKey() throws URISyntaxE assertThat(endpointUrl).endsWith("/ping?ikey=A-test-instrumentation-key"); // from resources/ApplicationInsights.xml assertThat(endpointUrl).isEqualTo( "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey=A-test-instrumentation-key"); - QuickPulseConfiguration.getInstance().reset(); } @Test @@ -66,47 +71,23 @@ void endpointChangesWithRedirectHeaderAndGetNewPingInterval() { headers.put("x-ms-qps-configuration-etag", "0::randometag::1::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); HttpPipeline httpPipeline = new HttpPipelineBuilder() .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) .tracer(new NoopTracer()) .build(); - QuickPulsePingSender quickPulsePingSender - = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(httpPipeline, + connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, "instance1", "machine1", + "qpid123", "testSdkVersion", quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat(1000).isEqualTo(quickPulseHeaderInfo.getQpsServicePollingInterval()); assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - QuickPulseConfiguration.getInstance().reset(); - - } - - @Test - void successfulPingReturnsWithEtagHeader() { - System.out.println("Etag: " + QuickPulseConfiguration.getInstance().getEtag()); - Map headers = new HashMap<>(); - headers.put("x-ms-qps-service-polling-interval-hint", "1000"); - headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); - headers.put("x-ms-qps-subscribed", "true"); - headers.put("x-ms-qps-configuration-etag", "1::randometag::2::"); - HttpHeaders httpHeaders = new HttpHeaders(headers); - ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); - HttpPipeline httpPipeline = new HttpPipelineBuilder() - .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders))) - .tracer(new NoopTracer()) - .build(); - QuickPulsePingSender quickPulsePingSender - = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); - QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); - assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); - assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("1::randometag::2::"); - QuickPulseConfiguration.getInstance().reset(); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("0::randometag::1::"); } @Test - void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { + void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() throws IOException { Map headers = new HashMap<>(); headers.put("x-ms-qps-service-polling-interval-hint", "1000"); headers.put("x-ms-qps-service-endpoint-redirect-v2", "https://new.endpoint.com"); @@ -114,6 +95,7 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { headers.put("x-ms-qps-configuration-etag", "2::randometag::3::"); HttpHeaders httpHeaders = new HttpHeaders(headers); ConnectionString connectionString = ConnectionString.parse("InstrumentationKey=fake-ikey"); + QuickPulseConfiguration quickPulseConfiguration = new QuickPulseConfiguration(); List> metrics = new ArrayList<>(); Map metric1 = new HashMap<>(); @@ -122,7 +104,19 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { metric1.put("TelemetryType", "Metric"); metric1.put("Projection", "my_gauge"); metric1.put("BackendAggregation", "Min"); - metric1.put("FilterGroups", new ArrayList<>()); + + ArrayList>>> filterGroups = new ArrayList<>(); + HashMap>> filterGroup = new HashMap<>(); + ArrayList> filters = new ArrayList<>(); + HashMap filterOne = new HashMap<>(); + filterOne.put("FieldName", "Test"); + filterOne.put("Predicate", "Equals"); + filterOne.put("Comparand", "Value"); + filters.add(filterOne); + filterGroup.put("Filters", filters); + filterGroups.add(filterGroup); + metric1.put("FilterGroups", filterGroups); + Map metric2 = new HashMap<>(); metric2.put("Id", "MyFruitCounter"); metric2.put("Aggregation", "Sum"); @@ -134,9 +128,13 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { metrics.add(metric2); Map metricsMap = new HashMap<>(); + metricsMap.put("DocumentStreams", null); + metricsMap.put("ETag", "2::randometag::3::"); metricsMap.put("Metrics", metrics); + metricsMap.put("QuotaInfo", null); ObjectMapper objectMapper = new ObjectMapper(); String jsonBody; + try { jsonBody = objectMapper.writeValueAsString(metricsMap); } catch (com.fasterxml.jackson.core.JsonProcessingException e) { @@ -148,32 +146,30 @@ void successfulPingReturnsWithEtagHeaderAndRequestedMetrics() { .httpClient(request -> Mono.just(new MockHttpResponse(request, 200, httpHeaders, bodyBytes))) .tracer(new NoopTracer()) .build(); - QuickPulsePingSender quickPulsePingSender - = new QuickPulsePingSender(httpPipeline, connectionString::getLiveEndpoint, - connectionString::getInstrumentationKey, null, "instance1", "machine1", "qpid123", "testSdkVersion"); + QuickPulsePingSender quickPulsePingSender = new QuickPulsePingSender(httpPipeline, + connectionString::getLiveEndpoint, connectionString::getInstrumentationKey, null, "instance1", "machine1", + "qpid123", "testSdkVersion", quickPulseConfiguration); QuickPulseHeaderInfo quickPulseHeaderInfo = quickPulsePingSender.ping(null); assertThat(QuickPulseStatus.QP_IS_ON).isEqualTo(quickPulseHeaderInfo.getQuickPulseStatus()); assertThat("https://new.endpoint.com").isEqualTo(quickPulseHeaderInfo.getQpsServiceEndpointRedirect()); - assertThat(QuickPulseConfiguration.getInstance().getEtag()).isEqualTo("2::randometag::3::"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().size()).isEqualTo(2); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getAggregation()) - .isEqualTo("Avg"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getTelemetryType()) - .isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getProjection()) - .isEqualTo("my_gauge"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("my_gauge").getId()).isEqualTo("my_gauge"); - System.out.println(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getAggregation()) - .isEqualTo("Sum"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getTelemetryType()) - .isEqualTo("Metric"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getProjection()) - .isEqualTo("MyFruitCounter"); - assertThat(QuickPulseConfiguration.getInstance().getMetrics().get("MyFruitCounter").getId()) - .isEqualTo("MyFruitCounter"); - QuickPulseConfiguration.getInstance().reset(); + assertThat(quickPulseConfiguration.getEtag()).isEqualTo("2::randometag::3::"); + assertThat(quickPulseConfiguration.getDerivedMetrics().size()).isEqualTo(1); + assertThat(quickPulseConfiguration.getDerivedMetrics().get("Metric").size()).isEqualTo(2); + ArrayList metricCategory + = quickPulseConfiguration.getDerivedMetrics().get("Metric"); + assertThat(metricCategory.get(0).getAggregation()).isEqualTo("Avg"); + assertThat(metricCategory.get(0).getTelemetryType()).isEqualTo("Metric"); + assertThat(metricCategory.get(0).getFilterGroups().size() == 1); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getFieldName()).isEqualTo("Test"); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getOperator()).isEqualTo("Equals"); + assertThat(metricCategory.get(0).getFilterGroups().get(0).getComparand()).isEqualTo("Value"); + assertThat(metricCategory.get(0).getProjection()).isEqualTo("my_gauge"); + assertThat(metricCategory.get(0).getId()).isEqualTo("my_gauge"); + assertThat(metricCategory.get(1).getAggregation()).isEqualTo("Sum"); + assertThat(metricCategory.get(1).getTelemetryType()).isEqualTo("Metric"); + assertThat(metricCategory.get(1).getProjection()).isEqualTo("MyFruitCounter"); + assertThat(metricCategory.get(1).getId()).isEqualTo("MyFruitCounter"); } } From f78b58cc3ef9082ee1e6be5f341dee9935a53fec Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Tue, 13 Aug 2024 16:37:59 -0700 Subject: [PATCH 20/22] removed unused import statements --- .../exporter/implementation/quickpulse/QuickPulsePingSender.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index fc1a034824e72..d6696e3a34346 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -18,7 +18,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Date; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; From aed5a9ce8df46d4334315a726699eca2ac475113 Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Tue, 13 Aug 2024 16:40:28 -0700 Subject: [PATCH 21/22] removed unused imports --- .../exporter/implementation/quickpulse/QuickPulsePingSender.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java index fc1a034824e72..d6696e3a34346 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulsePingSender.java @@ -18,7 +18,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Date; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; From e48e6ff7c1d630f58e8376a69c0ca3f2e6f1cd4a Mon Sep 17 00:00:00 2001 From: Naveen Sukumar Date: Thu, 15 Aug 2024 17:42:25 -0700 Subject: [PATCH 22/22] fixed metric receiver --- .../implementation/quickpulse/QuickPulseCoordinator.java | 2 ++ .../implementation/quickpulse/QuickPulseDataCollector.java | 1 - .../implementation/quickpulse/QuickPulseDataFetcher.java | 2 +- .../implementation/quickpulse/QuickPulseMetricReceiver.java | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java index fa597fc564db9..96e3022758817 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseCoordinator.java @@ -80,11 +80,13 @@ private long sendData() { case QP_IS_OFF: pingMode = true; collector.flushOtelMetrics(); + QuickPulseMetricReceiver.setQuickPulseHeaderInfo(currentQuickPulseHeaderInfo); return qpsServicePollingIntervalHintMillis > 0 ? qpsServicePollingIntervalHintMillis : waitBetweenPingsInMillis; case QP_IS_ON: + QuickPulseMetricReceiver.setQuickPulseHeaderInfo(currentQuickPulseHeaderInfo); return waitBetweenPostsInMillis; } diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java index 8990118a38ebc..0c96c7fde141a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataCollector.java @@ -471,7 +471,6 @@ public ArrayList processMetrics() { ConcurrentHashMap> requestedMetrics = quickPulseConfiguration.getDerivedMetrics(); ArrayList processedMetrics = new ArrayList<>(); - if (requestedMetrics.get("Metric") != null) { for (QuickPulseConfiguration.DerivedMetricInfo metricInfo : requestedMetrics.get("Metric")) { if (this.getMetrics().get(metricInfo.getProjection()) != null) { diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java index 6f39597972b2f..a2d7ff4e5f676 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseDataFetcher.java @@ -128,7 +128,7 @@ private String buildPostEntity(QuickPulseDataCollector.FinalCounters counters) t envelopes.add(postEnvelope); //By default, '/' is not escaped in JSON, so we need to escape it manually as the backend requires it. - return postEnvelope.toJsonString().replace("/", "\\/"); + return "[" + postEnvelope.toJsonString().replace("/", "\\/") + "]"; } private static List addMetricsToQuickPulseEnvelope( diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java index 7d199f337b701..51f31bd8d64fd 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/quickpulse/QuickPulseMetricReceiver.java @@ -43,7 +43,6 @@ public static synchronized void setQuickPulseHeaderInfo(QuickPulseHeaderInfo inf @Override public void run() { while (true) { - Collection metrics = quickPulseMetricReader.collectAllMetrics(); QuickPulseHeaderInfo headerInfo = getQuickPulseHeaderInfo();