From e0ebb9e0d8daaf0aea1a97a4e03dcafe972a2fbe Mon Sep 17 00:00:00 2001 From: Rajkumar Rangaraj Date: Fri, 2 Feb 2024 14:38:43 -0800 Subject: [PATCH] [AzureMonitorExporter] Fix Duplicate Timestamps in OTELRESOURCE Metrics (#41761) * Fix Duplicate Timestamps in OTELRESOURCE Metrics * Update changelog * Update tests --- .../CHANGELOG.md | 5 + .../Customizations/Models/TelemetryItem.cs | 5 + .../src/Internals/AzureMonitorResource.cs | 2 +- .../src/Internals/ResourceExtensions.cs | 9 +- .../src/Internals/TraceHelper.cs | 5 +- .../ResourceExtensionsTests.cs | 24 ++-- .../TelemetryItemTests.cs | 117 +++++++++++++++++- 7 files changed, 143 insertions(+), 24 deletions(-) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md index 518fcc4752b00..2ab18da5b4739 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md @@ -8,6 +8,11 @@ ### Bugs Fixed +* Fixed an issue where `_OTELRESOURCE_` metrics were emitted with duplicated + timestamps. This fix ensures accurate and distinct timestamping for all + `_OTELRESOURCE_` metrics. + ([#41761](https://github.com/Azure/azure-sdk-for-net/pull/41761)) + ### Other Changes ## 1.2.0 (2024-01-24) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/Models/TelemetryItem.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/Models/TelemetryItem.cs index 6cf8d0f9beb2f..0d7b2cd67daff 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/Models/TelemetryItem.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/Models/TelemetryItem.cs @@ -112,6 +112,11 @@ public TelemetryItem(DateTime time, AzureMonitorResource? resource, string instr SetResourceSdkVersionAndIkey(resource, instrumentationKey); } + public TelemetryItem(DateTime time, AzureMonitorResource? resource, string instrumentationKey, MonitorBase monitorBaseData) : this(time, resource, instrumentationKey) + { + Data = monitorBaseData; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void SetResourceSdkVersionAndIkey(AzureMonitorResource? resource, string instrumentationKey) { diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorResource.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorResource.cs index 45b62077af281..f1118901ef768 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorResource.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorResource.cs @@ -11,6 +11,6 @@ internal sealed class AzureMonitorResource internal string? RoleInstance { get; set; } - internal TelemetryItem? MetricTelemetry { get; set; } + internal MonitorBase? MonitorBaseData { get; set; } } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/ResourceExtensions.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/ResourceExtensions.cs index 32eae07644939..dbf3c448a0dc9 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/ResourceExtensions.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/ResourceExtensions.cs @@ -135,13 +135,10 @@ internal static class ResourceExtensions if (shouldReportMetricTelemetry && metricsData != null) { - azureMonitorResource.MetricTelemetry = new TelemetryItem(DateTime.UtcNow, azureMonitorResource, instrumentationKey!) + azureMonitorResource.MonitorBaseData = new MonitorBase { - Data = new MonitorBase - { - BaseType = "MetricData", - BaseData = metricsData - } + BaseType = "MetricData", + BaseData = metricsData }; } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs index 024ca61c62335..25a89ac194937 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs @@ -25,9 +25,10 @@ internal static List OtelToAzureMonitorTrace(Batch batc List telemetryItems = new List(); TelemetryItem telemetryItem; - if (batchActivity.Count > 0 && azureMonitorResource?.MetricTelemetry != null) + if (batchActivity.Count > 0 && azureMonitorResource?.MonitorBaseData != null) { - telemetryItems.Add(azureMonitorResource.MetricTelemetry); + var otelResourceMetricTelemetry = new TelemetryItem(DateTime.UtcNow, azureMonitorResource, instrumentationKey!, azureMonitorResource.MonitorBaseData); + telemetryItems.Add(otelResourceMetricTelemetry); } foreach (var activity in batchActivity) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/ResourceExtensionsTests.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/ResourceExtensionsTests.cs index 673990be73edb..7c8b40f1c93b0 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/ResourceExtensionsTests.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/ResourceExtensionsTests.cs @@ -46,11 +46,11 @@ public void DefaultResource(string? instrumentationKey, string envVarValue) Assert.Equal(Dns.GetHostName(), azMonResource?.RoleInstance); if (envVarValue == "true") { - Assert.Equal(instrumentationKey != null, azMonResource?.MetricTelemetry != null); + Assert.Equal(instrumentationKey != null, azMonResource?.MonitorBaseData != null); } else { - Assert.Null(azMonResource?.MetricTelemetry); + Assert.Null(azMonResource?.MonitorBaseData); } } finally @@ -221,10 +221,8 @@ public void SdkPrefixIsNotInResourceMetrics() var resource = ResourceBuilder.CreateDefault().AddAttributes(testAttributes).Build(); var azMonResource = resource.CreateAzureMonitorResource(InstrumentationKey); - Assert.Equal("Metric", azMonResource!.MetricTelemetry!.Name); - - var monitorBase = azMonResource.MetricTelemetry.Data; - var metricsData = monitorBase.BaseData as MetricsData; + var monitorBase = azMonResource?.MonitorBaseData; + var metricsData = monitorBase?.BaseData as MetricsData; var metricDataPoint = metricsData?.Metrics[0]; Assert.Equal("bar", metricsData?.Properties["foo"]); @@ -242,7 +240,7 @@ public void SdkPrefixIsNotInResourceMetrics() [Theory] [InlineData(null)] [InlineData(InstrumentationKey)] - public void MetricTelemetryHasAllResourceAttributes(string? instrumentationKey) + public void MonitorBaseDataHasAllResourceAttributes(string? instrumentationKey) { try { @@ -261,15 +259,13 @@ public void MetricTelemetryHasAllResourceAttributes(string? instrumentationKey) var resource = ResourceBuilder.CreateEmpty().AddAttributes(testAttributes).Build(); var azMonResource = resource.CreateAzureMonitorResource(instrumentationKey); - Assert.Equal(instrumentationKey != null, azMonResource?.MetricTelemetry != null); + Assert.Equal(instrumentationKey != null, azMonResource?.MonitorBaseData != null); if (instrumentationKey != null) { - Assert.Equal("Metric", azMonResource!.MetricTelemetry!.Name); - Assert.Equal(3, azMonResource.MetricTelemetry.Tags.Count); - Assert.NotNull(azMonResource.MetricTelemetry.Data); + Assert.NotNull(azMonResource?.MonitorBaseData); - var monitorBase = azMonResource.MetricTelemetry.Data; + var monitorBase = azMonResource.MonitorBaseData; var metricsData = monitorBase.BaseData as MetricsData; Assert.NotNull(metricsData?.Metrics); @@ -366,11 +362,11 @@ public void MetricTelemetryIsAddedToResourceBasedOnEnvVar(string envVarValue) if (envVarValue == "true") { - Assert.NotNull(azMonResource?.MetricTelemetry); + Assert.NotNull(azMonResource?.MonitorBaseData); } else { - Assert.Null(azMonResource?.MetricTelemetry); + Assert.Null(azMonResource?.MonitorBaseData); } } finally diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/TelemetryItemTests.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/TelemetryItemTests.cs index 8daebf2c7478a..c62eefa719125 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/TelemetryItemTests.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/TelemetryItemTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net; using Azure.Monitor.OpenTelemetry.Exporter.Internals; +using Azure.Monitor.OpenTelemetry.Exporter.Internals.Platform; using Azure.Monitor.OpenTelemetry.Exporter.Models; using OpenTelemetry; using OpenTelemetry.Resources; @@ -271,7 +272,7 @@ public void AiLocationIpIsNotSetByDefault() var telemetryItems = TraceHelper.OtelToAzureMonitorTrace(new Batch(new Activity[] { activity }, 1), null, "instrumentationKey", 1.0f); var telemetryItem = telemetryItems.FirstOrDefault(); - Assert.False(telemetryItem?.Tags.TryGetValue(ContextTagKeys.AiLocationIp.ToString(),out _)); + Assert.False(telemetryItem?.Tags.TryGetValue(ContextTagKeys.AiLocationIp.ToString(), out _)); } [Fact] @@ -408,6 +409,120 @@ public void RequestNameMatchesOperationNameForConsumerSpans() Assert.Equal(requestData.Name, telemetryItem?.Tags[ContextTagKeys.AiOperationName.ToString()]); } + [Fact] + public void OTelResourceMetricTelemetryHasAllResourceAttributes() + { + try + { + Environment.SetEnvironmentVariable(EnvironmentVariableConstants.EXPORT_RESOURCE_METRIC, "true"); + var instrumentationKey = "00000000-0000-0000-0000-000000000000"; + + var testAttributes = new Dictionary + { + {SemanticConventions.AttributeServiceName, "my-service" }, + {SemanticConventions.AttributeServiceNamespace, "my-namespace" }, + {SemanticConventions.AttributeServiceInstance, "my-instance" }, + {SemanticConventions.AttributeK8sDeployment, "my-deployment" }, + {SemanticConventions.AttributeK8sPod, "my-pod" }, + { "foo", "bar" } + }; + + using ActivitySource activitySource = new ActivitySource(ActivitySourceName); + using var activity = activitySource.StartActivity( + ActivityName, + ActivityKind.Server, + null, + startTime: DateTime.UtcNow); + + Assert.NotNull(activity); + + var resource = ResourceBuilder.CreateEmpty().AddAttributes(testAttributes).Build(); + var azMonResource = resource.CreateAzureMonitorResource(instrumentationKey); + + var telemetryItems = TraceHelper.OtelToAzureMonitorTrace(new Batch(new Activity[] { activity }, 1), null, instrumentationKey, 1.0f); + var telemetryItem = telemetryItems.FirstOrDefault(); + + var monitorBase = telemetryItem?.Data; + var metricsData = monitorBase?.BaseData as MetricsData; + + Assert.NotNull(metricsData?.Metrics); + + var metricDataPoint = metricsData?.Metrics[0]; + Assert.Equal("_OTELRESOURCE_", metricDataPoint?.Name); + Assert.Equal(0, metricDataPoint?.Value); + + Assert.Equal(6, metricsData?.Properties.Count); + + Assert.Equal("my-service", metricsData?.Properties[SemanticConventions.AttributeServiceName]); + Assert.Equal("my-namespace", metricsData?.Properties[SemanticConventions.AttributeServiceNamespace]); + Assert.Equal("my-instance", metricsData?.Properties[SemanticConventions.AttributeServiceInstance]); + Assert.Equal("my-deployment", metricsData?.Properties[SemanticConventions.AttributeK8sDeployment]); + Assert.Equal("my-pod", metricsData?.Properties[SemanticConventions.AttributeK8sPod]); + Assert.Equal("bar", metricsData?.Properties["foo"]); + } + catch (Exception) + { + Environment.SetEnvironmentVariable(EnvironmentVariableConstants.EXPORT_RESOURCE_METRIC, null); + } + } + + [Fact] + public void OTelResourceMetricTimesAreDifferent() + { + try + { + Environment.SetEnvironmentVariable(EnvironmentVariableConstants.EXPORT_RESOURCE_METRIC, "true"); + var instrumentationKey = "00000000-0000-0000-0000-000000000000"; + + var testAttributes = new Dictionary + { + { "foo", "bar" } + }; + + using ActivitySource activitySource = new ActivitySource(ActivitySourceName); + using var activity1 = activitySource.StartActivity( + ActivityName, + ActivityKind.Server, + null, + startTime: DateTime.UtcNow); + + Assert.NotNull(activity1); + + var resource = ResourceBuilder.CreateEmpty().AddAttributes(testAttributes).Build(); + var azMonResource = resource.CreateAzureMonitorResource(instrumentationKey); + + var telemetryItems = TraceHelper.OtelToAzureMonitorTrace(new Batch(new Activity[] { activity1 }, 1), null, instrumentationKey, 1.0f); + var telemetryItem1 = telemetryItems.FirstOrDefault(); + + var monitorBase = telemetryItem1?.Data; + var metricsData = monitorBase?.BaseData as MetricsData; + + Assert.NotNull(metricsData?.Metrics); + + var metricDataPoint = metricsData?.Metrics[0]; + Assert.Equal("_OTELRESOURCE_", metricDataPoint?.Name); + Assert.Equal(1, metricsData?.Properties.Count); + Assert.Equal("bar", metricsData?.Properties["foo"]); + + using var activity2 = activitySource.StartActivity( + ActivityName, + ActivityKind.Server, + null, + startTime: DateTime.UtcNow); + + Assert.NotNull(activity2); + + telemetryItems = TraceHelper.OtelToAzureMonitorTrace(new Batch(new Activity[] { activity2 }, 1), null, instrumentationKey, 1.0f); + var telemetryItem2 = telemetryItems.FirstOrDefault(); + + Assert.NotEqual(telemetryItem1?.Time, telemetryItem2?.Time); + } + catch (Exception) + { + Environment.SetEnvironmentVariable(EnvironmentVariableConstants.EXPORT_RESOURCE_METRIC, null); + } + } + /// /// If SERVICE.NAME is not defined, it will fall-back to "unknown_service". /// (https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/resource/semantic_conventions#semantic-attributes-with-sdk-provided-default-value).