Merge branch 'main' into vibankwa/add-otlp-excluded-attributes

.github/workflows/ci-instrumentation-libraries.yml

@@ -23,9 +23,6 @@ jobs:
 
     runs-on: ${{ matrix.os }}
 
-    env:
-      RunningDotNetPack: true
-
     steps:
     - uses: actions/checkout@v4
       with:
@@ -35,10 +32,10 @@ jobs:
       uses: actions/setup-dotnet@v3
 
     - name: Install dependencies
-      run: dotnet restore ./build/InstrumentationLibraries.proj
+      run: dotnet restore ./build/InstrumentationLibraries.proj -p:RunningDotNetPack=true
 
     - name: Build
-      run: dotnet build ./build/InstrumentationLibraries.proj --configuration Release --no-restore
+      run: dotnet build ./build/InstrumentationLibraries.proj --configuration Release --no-restore -p:RunningDotNetPack=true
 
     - name: Test ${{ matrix.version }}
-      run: dotnet test **/bin/Release/${{ matrix.version }}/OpenTelemetry.Instrumentation*.Tests.dll --logger:"console;verbosity=detailed"
+      run: dotnet test **/bin/Release/${{ matrix.version }}/OpenTelemetry.Instrumentation*.Tests.dll --framework ${{ matrix.version }} --configuration Release --no-restore --no-build --logger:"console;verbosity=detailed"

Directory.Packages.props

@@ -34,7 +34,10 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="[3.1.0,)" />
     <PackageVersion Include="Microsoft.Extensions.Options" Version="[3.1.0,)" />
     <PackageVersion Include="OpenTelemetry" Version="$(OTelLatestStableVer)" />
+    <PackageVersion Include="OpenTelemetry.Api" Version="$(OTelLatestStableVer)" />
     <PackageVersion Include="OpenTelemetry.Api.ProviderBuilderExtensions" Version="$(OTelLatestStableVer)" />
+    <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="$(OTelLatestStableVer)" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="$(OTelLatestStableVer)" />
     <PackageVersion Include="OpenTracing" Version="[0.12.1,0.13)" />
 
     <!--

build/InstrumentationLibraries.proj

@@ -4,6 +4,10 @@
   </ItemGroup>
 
   <Target Name="Build">
-    <MSBuild Projects="@(SolutionProjects)" Targets="Restore;Build" ContinueOnError="ErrorAndStop" />
+    <MSBuild Projects="@(SolutionProjects)" Targets="Build" ContinueOnError="ErrorAndStop" />
+  </Target>
+
+  <Target Name="Restore">
+    <MSBuild Projects="@(SolutionProjects)" Targets="Restore" ContinueOnError="ErrorAndStop" />
   </Target>
 </Project>

docs/logs/correlation/correlation.csproj

@@ -1,6 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Console\OpenTelemetry.Exporter.Console.csproj" />
   </ItemGroup>
 </Project>

docs/logs/customizing-the-sdk/customizing-the-sdk.csproj

@@ -1,6 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Console\OpenTelemetry.Exporter.Console.csproj" />
   </ItemGroup>
 </Project>

docs/logs/extending-the-sdk/extending-the-sdk.csproj

@@ -1,6 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
   </ItemGroup>
 </Project>

docs/logs/getting-started-aspnetcore/getting-started-aspnetcore.csproj

@@ -1,6 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Console\OpenTelemetry.Exporter.Console.csproj" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Extensions.Hosting\OpenTelemetry.Extensions.Hosting.csproj" />
   </ItemGroup>

docs/logs/getting-started-console/getting-started-console.csproj

@@ -1,6 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Console\OpenTelemetry.Exporter.Console.csproj" />
   </ItemGroup>
 </Project>

docs/logs/redaction/redaction.csproj

@@ -4,7 +4,6 @@
     <Nullable>disable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Console\OpenTelemetry.Exporter.Console.csproj" />
   </ItemGroup>
 </Project>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md

@@ -33,6 +33,12 @@ attributes will be exported when
   be available via environment variable mentioned above.
   ([#4925](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4925))
 
+* `LogRecord.CategoryName` will now be exported as
+[InstrumentationScope](https://github.com/open-telemetry/opentelemetry-dotnet/blob/3c2bb7c93dd2e697636479a1882f49bb0c4a362e/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/opentelemetry/proto/common/v1/common.proto#L71-L81)
+`name` field under
+[ScopeLogs](https://github.com/open-telemetry/opentelemetry-dotnet/blob/3c2bb7c93dd2e697636479a1882f49bb0c4a362e/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/opentelemetry/proto/logs/v1/logs.proto#L64-L75).
+([#4941](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4941))
+
 ## 1.6.0
 
 Released 2023-Sep-05

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OtlpLogRecordTransformer.cs

@@ -14,6 +14,7 @@
 // limitations under the License.
 // </copyright>
 
+using System.Collections.Concurrent;
 using System.Runtime.CompilerServices;
 using Google.Protobuf;
 using OpenTelemetry.Internal;
@@ -28,6 +29,8 @@ namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 
 internal sealed class OtlpLogRecordTransformer
 {
+    internal static readonly ConcurrentBag<OtlpLogs.ScopeLogs> LogListPool = new();
+
     private readonly SdkLimitOptions sdkLimitOptions;
     private readonly ExperimentalOptions experimentalOptions;
 
@@ -41,6 +44,9 @@ internal OtlpCollector.ExportLogsServiceRequest BuildExportRequest(
         OtlpResource.Resource processResource,
         in Batch<LogRecord> logRecordBatch)
     {
+        // TODO: https://github.com/open-telemetry/opentelemetry-dotnet/issues/4943
+        Dictionary<string, OtlpLogs.ScopeLogs> logsByCategory = new Dictionary<string, OtlpLogs.ScopeLogs>();
+
         var request = new OtlpCollector.ExportLogsServiceRequest();
 
         var resourceLogs = new OtlpLogs.ResourceLogs
@@ -49,21 +55,64 @@ internal OtlpCollector.ExportLogsServiceRequest BuildExportRequest(
         };
         request.ResourceLogs.Add(resourceLogs);
 
-        var scopeLogs = new OtlpLogs.ScopeLogs();
-        resourceLogs.ScopeLogs.Add(scopeLogs);
-
         foreach (var logRecord in logRecordBatch)
         {
             var otlpLogRecord = this.ToOtlpLog(logRecord);
             if (otlpLogRecord != null)
             {
+                if (!logsByCategory.TryGetValue(logRecord.CategoryName, out var scopeLogs))
+                {
+                    scopeLogs = this.GetLogListFromPool(logRecord.CategoryName);
+                    logsByCategory.Add(logRecord.CategoryName, scopeLogs);
+                    resourceLogs.ScopeLogs.Add(scopeLogs);
+                }
+
                 scopeLogs.LogRecords.Add(otlpLogRecord);
             }
         }
 
         return request;
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal void Return(OtlpCollector.ExportLogsServiceRequest request)
+    {
+        var resourceLogs = request.ResourceLogs.FirstOrDefault();
+        if (resourceLogs == null)
+        {
+            return;
+        }
+
+        foreach (var scope in resourceLogs.ScopeLogs)
+        {
+            scope.LogRecords.Clear();
+            LogListPool.Add(scope);
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal OtlpLogs.ScopeLogs GetLogListFromPool(string name)
+    {
+        if (!LogListPool.TryTake(out var logs))
+        {
+            logs = new OtlpLogs.ScopeLogs
+            {
+                Scope = new OtlpCommon.InstrumentationScope
+                {
+                    Name = name, // Name is enforced to not be null, but it can be empty.
+                    Version = string.Empty, // proto requires this to be non-null.
+                },
+            };
+        }
+        else
+        {
+            logs.Scope.Name = name;
+            logs.Scope.Version = string.Empty;
+        }
+
+        return logs;
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     internal OtlpLogs.LogRecord ToOtlpLog(LogRecord logRecord)
     {

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs

@@ -90,9 +90,11 @@ public override ExportResult Export(in Batch<LogRecord> logRecordBatch)
         // Prevents the exporter's gRPC and HTTP operations from being instrumented.
         using var scope = SuppressInstrumentationScope.Begin();
 
+        OtlpCollector.ExportLogsServiceRequest request = null;
+
         try
         {
-            var request = this.otlpLogRecordTransformer.BuildExportRequest(this.ProcessResource, logRecordBatch);
+            request = this.otlpLogRecordTransformer.BuildExportRequest(this.ProcessResource, logRecordBatch);
 
             if (!this.exportClient.SendExportRequest(request))
             {
@@ -104,6 +106,13 @@ public override ExportResult Export(in Batch<LogRecord> logRecordBatch)
             OpenTelemetryProtocolExporterEventSource.Log.ExportMethodException(ex);
             return ExportResult.Failure;
         }
+        finally
+        {
+            if (request != null)
+            {
+                this.otlpLogRecordTransformer.Return(request);
+            }
+        }
 
         return ExportResult.Success;
     }

src/OpenTelemetry.Instrumentation.AspNetCore/OpenTelemetry.Instrumentation.AspNetCore.csproj

@@ -20,7 +20,7 @@
     <Compile Include="$(RepoRoot)\src\Shared\Shims\NullableAttributes.cs" Link="Includes\Shims\NullableAttributes.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(RunningDotNetPack)' == '' OR '$(RunningDotNetPack)' == 'false'">
+  <ItemGroup Condition="'$(RunningDotNetPack)' != 'true'">
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
   </ItemGroup>
 

src/OpenTelemetry.Instrumentation.GrpcNetClient/OpenTelemetry.Instrumentation.GrpcNetClient.csproj

@@ -19,7 +19,7 @@
     <Compile Include="$(RepoRoot)\src\Shared\Shims\NullableAttributes.cs" Link="Includes\Shims\NullableAttributes.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(RunningDotNetPack)' == '' OR '$(RunningDotNetPack)' == 'false'">
+  <ItemGroup Condition="'$(RunningDotNetPack)' != 'true'">
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
   </ItemGroup>
 

src/OpenTelemetry.Instrumentation.Http/OpenTelemetry.Instrumentation.Http.csproj

@@ -18,7 +18,7 @@
     <Compile Include="$(RepoRoot)\src\Shared\Shims\NullableAttributes.cs" Link="Includes\Shims\NullableAttributes.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(RunningDotNetPack)' == '' OR '$(RunningDotNetPack)' == 'false'">
+  <ItemGroup Condition="'$(RunningDotNetPack)' != 'true'">
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
   </ItemGroup>
 

src/OpenTelemetry.Instrumentation.SqlClient/OpenTelemetry.Instrumentation.SqlClient.csproj

@@ -18,7 +18,7 @@
     <Compile Include="$(RepoRoot)\src\Shared\Shims\NullableAttributes.cs" Link="Includes\Shims\NullableAttributes.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(RunningDotNetPack)' == '' OR '$(RunningDotNetPack)' == 'false'">
+  <ItemGroup Condition="'$(RunningDotNetPack)' != 'true'">
     <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Api.ProviderBuilderExtensions\OpenTelemetry.Api.ProviderBuilderExtensions.csproj" />
   </ItemGroup>
 

src/OpenTelemetry/CHANGELOG.md

@@ -19,6 +19,11 @@
   Previously it would return `null`.
   ([#4885](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4885))
 
+* Updated `Microsoft.Extensions.Logging` package version to
+  `8.0.0-rc.1.23419.4`.
+  ([#4920](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4920),
+  [#4933](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4933))
+
 ## 1.6.0
 
 Released 2023-Sep-05

src/OpenTelemetry/Internal/Builder/ProviderBuilderServiceCollectionExtensions.cs

@@ -76,7 +76,7 @@ public static IServiceCollection AddOpenTelemetrySharedProviderBuilderServices(t
         // which sets default Propagators and default Activity Id format
         _ = Sdk.SuppressInstrumentation;
 
-        services.AddOptions();
+        services!.AddOptions();
 
         // Note: When using a host builder IConfiguration is automatically
         // registered and this registration will no-op. This only runs for

src/OpenTelemetry/Logs/ILogger/OpenTelemetryLogger.cs

@@ -128,7 +128,8 @@ public bool IsEnabled(LogLevel logLevel)
         return logLevel != LogLevel.None;
     }
 
-    public IDisposable BeginScope<TState>(TState state) => this.ScopeProvider?.Push(state) ?? NullScope.Instance;
+    public IDisposable BeginScope<TState>(TState state)
+        where TState : notnull => this.ScopeProvider?.Push(state) ?? NullScope.Instance;
 
     internal static void SetLogRecordSeverityFields(ref LogRecordData logRecordData, LogLevel logLevel)
     {

src/OpenTelemetry/OpenTelemetry.csproj

@@ -6,6 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging" />
     <PackageReference Include="Microsoft.Extensions.Logging.Configuration" />
   </ItemGroup>
 

src/Shared/Options/ConfigurationExtensions.cs

@@ -147,7 +147,7 @@ public static IServiceCollection RegisterOptionsFactory<T>(
         Debug.Assert(services != null, "services was null");
         Debug.Assert(optionsFactoryFunc != null, "optionsFactoryFunc was null");
 
-        services.TryAddSingleton<IOptionsFactory<T>>(sp =>
+        services!.TryAddSingleton<IOptionsFactory<T>>(sp =>
         {
             return new DelegatingOptionsFactory<T>(
                 (c, n) => optionsFactoryFunc!(c),
@@ -168,7 +168,7 @@ public static IServiceCollection RegisterOptionsFactory<T>(
         Debug.Assert(services != null, "services was null");
         Debug.Assert(optionsFactoryFunc != null, "optionsFactoryFunc was null");
 
-        services.TryAddSingleton<IOptionsFactory<T>>(sp =>
+        services!.TryAddSingleton<IOptionsFactory<T>>(sp =>
         {
             return new DelegatingOptionsFactory<T>(
                 (c, n) => optionsFactoryFunc!(sp, c, n),

test/Benchmarks/Benchmarks.csproj

@@ -15,7 +15,6 @@
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" />
-    <PackageReference Include="Microsoft.Extensions.Logging" />
     <PackageReference Include="Moq" />
   </ItemGroup>
 

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpLogExporterTests.cs

@@ -26,6 +26,7 @@
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Logs;
+using OpenTelemetry.Resources;
 using OpenTelemetry.Tests;
 using OpenTelemetry.Trace;
 using Xunit;
@@ -1222,6 +1223,64 @@ public void AddOtlpLogExporterLogRecordProcessorOptionsTest(ExportProcessorType
         }
     }
 
+    [Fact]
+    public void ValidateInstrumentationScope()
+    {
+        var logRecords = new List<LogRecord>();
+        using var loggerFactory = LoggerFactory.Create(builder =>
+        {
+            builder
+                .AddOpenTelemetry(options => options
+                    .AddInMemoryExporter(logRecords));
+        });
+
+        var logger1 = loggerFactory.CreateLogger("OtlpLogExporterTests-A");
+        logger1.LogInformation("Hello from red-tomato");
+
+        var logger2 = loggerFactory.CreateLogger("OtlpLogExporterTests-B");
+        logger2.LogInformation("Hello from green-tomato");
+
+        Assert.Equal(2, logRecords.Count);
+
+        var batch = new Batch<LogRecord>(logRecords.ToArray(), logRecords.Count);
+        var logRecordTransformer = new OtlpLogRecordTransformer(new(), new());
+
+        var resourceBuilder = ResourceBuilder.CreateEmpty();
+        var processResource = resourceBuilder.Build().ToOtlpResource();
+
+        var request = logRecordTransformer.BuildExportRequest(processResource, batch);
+
+        Assert.Single(request.ResourceLogs);
+
+        var scope1 = request.ResourceLogs[0].ScopeLogs.First();
+        var scope2 = request.ResourceLogs[0].ScopeLogs.Last();
+
+        Assert.Equal("OtlpLogExporterTests-A", scope1.Scope.Name);
+        Assert.Equal("OtlpLogExporterTests-B", scope2.Scope.Name);
+
+        Assert.Single(scope1.LogRecords);
+        Assert.Single(scope2.LogRecords);
+
+        var logrecord1 = scope1.LogRecords[0];
+        var logrecord2 = scope2.LogRecords[0];
+
+        Assert.Equal("Hello from red-tomato", logrecord1.Body.StringValue);
+
+        Assert.Equal("Hello from green-tomato", logrecord2.Body.StringValue);
+
+        // Validate LogListPool
+        Assert.Empty(OtlpLogRecordTransformer.LogListPool);
+        logRecordTransformer.Return(request);
+        Assert.Equal(2, OtlpLogRecordTransformer.LogListPool.Count);
+
+        request = logRecordTransformer.BuildExportRequest(processResource, batch);
+
+        Assert.Single(request.ResourceLogs);
+
+        // ScopeLogs will be reused.
+        Assert.Empty(OtlpLogRecordTransformer.LogListPool);
+    }
+
     private static OtlpCommon.KeyValue TryGetAttribute(OtlpLogs.LogRecord record, string key)
     {
         return record.Attributes.FirstOrDefault(att => att.Key == key);