update BenchmarkDotNet to latest version to take advantage of improvements (#511)

adamsitnik · michellemcdaniel · commit fe1c35542bdd · 2019-08-14T14:04:38.000-07:00
* update BenchmarkDotNet to latest version to take advantage of improvements

* use FullNameProvider.GetBenchmarkName without reflection since it's public now

* update BDN to version that supports .NET Core 5.0

* add 5.0 to the matrix to make sure that this PR really fixes to 5.0 problem

* Add private jobs for microbenchmarks for netcoreapp5.0
diff --git a/README.md b/README.md
@@ -24,6 +24,7 @@ This project has adopted the code of conduct defined by the Contributor Covenant
 
 | Framework | Windows RS4 x64                                                                             | Windows RS4 x86                                                                             | Ubuntu 16.04 x64                                                                            | Ubuntu 16.04 ARM64                                                                              |
 | :-------- | :-----------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------: |
+| Core 5.0  | [![micro_windows_RS4_x64_netcoreapp5.0_icon]][micro_windows_RS4_x64_netcoreapp5.0_status] | [![micro_windows_RS4_x86_netcoreapp5.0_icon]][micro_windows_RS4_x86_netcoreapp5.0_status] | [![micro_ubuntu_1604_x64_netcoreapp5.0_icon]][micro_ubuntu_1604_x64_netcoreapp5.0_status] | Disabled |
 | Core 3.0  | [![micro_windows_RS4_x64_netcoreapp3.0_icon]][micro_windows_RS4_x64_netcoreapp3.0_status] | [![micro_windows_RS4_x86_netcoreapp3.0_icon]][micro_windows_RS4_x86_netcoreapp3.0_status] | [![micro_ubuntu_1604_x64_netcoreapp3.0_icon]][micro_ubuntu_1604_x64_netcoreapp3.0_status] | Disabled |
 | Core 2.2  | [![micro_windows_RS4_x64_netcoreapp2.2_icon]][micro_windows_RS4_x64_netcoreapp2.2_status] |                                                                                             | [![micro_ubuntu_1604_x64_netcoreapp2.2_icon]][micro_ubuntu_1604_x64_netcoreapp2.2_status] | N/A                                                                                             |
 | Core 2.1  | [![micro_windows_RS4_x64_netcoreapp2.1_icon]][micro_windows_RS4_x64_netcoreapp2.1_status] |                                                                                             | [![micro_ubuntu_1604_x64_netcoreapp2.1_icon]][micro_ubuntu_1604_x64_netcoreapp2.1_status] | N/A                                                                                             |
@@ -53,6 +54,8 @@ This project has adopted the code of conduct defined by the Contributor Covenant
 [//]: # (These are the micro links)
 
 [//]: # (These are the windows x64 links)
+[micro_windows_RS4_x64_netcoreapp5.0_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=windows%20RS4%20x64%20micro&configuration=windows%20RS4%20x64%20micro%20netcoreapp5.0
+[micro_windows_RS4_x64_netcoreapp5.0_icon]:       https://dev.azure.com/dnceng/public/_apis/build/status/dotnet/performance/performance-ci?branchName=master&jobName=windows%20RS4%20x64%20micro&configuration=windows%20RS4%20x64%20micro%20netcoreapp5.0
 [micro_windows_RS4_x64_netcoreapp3.0_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=windows%20RS4%20x64%20micro&configuration=windows%20RS4%20x64%20micro%20netcoreapp3.0
 [micro_windows_RS4_x64_netcoreapp3.0_icon]:       https://dev.azure.com/dnceng/public/_apis/build/status/dotnet/performance/performance-ci?branchName=master&jobName=windows%20RS4%20x64%20micro&configuration=windows%20RS4%20x64%20micro%20netcoreapp3.0
 [micro_windows_RS4_x64_netcoreapp2.2_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=windows%20RS4%20x64%20micro&configuration=windows%20RS4%20x64%20micro%20netcoreapp2.2
@@ -68,6 +71,8 @@ This project has adopted the code of conduct defined by the Contributor Covenant
 [micro_windows_RS4_x86_netcoreapp3.0_icon]:       https://dev.azure.com/dnceng/public/_apis/build/status/dotnet/performance/performance-ci?branchName=master&jobName=windows%20RS4%20x86%20micro&configuration=windows%20RS4%20x86%20micro%20netcoreapp3.0
 
 [//]: # (These are the ubuntu x64 links)
+[micro_ubuntu_1604_x64_netcoreapp5.0_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=ubuntu%201604%20x64%20micro&configuration=ubuntu%201604%20x64%20micro%20netcoreapp5.0
+[micro_ubuntu_1604_x64_netcoreapp5.0_icon]:       https://dev.azure.com/dnceng/public/_apis/build/status/dotnet/performance/performance-ci?branchName=master&jobName=ubuntu%201604%20x64%20micro&configuration=ubuntu%201604%20x64%20micro%20netcoreapp5.0
 [micro_ubuntu_1604_x64_netcoreapp3.0_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=ubuntu%201604%20x64%20micro&configuration=ubuntu%201604%20x64%20micro%20netcoreapp3.0
 [micro_ubuntu_1604_x64_netcoreapp3.0_icon]:       https://dev.azure.com/dnceng/public/_apis/build/status/dotnet/performance/performance-ci?branchName=master&jobName=ubuntu%201604%20x64%20micro&configuration=ubuntu%201604%20x64%20micro%20netcoreapp3.0
 [micro_ubuntu_1604_x64_netcoreapp2.2_status]:     https://dev.azure.com/dnceng/public/_build/latest?definitionId=271&branchName=master&jobName=ubuntu%201604%20x64%20micro&configuration=ubuntu%201604%20x64%20micro%20netcoreapp2.2
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -62,6 +62,7 @@ jobs:
       csproj: src\benchmarks\micro\MicroBenchmarks.csproj
       runCategories: 'coreclr corefx' 
       frameworks: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
+        - netcoreapp5.0
         - netcoreapp3.0
         - netcoreapp2.2
         - netcoreapp2.1
@@ -135,6 +136,7 @@ jobs:
       csproj: src/benchmarks/micro/MicroBenchmarks.csproj
       runCategories: 'coreclr corefx'
       frameworks: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
+        - netcoreapp5.0
         - netcoreapp3.0
         - netcoreapp2.2
         - netcoreapp2.1
@@ -186,7 +188,8 @@ jobs:
       csproj: src\benchmarks\micro\MicroBenchmarks.csproj
       runCategories: 'coreclr corefx'
       benchviewCategory: 'coreclr'
-      frameworks: # for private jobs we want to benchmark .NET Core 3.0 only
+      frameworks: # for private jobs we want to benchmark .NET Core 3.0 and 5.0 only
+        - netcoreapp5.0
         - netcoreapp3.0
       
   # Windows x86 micro benchmarks
@@ -201,7 +204,8 @@ jobs:
       csproj: src\benchmarks\micro\MicroBenchmarks.csproj
       runCategories: 'coreclr corefx'
       benchviewCategory: 'coreclr'
-      frameworks: # for private jobs we want to benchmark .NET Core 3.0 only
+      frameworks: # for private jobs we want to benchmark .NET Core 3.0 and 5.0 only
+        - netcoreapp5.0
         - netcoreapp3.0
 
   # Windows x64 ML.NET benchmarks
@@ -247,7 +251,8 @@ jobs:
       csproj: src/benchmarks/micro/MicroBenchmarks.csproj
       runCategories: 'coreclr corefx'
       benchviewCategory: 'coreclr'
-      frameworks: # for private jobs we want to benchmark .NET Core 3.0 only
+      frameworks: # for private jobs we want to benchmark .NET Core 3.0 and 5.0 only
+        - netcoreapp5.0
         - netcoreapp3.0
 
   # Ubuntu 1804 x64 ML.NET benchmarks
diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
@@ -6,8 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.11.3.1003" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.11.3.1003" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.11.5.1143" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.11.5.1143" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs
@@ -30,14 +30,10 @@ public override void ExportToLog(Summary summary, ILogger logger)
             if (reporter == null) // not running in the perf lab
                 return;
 
-            var nameDelegate = typeof(BenchmarkCase).Assembly.GetType("BenchmarkDotNet.Exporters.FullNameProvider").GetMethod("GetBenchmarkName", BindingFlags.Static | BindingFlags.NonPublic);
-
-
             foreach (var report in summary.Reports)
             {
                 var test = new Test();
-                //test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase);
-                test.Name = (string)nameDelegate.Invoke(null, new[] { report.BenchmarkCase });
+                test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase);
                 test.Categories = report.BenchmarkCase.Descriptor.Categories;
                 var results = from result in report.AllMeasurements
                               where result.IterationMode == Engines.IterationMode.Workload && result.IterationStage == Engines.IterationStage.Result
diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
@@ -18,6 +18,7 @@ public static IConfig Create(DirectoryInfo artifactsPath, ImmutableHashSet<strin
                     .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us
                     .WithMinIterationCount(15)
                     .WithMaxIterationCount(20) // we don't want to run more that 20 iterations
+                    .DontEnforcePowerPlan() // make sure BDN does not try to enforce High Performance power plan on Windows
                     .AsDefault()) // tell BDN that this are our default settings
                 .WithArtifactsPath(artifactsPath.FullName)
                 .With(MemoryDiagnoser.Default) // MemoryDiagnoser is enabled by default
