Add tool chain for Netcore Mono AOT. (#1662)

* Add tool chain for Netcore Mono AOT.

Co-authored-by: Adam Sitnik <[email protected]>

Author: Nathan Ricci

NuGet.Config

@@ -11,5 +11,7 @@
     <add key="perfolizer-nightly" value="https://www.myget.org/F/perfolizer/api/v3/index.json" />
     <!-- reuquired to run Mono Wasm benchmarks -->
     <add key="dotnet5" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
+    <!-- reuquired to run Mono AOT benchmarks -->
+    <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
   </packageSources>
 </configuration>

docs/articles/configs/toolchains.md

@@ -353,3 +353,38 @@ Now you should be able to run the Wasm benchmarks!
 
 [!include[IntroWasm](../samples/IntroWasm.md)]
 
+## MonoAotLLVM
+
+BenchmarkDotNet supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses llvm on the back end).
+
+Using this tool chain requires the following flags:
+
+```
+--runtimes monoaotllvm
+--aotcompilerpath <path to mono aot compiler>
+--customruntimepack <path to runtime pack>
+```
+
+and optionally (defaults to mini)
+
+```
+--aotcompilermode <mini|llvm>  
+```
+
+As of this writing, the mono aot compiler is not available as a seperate download or nuget package. Therefore, it is required to build the compiler in the [dotnet/runtime repository].
+
+The compiler binary (mono-sgen) is built as part of the `mono` subset, so it can be built (along with the runtime pack) like so (in the root of [dotnet/runtime]).
+
+`./build.sh -subset mono+libs -c Release`
+
+The compiler binary should be generated here (modify for your platform):
+
+```
+<runtime root>/artifacts/obj/mono/OSX.x64.Release/mono/mini/mono-sgen
+```
+
+And the runtime pack should be generated here:
+
+```
+<runtimeroot>artifacts/bin/microsoft.netcore.app.runtime.osx-x64/Release/
+```

src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs

@@ -138,6 +138,11 @@ public enum RuntimeMoniker
         /// <summary>
         /// WebAssembly with .net6.0
         /// </summary>
-        WasmNet60
+        WasmNet60,
+
+        /// <summary>
+        /// Mono with the Ahead of Time LLVM Compiler backend
+        /// </summary>
+        MonoAOTLLVM
     }
 }

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -7,6 +7,7 @@
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Helpers;
+using BenchmarkDotNet.Toolchains.MonoAotLLVM;
 using CommandLine;
 using CommandLine.Text;
 using JetBrains.Annotations;
@@ -180,9 +181,15 @@ public class CommandLineOptions
         [Option("wasmArgs", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by Wasm toolchain.")]
         public string WasmJavaScriptEngineArguments { get; set; }
 
-        [Option("customRuntimePack", Required = false, HelpText = "Specify the path to a custom runtime pack. Only used for wasm currently.")]
+        [Option("customRuntimePack", Required = false, HelpText = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently.")]
         public string CustomRuntimePack { get; set; }
 
+        [Option("AOTCompilerPath", Required = false, HelpText = "Path to Mono AOT compiler, used for MonoAotLLVM.")]
+        public FileInfo AOTCompilerPath { get; set; }
+
+        [Option("AOTCompilerMode", Required = false, Default = MonoAotCompilerMode.mini, HelpText = "Mono AOT compiler mode, either 'mini' or 'llvm'")]
+        public MonoAotCompilerMode AOTCompilerMode { get; set; }
+
         internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any();
 
         [Usage(ApplicationAlias = "")]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -24,6 +24,7 @@
 using BenchmarkDotNet.Toolchains.DotNetCli;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using BenchmarkDotNet.Toolchains.MonoWasm;
+using BenchmarkDotNet.Toolchains.MonoAotLLVM;
 using CommandLine;
 using Perfolizer.Horology;
 using Perfolizer.Mathematics.OutlierDetection;
@@ -116,6 +117,10 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
                     logger.WriteLineError($"The provided {nameof(options.WasmMainJs)} \"{options.WasmMainJs}\" does NOT exist. It MUST be provided.");
                     return false;
                 }
+                else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist()))
+                {
+                     logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{ options.AOTCompilerPath }\" does NOT exist. It MUST be provided.");
+                }
             }
 
             foreach (string exporter in options.Exporters)
@@ -386,7 +391,22 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                     return MakeWasmJob(baseJob, options, timeOut, "net5.0");
                 case RuntimeMoniker.WasmNet60:
                     return MakeWasmJob(baseJob, options, timeOut, "net6.0");
+                case RuntimeMoniker.MonoAOTLLVM:
+                    var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath);
+
+                    var toolChain = MonoAotLLVMToolChain.From(
+                    new NetCoreAppSettings(
+                        targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker,
+                        runtimeFrameworkVersion: null,
+                        name: monoAotLLVMRuntime.Name,
+                        customDotNetCliPath: options.CliPath?.FullName,
+                        packagesPath: options.RestorePath?.FullName,
+                        timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout,
+                        customRuntimePack: options.CustomRuntimePack,
+                        aotCompilerPath: options.AOTCompilerPath.ToString(),
+                        aotCompilerMode: options.AOTCompilerMode));
 
+                    return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain);
                 default:
                     throw new NotSupportedException($"Runtime {runtimeId} is not supported");
             }

src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs

@@ -0,0 +1,44 @@
+using System;
+using System.ComponentModel;
+using System.IO;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Jobs;
+
+namespace BenchmarkDotNet.Environments
+{
+    public class MonoAotLLVMRuntime : Runtime, IEquatable<MonoAotLLVMRuntime>
+    {
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        internal static readonly MonoAotLLVMRuntime Default = new MonoAotLLVMRuntime();
+
+        public FileInfo AOTCompilerPath { get; }
+
+        /// <summary>
+        /// creates new instance of MonoAotLLVMRuntime
+        /// </summary>
+        public MonoAotLLVMRuntime(FileInfo aotCompilerPath, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM") : base(RuntimeMoniker.MonoAOTLLVM, msBuildMoniker, displayName)
+        {
+            if (aotCompilerPath == null)
+                throw new ArgumentNullException(paramName: nameof(aotCompilerPath));
+            if (aotCompilerPath.IsNotNullButDoesNotExist())
+                throw new FileNotFoundException($"Provided {nameof(aotCompilerPath)} file: \"{aotCompilerPath.FullName}\" doest NOT exist");
+
+            AOTCompilerPath = aotCompilerPath;
+        }
+
+        // this ctor exists only for the purpose of having .Default property that returns something consumable by RuntimeInformation.GetCurrentRuntime()
+        private MonoAotLLVMRuntime(string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM") : base(RuntimeMoniker.MonoAOTLLVM, msBuildMoniker, displayName)
+        {
+            AOTCompilerPath = new FileInfo("fake");
+        }
+
+        public override bool Equals(object obj)
+            => obj is MonoAotLLVMRuntime other && Equals(other);
+
+        public bool Equals(MonoAotLLVMRuntime other)
+            => other != null && base.Equals(other) && other.AOTCompilerPath == AOTCompilerPath;
+
+        public override int GetHashCode()
+            => base.GetHashCode() ^ AOTCompilerPath.GetHashCode();
+    }
+}

src/BenchmarkDotNet/Portability/RuntimeInformation.cs

@@ -44,6 +44,24 @@ public static bool IsCoreRT
 
         public static bool IsWasm => IsOSPlatform(OSPlatform.Create("BROWSER"));
 
+        public static bool IsAot { get; } = IsAotMethod(); // This allocates, so we only want to call it once statically.
+
+        private static bool IsAotMethod()
+        {
+            Type runtimeFeature = Type.GetType("System.Runtime.CompilerServices.RuntimeFeature");
+            if (runtimeFeature != null)
+            {
+                MethodInfo methodInfo = runtimeFeature.GetProperty("IsDynamicCodeCompiled", BindingFlags.Public | BindingFlags.Static)?.GetMethod;
+
+                if (methodInfo != null)
+                {
+                    return !(bool)methodInfo.Invoke(null, null);
+                }
+            }
+
+            return false;
+        }
+
         public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true");
 
         internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty;
@@ -197,6 +215,8 @@ internal static string GetRuntimeVersion()
         internal static Runtime GetCurrentRuntime()
         {
             //do not change the order of conditions because it may cause incorrect determination of runtime
+            if (IsAot && IsMono)
+                return MonoAotLLVMRuntime.Default;
             if (IsMono)
                 return MonoRuntime.Default;
             if (IsFullFramework)
@@ -252,7 +272,7 @@ internal static bool HasRyuJit()
 
         internal static string GetJitInfo()
         {
-            if (IsCoreRT || IsNetNative)
+            if (IsCoreRT || IsNetNative || IsAot)
                 return "AOT";
             if (IsMono || IsWasm)
                 return ""; // There is no helpful information about JIT on Mono

src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt

@@ -0,0 +1,66 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTarget="Publish">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <OutputPath>bin</OutputPath>
+    <TargetFramework>$TFM$</TargetFramework>
+    <MicrosoftNetCoreAppRuntimePackDir>$RUNTIMEPACK$</MicrosoftNetCoreAppRuntimePackDir>
+    <RuntimeIdentifier>$RUNTIMEIDENTIFIER$</RuntimeIdentifier>
+    <EnableTargetingPackDownload>false</EnableTargetingPackDownload>
+    <PublishTrimmed>false</PublishTrimmed>
+    <AssemblyName>$PROGRAMNAME$</AssemblyName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Runtime.MonoAOTCompiler.Task" version="6.0.0-*" GeneratePathProperty="true" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="$CODEFILENAME$" Exclude="bin\**;obj\**;**\*.xproj;packages\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$CSPROJPATH$" />
+  </ItemGroup>
+
+  <!-- Redirect 'dotnet publish' to in-tree runtime pack -->
+  <Target Name="TrickRuntimePackLocation" AfterTargets="ProcessFrameworkReferences">
+    <ItemGroup>
+      <RuntimePack>
+        <PackageDirectory>$(MicrosoftNetCoreAppRuntimePackDir)</PackageDirectory>
+      </RuntimePack>
+    </ItemGroup>
+    <Message Text="Packaged ID: %(RuntimePack.PackageDirectory)" Importance="high" />
+  </Target>
+
+  <UsingTask TaskName="MonoAOTCompiler" AssemblyFile="$(PkgMicrosoft_NET_Runtime_MonoAOTCompiler_Task)/MonoAOTCompiler.dll" /> 
+
+  <Target Name="AotApp" AfterTargets="Build" DependsOnTargets="Publish">
+
+   <PropertyGroup>
+      <PublishDirFullPath>$([System.IO.Path]::GetFullPath($(PublishDir)))</PublishDirFullPath>
+   </PropertyGroup>
+
+   <ItemGroup>
+     <AotInputAssemblies Include="$(PublishDirFullPath)\*.dll">
+        <AotArguments>@(MonoAOTCompilerDefaultAotArguments, ';')</AotArguments>
+        <ProcessArguments>@(MonoAOTCompilerDefaultProcessArguments, ';')</ProcessArguments>
+     </AotInputAssemblies>
+   </ItemGroup>
+
+   <MonoAOTCompiler
+        CompilerBinaryPath="$COMPILERBINARYPATH$"
+        Mode="Normal"
+        OutputType="Normal"
+        Assemblies="@(AotInputAssemblies)"
+        UseLLVM="$USELLVM$"
+        LLVMPath="$(MicrosoftNetCoreAppRuntimePackDir)\runtimes\$(RuntimeIdentifier)\native"
+	    OutputDir="$(PublishDir)"
+        UseAotDataFile="false">
+        <Output TaskParameter="CompiledAssemblies" ItemName="BundleAssemblies" />
+    </MonoAOTCompiler>
+
+   <Message Text="CompiledAssemblies: $(BundleAssemblies)" Importance="high"/>
+
+   </Target>
+
+</Project>

src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs

@@ -133,7 +133,7 @@ public CustomDotNetCliToolchainBuilder Timeout(TimeSpan timeout)
             return this;
         }
 
-        protected static string GetPortableRuntimeIdentifier()
+        internal static string GetPortableRuntimeIdentifier()
         {
             // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier()
             // returns win10-x64, we want the simpler form win-x64

src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs

@@ -1,6 +1,8 @@
 using System;
+using BenchmarkDotNet.Toolchains.MonoAotLLVM;
 using JetBrains.Annotations;
 
+
 namespace BenchmarkDotNet.Toolchains.DotNetCli
 {
     /// <summary>
@@ -36,6 +38,9 @@ public class NetCoreAppSettings
         /// </param>
         /// <param name="packagesPath">the directory to restore packages to</param>
         /// <param name="timeout">timeout to build the benchmark</param>
+        /// <param name="customRuntimePack">path to a custom runtime pack</param>
+        /// <param name="aotCompilerPath">path to Mono AOT compiler</param>
+        /// <param name="aotCompilerMode">Mono AOT compiler moder</param>
         /// </summary>
         [PublicAPI]
         public NetCoreAppSettings(
@@ -45,7 +50,9 @@ public NetCoreAppSettings(
             string customDotNetCliPath = null,
             string packagesPath = null,
             TimeSpan? timeout = null,
-            string customRuntimePack = null
+            string customRuntimePack = null,
+            string aotCompilerPath = null,
+            MonoAotCompilerMode aotCompilerMode = MonoAotCompilerMode.mini
             )
         {
             TargetFrameworkMoniker = targetFrameworkMoniker;
@@ -56,6 +63,8 @@ public NetCoreAppSettings(
             PackagesPath = packagesPath;
             Timeout = timeout ?? DefaultBuildTimeout;
             CustomRuntimePack = customRuntimePack;
+            AOTCompilerPath = aotCompilerPath;
+            AOTCompilerMode = aotCompilerMode;
         }
 
         /// <summary>
@@ -87,6 +96,16 @@ public NetCoreAppSettings(
         /// </summary>
         public string CustomRuntimePack { get; }
 
+        /// <summary>
+        /// Path to the Mono AOT Compiler
+        /// </summary>
+        public string AOTCompilerPath { get; }
+
+        /// <summary>
+        /// Mono AOT Compiler mode, either 'mini' or 'llvm'
+        /// </summary>
+        public MonoAotCompilerMode AOTCompilerMode { get; }
+
         public NetCoreAppSettings WithCustomDotNetCliPath(string customDotNetCliPath, string displayName = null)
             => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, displayName ?? Name, customDotNetCliPath, PackagesPath, Timeout);
 
@@ -95,6 +114,5 @@ public NetCoreAppSettings WithCustomPackagesRestorePath(string packagesPath, str
 
         public NetCoreAppSettings WithTimeout(TimeSpan? timeOut)
             => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, Name, CustomDotNetCliPath, PackagesPath, timeOut ?? Timeout);
-
     }
 }

src/BenchmarkDotNet/Toolchains/Executor.cs

@@ -120,6 +120,11 @@ private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsP
                     start.Arguments = $"{wasm.JavaScriptEngineArguments} runtime.js -- --run {artifactsPaths.ProgramName}.dll {args} ";
                     start.WorkingDirectory = artifactsPaths.BinariesDirectoryPath;
                     break;
+                case MonoAotLLVMRuntime _:
+                    start.FileName = exePath;
+                    start.Arguments = args;
+                    start.WorkingDirectory = artifactsPaths.BinariesDirectoryPath;
+                    break;
                 default:
                     throw new NotSupportedException("Runtime = " + runtime);
             }

src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotCompilerMode.cs

@@ -0,0 +1,8 @@
+namespace BenchmarkDotNet.Toolchains.MonoAotLLVM
+{
+    public enum MonoAotCompilerMode
+    {
+        mini = 0, // default
+        llvm
+    }
+}