Add new Mosa.Utility.CreateCoreLib tool

Source/Directory.Build.props

@@ -4,7 +4,7 @@
 	<PropertyGroup>
 		<OutputPath>../../bin</OutputPath>
 		<TargetFramework>net8.0</TargetFramework>
-		<LangVersion>11.0</LangVersion>
+		<LangVersion>12.0</LangVersion>
 		<Nullable>disable</Nullable>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<!--		<WarningsAsErrors>nullable</WarningsAsErrors>-->

Source/Docs/contents.txt

@@ -21,6 +21,7 @@
    :maxdepth: 2
 
    tool-compiler
+   tool-createcorelib
    tool-launcher
    tool-launcher-console
    tool-explorer

Source/Docs/tool-createcorelib.rst

@@ -0,0 +1,63 @@
+##################
+MOSA CreateCoreLib
+##################
+
+The **MOSA CreateCoreLib** tool is a console application that automatically creates a **stub** core library (that is, a
+core library with only the reference API surface but no implementation).
+
+Usage
+-----
+
+A quick example that creates a core library project (excluding the C# project file) inside the `Source/Mosa.TinyCoreLib`
+directory:
+
+.. code-block:: text
+
+	Mosa.Tool.CreateCoreLib --output Source/Mosa.TinyCoreLib
+
+.. code-block:: text
+
+	Cloning .NET runtime GitHub repository...
+	Cloning into 'runtime'...
+	remote: Enumerating objects: 65940, done.
+	remote: Counting objects: 100% (65940/65940), done.
+	remote: Compressing objects: 100% (42247/42247), done.
+	remote: Total 65940 (delta 25891), reused 34783 (delta 20941), pack-reused 0
+	Receiving objects: 100% (65940/65940), 88.07 MiB | 9.20 MiB/s, done.
+	Resolving deltas: 100% (25891/25891), done.
+	Updating files: 100% (58893/58893), done.
+	Parsing input files...
+	Patching System.DirectoryServices.manual.cs...
+	Patching System.Net.Http.Json.cs...
+	Patching System.Net.Http.WinHttpHandler.cs...
+	Patching System.Configuration.ConfigurationManager.cs...
+	Patching System.Data.Common.cs...
+	Compiling source files...
+	Decompiling assembly...
+	Removing project file...
+	Removing Properties folder...
+	Patching System.Collections.Generic/PriorityQueue.cs...
+	Patching System.Collections.Generic/PriorityQueue.cs...
+	Patching System/Nullable.cs...
+	Patching System/ReadOnlySpan.cs...
+	Patching System.Runtime.InteropServices/Marshal.cs...
+	Patching System.Runtime.InteropServices/MemoryMarshal.cs...
+	Patching System.Runtime.CompilerServices/Unsafe.cs...
+	Patching System.Numerics/Vector.cs...
+	Patching System.Runtime.Intrinsics/Vector64.cs...
+	Patching System.Runtime.Intrinsics/Vector128.cs...
+	Patching System.Runtime.Intrinsics/Vector256.cs...
+	Patching System.Runtime.Intrinsics/Vector512.cs...
+	Patching System.Threading/Volatile.cs...
+	Patching System.Threading/Interlocked.cs...
+
+Command Line Options
+--------------------
+
+Here are all the available command line options:
+
+`--output/-o`
+	Sets the output directory.
+
+`--copy-files/-c`
+	If enabled, only copies and patches the source files and stops.

Source/Mosa.Linux.sln

@@ -147,6 +147,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BareMetal.Demos", "BareMeta
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Documentation", "Documentation", "{A4028807-1B21-4D14-9CE3-5FD1AAD9EDD7}"
 	ProjectSection(SolutionItems) = preProject
+		Docs\a-dive-into-baremetal.rst = Docs\a-dive-into-baremetal.rst
 		Docs\authors.rst = Docs\authors.rst
 		Docs\build-status.rst = Docs\build-status.rst
 		Docs\command-line-arguments.rst = Docs\command-line-arguments.rst
@@ -163,18 +164,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Documentation", "Documentat
 		Docs\introduction.rst = Docs\introduction.rst
 		Docs\license.rst = Docs\license.rst
 		Docs\mosa-runtime-tables.dot = Docs\mosa-runtime-tables.dot
+		Docs\optimization-options.rst = Docs\optimization-options.rst
+		Docs\project-structure.rst = Docs\project-structure.rst
 		Docs\runtime-tables.rst = Docs\runtime-tables.rst
 		Docs\settings-options.rst = Docs\settings-options.rst
 		Docs\tool-compiler.rst = Docs\tool-compiler.rst
+		Docs\tool-createcorelib.rst = Docs\tool-createcorelib.rst
 		Docs\tool-debugger.rst = Docs\tool-debugger.rst
 		Docs\tool-explorer.rst = Docs\tool-explorer.rst
 		Docs\tool-launcher-console.rst = Docs\tool-launcher-console.rst
 		Docs\tool-launcher.rst = Docs\tool-launcher.rst
 		Docs\unit-tests.rst = Docs\unit-tests.rst
 		Docs\usb-flash-drive-installation.rst = Docs\usb-flash-drive-installation.rst
-		Docs\project-structure.rst = Docs\project-structure.rst
-		Docs\optimization-options.rst = Docs\optimization-options.rst
-		Docs\a-dive-into-baremetal.rst = Docs\a-dive-into-baremetal.rst
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mosa.BareMetal.HelloWorld.x64", "Mosa.BareMetal.HelloWorld.x64\Mosa.BareMetal.HelloWorld.x64.csproj", "{5F4136A8-EB83-41BE-9D81-B16E723AAE29}"
@@ -239,6 +240,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mosa.Workspace.GDB.Debug",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mosa.Tool.Explorer.Avalonia", "Mosa.Tool.Explorer.Avalonia\Mosa.Tool.Explorer.Avalonia.csproj", "{FBC3B0CD-D775-41C6-A735-F59118838397}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mosa.Tool.CreateCoreLib", "Mosa.Tool.CreateCoreLib\Mosa.Tool.CreateCoreLib.csproj", "{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1405,6 +1408,24 @@ Global
 		{FBC3B0CD-D775-41C6-A735-F59118838397}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{FBC3B0CD-D775-41C6-A735-F59118838397}.Release|x86.ActiveCfg = Release|Any CPU
 		{FBC3B0CD-D775-41C6-A735-F59118838397}.Release|x86.Build.0 = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Debug|x86.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|Any CPU.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|Any CPU.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|Mixed Platforms.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|x86.ActiveCfg = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Description|x86.Build.0 = Debug|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|x86.ActiveCfg = Release|Any CPU
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1479,6 +1500,7 @@ Global
 		{00512754-29C4-4AA3-8619-8C04120D7B55} = {D1C4B715-9764-4430-B3D3-676B0EBCE75A}
 		{A4028807-1B21-4D14-9CE3-5FD1AAD9EDD7} = {F0EFF742-92D5-4219-939A-8F6F8DAB24E5}
 		{FBC3B0CD-D775-41C6-A735-F59118838397} = {D032B24A-CE3A-4881-BACE-CC4FE0AFD69D}
+		{3347E41D-AD8B-4891-9A9B-5EE36BACA6F1} = {D032B24A-CE3A-4881-BACE-CC4FE0AFD69D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {C22A5C94-6B05-4B1B-845A-A2EA7615E093}

Source/Mosa.Tool.CreateCoreLib/Mosa.Tool.CreateCoreLib.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+		<OutputType>Exe</OutputType>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+
+    <ItemGroup>
+		<PackageReference Include="CommandLineParser" Version="2.9.1" />
+		<PackageReference Include="ICSharpCode.Decompiler" Version="8.2.0.7535" />
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
+    </ItemGroup>
+
+</Project>

Source/Mosa.Tool.CreateCoreLib/Options.cs

@@ -0,0 +1,12 @@
+using CommandLine;
+
+namespace Mosa.Tool.CreateCoreLib;
+
+public class Options
+{
+    [Option('o', "output", Required = true, HelpText = "Sets the output directory.")]
+    public string? OutputDirectory { get; set; }
+
+    [Option('c', "copy-files", Required = false, HelpText = "If enabled, only copies and patches the source files and stops.")]
+    public bool CopyFiles { get; set; }
+}

Source/Mosa.Tool.CreateCoreLib/Patcher.cs

@@ -0,0 +1,71 @@
+using System.Diagnostics;
+
+namespace Mosa.Tool.CreateCoreLib;
+
+/// <summary>
+/// A custom patch system. It isn't designed to be particularly fast, but is rather designed with the simplicity of a singular patch in mind.
+/// </summary>
+public static class Patcher
+{
+    /*
+     * Modes:
+     * rs - [r]emove line if it [s]tarts with str1
+     * R - [R]eplace all occurences of str1 with str2
+     */
+    public record PatchRecord(string File, string Mode, string Str1, string? Str2);
+
+    // First round of patches before compilation (i.e. with files directly from the repository), they use paths relative to the current directory.
+    // The reference API source files weren't designed to be compiled as-is, so they have to be patched very lightly.
+	public static readonly PatchRecord[] FirstPatches =
+    [
+        new PatchRecord("runtime/src/libraries/System.DirectoryServices/ref/System.DirectoryServices.manual.cs", "rs", "[assembly: TypeForwardedTo(", null),
+        new PatchRecord("runtime/src/libraries/System.Net.Http.Json/ref/System.Net.Http.Json.cs", "R", "protected override bool TryComputeLength", "protected internal override bool TryComputeLength"),
+        new PatchRecord("runtime/src/libraries/System.Net.Http.WinHttpHandler/ref/System.Net.Http.WinHttpHandler.cs", "R", "protected override System.Threading.Tasks.Task<System.Net.Http.HttpResponseMessage> SendAsync", "protected internal override System.Threading.Tasks.Task<System.Net.Http.HttpResponseMessage> SendAsync"),
+        new PatchRecord("runtime/src/libraries/System.Data.Common/ref/System.Data.Common.cs", "R", "protected override System.Xml.XPath.XPathNavigator? CreateNavigator", "protected internal override System.Xml.XPath.XPathNavigator? CreateNavigator"),
+        new PatchRecord("runtime/src/libraries/System.Configuration.ConfigurationManager/ref/System.Configuration.ConfigurationManager.cs", "R", "[System.ObsoleteAttribute(System.Obsoletions.BinaryFormatterMessage + @\"", "[System.ObsoleteAttribute(\"BinaryFormatter serialization is obsolete and should not be used. See https://aka.ms/binaryformatter for more information")
+    ];
+
+    // Second round of patches after decompilation, they use paths relative to the output directory.
+	// The code produced by ICSharpCode's decompiler isn't perfected, and isn't designed to be compiled directly either.
+	// While it requires a bit more patching than before, it still isn't a dramatic amount of patches.
+    public static readonly PatchRecord[] SecondPatches =
+    [
+        new PatchRecord("System.Collections.Generic/PriorityQueue.cs", "R", "IEnumerator<(TElement, TPriority)>", "IEnumerator<(TElement Element, TPriority Priority)>"),
+        new PatchRecord("System.Collections.Generic/PriorityQueue.cs", "R", "IEnumerable<(TElement, TPriority)>", "IEnumerable<(TElement Element, TPriority Priority)>"),
+        new PatchRecord("System/Nullable.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System/ReadOnlySpan.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.InteropServices/Marshal.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.InteropServices/MemoryMarshal.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.CompilerServices/Unsafe.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Numerics/Vector.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.Intrinsics/Vector64.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.Intrinsics/Vector128.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.Intrinsics/Vector256.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Runtime.Intrinsics/Vector512.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Threading/Volatile.cs", "R", "[RequiresLocation]", string.Empty),
+        new PatchRecord("System.Threading/Interlocked.cs", "R", "[RequiresLocation]", string.Empty)
+    ];
+
+    public static string Patch(PatchRecord patch, string path = "")
+	{
+		var file = !string.IsNullOrEmpty(path) ? Path.Combine(path, patch.File) : patch.File;
+
+		switch (patch.Mode)
+        {
+            case "rs":
+            {
+                var code = File.ReadAllLines(file).ToList();
+                var temp = code[..];
+
+                foreach (var line in temp)
+                    if (line.StartsWith(patch.Str1))
+						code.Remove(line);
+
+                return string.Join('\n', code);
+            }
+            case "R": return File.ReadAllText(file).Replace(patch.Str1, patch.Str2);
+        }
+
+        throw new UnreachableException();
+    }
+}

Source/Mosa.Tool.CreateCoreLib/Program.cs

@@ -0,0 +1,124 @@
+using System.Diagnostics;
+using CommandLine;
+using ICSharpCode.Decompiler.CSharp.ProjectDecompiler;
+using ICSharpCode.Decompiler.Metadata;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Mosa.Tool.CreateCoreLib;
+
+var options = Parser.Default.ParseArguments<Options>(args).Value;
+if (options == null) return;
+
+var outputDirectory = options.OutputDirectory;
+var copyFiles = options.CopyFiles;
+
+if (string.IsNullOrEmpty(outputDirectory))
+{
+    Console.WriteLine("Invalid output directory. Usage: Mosa.Tool.CreateCoreLib --output/-o <directory>");
+    return;
+}
+
+Directory.CreateDirectory(outputDirectory);
+
+if (!Directory.Exists("runtime"))
+{
+	Console.WriteLine("Cloning .NET runtime GitHub repository...");
+	Process.Start("git", "clone https://github.com/dotnet/runtime -b release/8.0 --depth=1")?.WaitForExit();
+}
+
+Console.WriteLine(copyFiles ? "Copying files..." : "Parsing input files...");
+
+var syntaxTrees = new List<SyntaxTree>();
+
+foreach (var folder in Directory.EnumerateDirectories(Path.Combine("runtime", "src", "libraries")))
+{
+	// We don't need those namespaces so we exclude them.
+    if (folder.Contains("Microsoft.Bcl.") || folder.Contains("Microsoft.Extensions.")) continue;
+
+    var refDirectory = Path.Combine(folder, "ref");
+    if (!Directory.Exists(refDirectory))
+		continue;
+
+    foreach (var file in Directory.GetFiles(refDirectory, "*.cs"))
+    {
+		// All of these files contain "type forwards", which basically forward certain types to other assemblies.
+		// Since we have a monolithic assembly, we can safely ignore those (though we'll still have to patch a few).
+		// They also cause compilation errors because they're not designed to be compiled, so removing them is a must.
+        if (file.EndsWith(".Forwards.cs") || file.EndsWith(".netframework.cs") || file.EndsWith(".TypeForwards.cs")
+            || file.Contains(".Typeforwards."))
+			continue;
+
+        var fileName = Path.GetFileName(file);
+        string? text = null;
+
+        var patch = Patcher.FirstPatches.FirstOrDefault(x => file.EndsWith(x.File));
+        if (patch != default)
+        {
+            Console.WriteLine($"Patching {fileName}...");
+            text = Patcher.Patch(patch);
+        }
+
+        if (copyFiles)
+        {
+            var outputPath = Path.Combine(outputDirectory, fileName);
+
+            if (text != null)
+                File.WriteAllText(outputPath, text);
+            else
+                File.Copy(file, outputPath, true);
+
+            continue;
+        }
+
+        text ??= File.ReadAllText(file);
+        syntaxTrees.Add(CSharpSyntaxTree.ParseText(text, new CSharpParseOptions(preprocessorSymbols: ["NETCOREAPP"])));
+    }
+}
+
+if (copyFiles) return;
+
+var compilation = CSharpCompilation.Create("System.Runtime", syntaxTrees, null,
+    new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: true));
+var outputFile = compilation.AssemblyName + ".dll";
+
+Console.WriteLine("Compiling source files...");
+
+using (var stream = File.OpenWrite("System.Runtime.dll"))
+{
+    var result = compilation.Emit(stream);
+    if (!result.Success)
+    {
+        var failures = result.Diagnostics
+            .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error);
+
+        foreach (var diagnostic in failures)
+			Console.WriteLine($"{diagnostic.Id}: {diagnostic.GetMessage()}");
+
+        return;
+    }
+}
+
+Console.WriteLine("Decompiling assembly...");
+
+var module = new PEFile(outputFile);
+var resolver = new UniversalAssemblyResolver(outputFile, true, module.Metadata.DetectTargetFrameworkId());
+var decompiler = new WholeProjectDecompiler(resolver);
+
+decompiler.DecompileProject(module, outputDirectory);
+
+// We don't need the project file because we should already have one. Even if it's missing, it's trivial to create.
+Console.WriteLine("Removing project file...");
+File.Delete(Path.Combine(outputDirectory, Path.ChangeExtension(outputFile, "csproj")));
+
+// The assembly will generate its own assembly information at compile time, so we don't need to pre-define it.
+Console.WriteLine("Removing Properties folder...");
+Directory.Delete(Path.Combine(outputDirectory, "Properties"), true);
+
+foreach (var patch in Patcher.SecondPatches)
+{
+    var file = Path.Combine(outputDirectory, patch.File);
+    Console.WriteLine($"Patching {patch.File}...");
+
+    var code = Patcher.Patch(patch, outputDirectory);
+    File.WriteAllText(file, code);
+}