LazyEncodedString everywhere (#247)

* Massive update to support LazyEncodedString Everywhere
* Not complete!
* This compiles the interop and wrapper libraries only, no impacted libs are updated yet.

* Completed changes with updates to consumers
* Includes additional testing

* More changes to support LazyEncodedString everywhere.

* Updated doc comments on `DisposeMEssageMarshaller type for clarity

* Attempting to work around problems in JIT on Windows x64.
* Apparently the JIT is assuming a 32 bit relocation as the default.
    - See llvm/llvm-project#65641
* This adds support for JIT builder and, in particular, TargetMachineBuilder so that the CodeModel is explicit in children
* Static usings are now explicit everywhere they are needed.
    - This removes static usings from the global usings.

* Additional support for LazyEncodedString
* Converted name of static class for LLVM-C interop `Error` for consistency
* Added marshaling form of `LLVMCreateStringError()`
* Added `LLVMErrorRef.CreateForNativeOut()`
    - Eliminates need for annoying IDisposable warning suppression in many cases.
    - This support out/return of native handle for reverse P/Invoke scenarios.
* Added setting the triple to the module transform for Kaleidoscope
    - Ensures the triple is set and matches the JIT.
* Added `MarshalGCHandle.TryGet<>()` to support more intuitive acquisition of a managed context in native callbacks.
* Moved static factory for `ErrorInfo` to a constructor.
   - Static factory was used for out/return to native but that causes a lot of suppression of warnings for dispose so isn't used. Instead, most APIs will use the static methods on LLVMErrorRef to report any errors. If the callback includes calling through a delegate, then it might use the constructor to build the resulting `ErrorInfo` to pass to native code.

* Updated version checks in interop code
* Since native API surface changed to support this build the version bump is needed

* Additional support for LazyEncodedString
* Added LibLLVMAttributeAllowedOn_All to make testing easier
* Changed signature of LibLLVMGetAttributeInfo to use an out param as the result instead of a raw pointer
    - Wrapper layout is identical to the interop form so a simple cast through a pointer is all that is needed to "project"
* Doc comment updates/fixes

* [Build Fix] Corrected test
* Test file was removed locally and missed
    - commit only included staged files so this edit and replace got in the way.

---------

Co-authored-by: smaillet <[email protected]>

Author: smaillet

.editorconfig

@@ -30,7 +30,7 @@ csharp_style_conditional_delegate_call = true:warning
 csharp_style_expression_bodied_accessors = true:suggestion
 csharp_style_expression_bodied_indexers = true:suggestion
 csharp_style_expression_bodied_methods = true:suggestion
-csharp_style_expression_bodied_operators = true:suggestion
+csharp_style_expression_bodied_operators = when_on_single_line:silent
 csharp_style_expression_bodied_properties = true:warning
 csharp_style_inlined_variable_declaration = true:warning
 csharp_style_pattern_matching_over_as_with_null_check = true:warning
@@ -89,6 +89,8 @@ csharp_prefer_simple_default_expression = true:error
 csharp_space_between_method_declaration_parameter_list_parentheses = true
 csharp_space_between_method_declaration_empty_parameter_list_parentheses = true
 dotnet_diagnostic.SA0001.severity = none
+csharp_style_unused_value_assignment_preference = discard_variable:suggestion
+csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true:silent
 
 # Analysis and refactoring rules for Ubiquity.NET
 # Description: Code analysis rules for Ubiquity.NET projects
@@ -1594,3 +1596,4 @@ dotnet_style_prefer_simplified_interpolation = true:suggestion
 dotnet_style_prefer_collection_expression = when_types_loosely_match:suggestion
 dotnet_style_namespace_match_folder = true:suggestion
 dotnet_diagnostic.MSTEST0032.severity = none
+dotnet_code_quality_unused_parameters = all:suggestion

IgnoredWords.dic

@@ -9,15 +9,18 @@ app
 apps
 argkind
 ascii
+atomicrmw
 attrib
 Attribs
 AttributeSet
+Attrs
 baz
 binaryop
 binlogs
 binop
 bitcode
 Blinky
+blittable
 blockdiag
 blog
 bool
@@ -26,8 +29,13 @@ buildtransitive
 builtinop
 byref
 byval
+castable
 cibuild
 Cmp
+Comdat
+Comdats
+compat
+Concat
 Config
 const
 contentfiles
@@ -41,14 +49,17 @@ dllimport
 docfx
 docfxconsole
 dotnet
+endianess
 enum
 Enums
 env
 exe
 facepalm
+finalizers
 foo
 fullsrc
 func
+getelementptr
 gh
 github
 Globalization
@@ -64,15 +75,18 @@ Llilum
 llvm
 llvmversion
 LValue
+marshallers
 marshalling
 memcopy
 metadata
 Mips
 msbuild
 msg
 nav
+nint
 noinline
 nounwind
+nuint
 nullability
 Nullable
 optimizenone
@@ -84,25 +98,34 @@ Pre
 proj
 readonly
 refactor
+refcount
 repl
 repo
 RMW
 runtimes
 RValue
+sizeof
 src
+Stateful
 struct
 structs
 Subrange
+Sym
 telliam
 tl
 trx
 typdef
 Typedef
+typeof
 uid
+uint
 unaryop
+Undefine
 undiscoverable
+Unhandled
 uniqued
 uniqueing
+unmarshal
 userdefinedop
 Users
 usings
@@ -112,6 +135,8 @@ vcxproj
 versioned
 versioning
 Xchg
+Xor
 xref
+xxx
 yaml
 yyyy

docfx/templates/Ubiquity/readme.md

@@ -1,8 +1,19 @@
 # Ubiquity DOCFX template
 This template adds support to the syntax highlighting provided by [HightlightJS](https://highlightjs.readthedocs.io/en/latest/supported-languages.html).
-The languages added are for ANTLR (Which seems bizarre it isn't already covered given the
-esoteric nature of some of the supported languages...) and of course the `Kaleidoscope`
-language, which was made up entirely for the purposes of LLVM tutorials. (No surprise that one
-isn't supported) [Though it oddly IS supported directly in the [Local MarkDig
-based editor](https://github.com/MadsKristensen/MarkdownEditor2022) used to edit these files...
+The languages added are for ANTLR (Which seems bizarre it isn't already covered given
+the esoteric nature of some of the supported languages...) and of course the `Kaleidoscope`
+language, which was made up entirely for the purposes of LLVM tutorials. (No surprise that
+one isn't supported) [Though it oddly IS supported directly in the [Local MarkDig based
+editor](https://github.com/MadsKristensen/MarkdownEditor2022) used to edit these files...
 
+## Theming
+This template also updates the theme for SVG image backgrounds to improve readability
+of the parse diagrams. It also updates the HighlightJS classifiers theming to better
+show various parts of the ANTLR language.
+
+The default theming is limited, especially for the ANTLR language, as it doesn't
+support distinction between function definitions and invocations. (Both have
+essentially the default text formatting so it's like none is applied). HighlightJS has
+rather limited "scopes" and mapping to them for a language like ANTLR4 Lex/Grammar is
+a challenge. It is NOT an imperative language (but it does generate to that) but the
+HighlightJS is mostly focused on those.

src/Interop/InteropTests/AttributeTests.cs

@@ -28,48 +28,41 @@ public class AttributeTests
         public void CustomAttributeTests( )
         {
             using LLVMContextRef ctx = LLVMContextCreate();
-            LazyEncodedString name = new("custom");
-            LazyEncodedString value = new("custom value");
+            LazyEncodedString name = new("custom"u8);
+            LazyEncodedString value = "custom value"u8;
 
-            LLVMAttributeRef attribValue;
-
-            using var memName = name.Pin();
-            using var memValue = value.Pin();
-            unsafe
-            {
-                attribValue = LLVMCreateStringAttribute(ctx, (byte*)memName.Pointer, (uint)name.NativeStrLen, (byte*)memValue.Pointer, (uint)value.NativeStrLen);
-            }
+            LLVMAttributeRef attribValue = LLVMCreateStringAttribute(ctx, name, value);
 
             Assert.IsFalse(attribValue.IsNull);
 
             // LibLLVM should support Custom attributes as well.
-            LibLLVMAttributeInfo info;
             LazyEncodedString attribName = new("custom");
-            using var mem = attribName.Pin();
-            unsafe
-            {
-                using LLVMErrorRef errorRef = LibLLVMGetAttributeInfo((byte*)mem.Pointer, attribName.NativeStrLen, &info);
-                errorRef.ThrowIfFailed();
-            }
+            using LLVMErrorRef errorRef = LibLLVMGetAttributeInfo(attribName, out LibLLVMAttributeInfo info);
+            errorRef.ThrowIfFailed();
 
             Assert.AreEqual(LibLLVMAttributeArgKind.LibLLVMAttributeArgKind_String, info.ArgKind);
         }
 
         [TestMethod]
         public void AttributeListAttainable( )
         {
-            size_t len = LibLLVMGetNumKnownAttribs();
+            int len = checked((int)LibLLVMGetNumKnownAttribs());
             unsafe
             {
                 byte** ppData = stackalloc byte*[len];
-                using LLVMErrorRef errorRef = LibLLVMGetKnownAttributeNames(len, ppData);
+                using LLVMErrorRef errorRef = LibLLVMGetKnownAttributeNames(ppData, (nuint)len);
                 errorRef.ThrowIfFailed();
 
                 // make sure conversion is plausible.
                 var bldr = ImmutableArray.CreateBuilder<LazyEncodedString>(len);
                 for(int i=0; i < len; ++i)
                 {
-                    bldr.Add(new(ppData[i]));
+                    // https://github.com/microsoft/testfx/issues/5543
+#pragma warning disable MSTEST0037 // Use proper 'Assert' methods
+                    Assert.IsTrue(ppData[i] is not null);
+#pragma warning restore MSTEST0037 // Use proper 'Assert' methods
+
+                    bldr.Add( LazyEncodedString.FromUnmanaged(ppData[i])!);
                 }
 
                 var names = bldr.ToImmutable();

src/Interop/InteropTests/DataLayoutTests.cs

@@ -4,11 +4,11 @@
 // </copyright>
 // -----------------------------------------------------------------------
 
-using System;
 using System.Runtime.InteropServices;
 
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Ubiquity.NET.Llvm.Interop.ABI.libllvm_c;
+
+using Ubiquity.NET.InteropHelpers;
 
 using static Ubiquity.NET.Llvm.Interop.ABI.libllvm_c.DataLayoutBindings;
 
@@ -20,20 +20,14 @@ public class DataLayoutTests
         [TestMethod]
         public void TestParseKnownBad( )
         {
-            ReadOnlySpan<byte> utf8Span = "badlayout"u8;
-            unsafe
+            LazyEncodedString utf8Span = "badlayout"u8;
+            using var errorRef = LibLLVMParseDataLayout(utf8Span, out LLVMTargetDataRef retVal);
+            using(retVal)
             {
-                fixed(byte* p = &MemoryMarshal.GetReference(utf8Span))
-                {
-                    using var ErrorRef = LibLLVMParseDataLayout(p, utf8Span.Length, out LLVMTargetDataRef retVal);
-                    using(retVal)
-                    {
-                        Assert.IsTrue(retVal.IsInvalid);
-                        Assert.IsTrue(ErrorRef.Failed);
-                        string errMsg = ErrorRef.ToString();
-                        Assert.IsFalse(string.IsNullOrWhiteSpace(errMsg));
-                    }
-                }
+                Assert.IsTrue(retVal.IsInvalid);
+                Assert.IsTrue(errorRef.Failed);
+                string errMsg = errorRef.ToString();
+                Assert.IsFalse(string.IsNullOrWhiteSpace(errMsg), "Failure should have an error message");
             }
         }
     }

src/Interop/InteropTests/DebugRecordTests.cs

@@ -63,21 +63,20 @@ public void DebugRecordEnumerationSucceeds( )
             using LLVMDIBuilderRef diBuilder = LLVMCreateDIBuilder(module);
             LLVMMetadataRef int32DiType = LLVMDIBuilderCreateBasicType(
                 diBuilder,
-                "int32_t",
-                7,
+                "int32_t"u8,
                 32,
                 (uint)LibLLVMDwarfAttributeEncoding.DW_ATE_signed,
                 LLVMDIFlags.LLVMDIFlagPublic
             );
 
-            LLVMMetadataRef int32PtrDiType = LLVMDIBuilderCreatePointerType(diBuilder, int32DiType, 32, 0, 0, "int*", 4);
-            LLVMMetadataRef emptyExpression = LLVMDIBuilderCreateExpression(diBuilder, [], 0);
+            LLVMMetadataRef int32PtrDiType = LLVMDIBuilderCreatePointerType(diBuilder, int32DiType, 32, 0, 0, "int*"u8);
+            LLVMMetadataRef emptyExpression = LLVMDIBuilderCreateExpression(diBuilder, []);
             LLVMMetadataRef diFuncType = LLVMDIBuilderCreateSubroutineType(diBuilder, default, [], 0, LLVMDIFlags.LLVMDIFlagPrivate);
             LLVMMetadataRef scope = LLVMDIBuilderCreateFunction(
                 diBuilder,
                 Scope: default,
-                Name: "TestFunc", NameLen: 8,
-                LinkageName: "TestFunc", LinkageNameLen: 8,
+                Name: "TestFunc"u8,
+                LinkageName: "_TestFunc"u8,
                 File: default,
                 LineNo: 0,
                 Ty: diFuncType,

src/Interop/InteropTests/LibraryInitTests.cs

@@ -13,13 +13,14 @@
 using Ubiquity.NET.Llvm.Interop.ABI.libllvm_c;
 
 using static Ubiquity.NET.Llvm.Interop.ABI.llvm_c.TargetMachine;
+using Ubiquity.NET.InteropHelpers;
 
 namespace Ubiquity.NET.Llvm.Interop.UT
 {
     [TestClass]
     public class LibraryInitTests
     {
-        // Since these test validates the initialize and dispose of the the native library the
+        // Since these test validates the initialize and dispose of the native library the
         // native code is no longer usable in the process that runs this test. Therefore this
         // is set to use a distinct process. (Sadly, this doesn't work in the VS test explorer
         // so it is simply marked as "skipped" there. Command line dotnet test via the scripts
@@ -66,12 +67,13 @@ public void TestLibraryReInit( )
                 // NOTE: There are multiple actual targets under the one registered target
                 //       The registration is more along the lines of a "family" of targets...
                 Assert.AreEqual(4, targets.Length);
-                string?[] targetNames = [ .. targets.Select(h=>LLVMGetTargetName(h)) ];
+                LazyEncodedString?[] targetNames = [ .. targets.Select(h=>LLVMGetTargetName(h)) ];
                 // order of names/targets is not guaranteed, so just test for presence via Contains().
-                Assert.IsTrue(targetNames.Contains("thumbeb"));
-                Assert.IsTrue(targetNames.Contains("thumb"));
-                Assert.IsTrue(targetNames.Contains("armeb"));
-                Assert.IsTrue(targetNames.Contains("arm"));
+                // Also test that u8 literal is viable
+                Assert.IsTrue(targetNames.Contains<LazyEncodedString?>("thumbeb"u8));
+                Assert.IsTrue(targetNames.Contains<LazyEncodedString?>("thumb"));
+                Assert.IsTrue(targetNames.Contains<LazyEncodedString?>("armeb"));
+                Assert.IsTrue(targetNames.Contains<LazyEncodedString?>("arm"));
             }
 
             // After dispose - "that's all she wrote", LLVM native libraries