From 47645e68bb5671fda8865f9e9e9c1988c9883609 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 19 Mar 2025 13:51:16 +0100 Subject: [PATCH 01/41] Package CLR host static archives --- .../create-packs/Microsoft.Android.Runtime.proj | 15 +++++++++++++-- src/native/clr/host/CMakeLists.txt | 4 +--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 603124f0466..6d77d8d9054 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -50,8 +50,19 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - - + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a" /> diff --git a/src/native/clr/host/CMakeLists.txt b/src/native/clr/host/CMakeLists.txt index d53734eaf8c..73908f7fb85 100644 --- a/src/native/clr/host/CMakeLists.txt +++ b/src/native/clr/host/CMakeLists.txt @@ -3,7 +3,7 @@ include(CheckIncludeFile) include(CheckCXXSymbolExists) -set(BUILD_STATIC_LIBRARY OFF) +set(BUILD_STATIC_LIBRARY ON) if(DEBUG_BUILD) # Convince NDK to really optimize our Debug builds. Without this, NDK's cmake toolchain definition @@ -12,8 +12,6 @@ if(DEBUG_BUILD) set(XA_COMPILER_FLAGS_DEBUG "-fno-limit-debug-info -O2") set(CMAKE_C_FLAGS_DEBUG ${XA_COMPILER_FLAGS_DEBUG}) set(CMAKE_CXX_FLAGS_DEBUG ${XA_COMPILER_FLAGS_DEBUG}) -elseif(NOT ANALYZERS_ENABLED) - set(BUILD_STATIC_LIBRARY OFF) # Turn off for now, until we implement dynamic runtime linking at app build time endif() # Library directories From ae6d47bc38425820c69fbd5b8aeecb5d48e13708 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 19 Mar 2025 14:28:41 +0100 Subject: [PATCH 02/41] Package NDK redistributables into our runtime packs --- Configuration.props | 6 +++- .../Microsoft.Android.Runtime.proj | 10 +++++-- .../xaprepare/Application/KnownProperties.cs | 2 ++ .../Application/Properties.Defaults.cs.in | 4 ++- .../xaprepare/ConfigAndData/Configurables.cs | 9 ++++++ .../xaprepare/Steps/Step_Android_SDK_NDK.cs | 30 +++++++++++++++---- .../xaprepare/xaprepare/xaprepare.targets | 2 ++ 7 files changed, 54 insertions(+), 9 deletions(-) diff --git a/Configuration.props b/Configuration.props index 3f7434a0ae0..99e44109931 100644 --- a/Configuration.props +++ b/Configuration.props @@ -228,10 +228,14 @@ AndroidRuntime="CoreCLR" /> + - <_RuntimeRedistDirName>redist + <_MonoRuntimeFlavorDirName>mono + <_CLRRuntimeFlavorDirName>clr + + + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index 6c3a48b3131..b5313528959 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -258,6 +258,7 @@ _ResolveAssemblies MSBuild target. + + + + + + + <_UnifiedNativeRuntime Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libnetdroid-unified.so"> + %(_BuildTargetAbis.Identity) + libmonodroid.so + + + + + <_ResolvedNativeArchive Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.a' " /> + <_ResolvedNativeObjectFile Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.o' " /> + <_ApplicationSharedLibrary Include="@(_UnifiedNativeRuntime)" /> + + + + + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs index 71e29ee02aa..25f2e506e74 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs @@ -110,7 +110,11 @@ void AddRuntimeLibraries (PackageFileListBuilder apk, string [] supportedAbis) foreach (ITaskItem item in ApplicationSharedLibraries) { if (string.Compare (abi, item.GetMetadata ("abi"), StringComparison.Ordinal) != 0) continue; - AddNativeLibraryToArchive (apk, abi, item.ItemSpec, Path.GetFileName (item.ItemSpec), item); + string? inArchiveFileName = item.GetMetadata ("ArchiveFileName"); + if (String.IsNullOrEmpty (inArchiveFileName)) { + inArchiveFileName = Path.GetFileName (item.ItemSpec); + } + AddNativeLibraryToArchive (apk, abi, item.ItemSpec, inArchiveFileName, item); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 31818e8cafc..0b3e6d86757 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -58,6 +58,8 @@ public class GenerateJavaStubs : AndroidTask public string CodeGenerationTarget { get; set; } = ""; + public bool EnableNativeRuntimeLinking { get; set; } + // These two properties are temporary and are used to ensure we still generate the // same files as before using the new _LinkAssembliesNoShrink JLO scanning. They will be removed in the future. public bool RunCheckedBuild { get; set; } @@ -156,6 +158,7 @@ void Run (bool useMarshalMethods) // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture var nativeCodeGenStates = new ConcurrentDictionary (); NativeCodeGenState? templateCodeGenState = null; + PinvokeScanner? pinvokeScanner = EnableNativeRuntimeLinking ? new PinvokeScanner (Log) : null; var firstArch = allAssembliesPerArch.First ().Key; var generateSucceeded = true; @@ -181,6 +184,15 @@ void Run (bool useMarshalMethods) } nativeCodeGenStates.TryAdd (arch, state); + + if (pinvokeScanner != null && state != null) { + (success, List pinfos) = ScanForUsedPinvokes (pinvokeScanner, arch, state.Resolver); + if (!success) { + return; + } + state.PinvokeInfos = pinfos; + Log.LogDebugMessage ($"Number of unique p/invokes for architecture '{arch}': {pinfos.Count}"); + } }); // If we hit an error generating the Java code, we should bail out now @@ -200,6 +212,31 @@ void Run (bool useMarshalMethods) BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build); } + (bool success, List? pinfos) ScanForUsedPinvokes (PinvokeScanner scanner, AndroidTargetArch arch, XAAssemblyResolver resolver) + { + if (!EnableNativeRuntimeLinking) { + return (true, null); + } + + var frameworkAssemblies = new List (); + + foreach (ITaskItem asm in ResolvedAssemblies) { + string? metadata = asm.GetMetadata ("FrameworkAssembly"); + if (String.IsNullOrEmpty (metadata)) { + continue; + } + + if (!Boolean.TryParse (metadata, out bool isFrameworkAssembly) || !isFrameworkAssembly) { + continue; + } + + frameworkAssemblies.Add (asm); + } + + var pinfos = scanner.Scan (arch, resolver, frameworkAssemblies); + return (true, pinfos); + } + internal static Dictionary MaybeGetArchAssemblies (Dictionary> dict, AndroidTargetArch arch) { if (!dict.TryGetValue (arch, out Dictionary archDict)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index a82c4c0408f..3c0980a3436 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -30,10 +30,243 @@ public override bool RunTask () pkgmgr.WriteLine ("public class MonoPackageManager_Resources {"); pkgmgr.WriteLine ("\tpublic static String[] Assemblies = new String[]{"); - pkgmgr.WriteLine ("\t\t/* We need to ensure that \"{0}\" comes first in this list. */", mainFileName); - pkgmgr.WriteLine ("\t\t\"" + mainFileName + "\","); - foreach (var assembly in ResolvedUserAssemblies) { - if (string.Compare (Path.GetFileName (assembly.ItemSpec), mainFileName, StringComparison.OrdinalIgnoreCase) == 0) + public ITaskItem[] NativeLibraries { get; set; } + + public ITaskItem[] MonoComponents { get; set; } + + public ITaskItem[] SatelliteAssemblies { get; set; } + + public bool UseAssemblyStore { get; set; } + + [Required] + public string OutputDirectory { get; set; } + + [Required] + public string EnvironmentOutputDirectory { get; set; } + + [Required] + public string IntermediateOutputDirectory { get; set; } = ""; + + [Required] + public string MainAssembly { get; set; } + + [Required] + public string TargetFrameworkVersion { get; set; } + + [Required] + public string Manifest { get; set; } + + [Required] + public string [] SupportedAbis { get; set; } + + [Required] + public string AndroidPackageName { get; set; } + + [Required] + public bool EnablePreloadAssembliesDefault { get; set; } + + [Required] + public bool TargetsCLR { get; set; } + + public bool EnableMarshalMethods { get; set; } + public bool EnableManagedMarshalMethodsLookup { get; set; } + public string RuntimeConfigBinFilePath { get; set; } + public string ProjectRuntimeConfigFilePath { get; set; } = String.Empty; + public string BoundExceptionType { get; set; } + + public string PackageNamingPolicy { get; set; } + public string Debug { get; set; } + public ITaskItem[] Environments { get; set; } + public string AndroidAotMode { get; set; } + public bool AndroidAotEnableLazyLoad { get; set; } + public bool EnableLLVM { get; set; } + public string HttpClientHandlerType { get; set; } + public string TlsProvider { get; set; } + public string AndroidSequencePointsMode { get; set; } + public bool EnableSGenConcurrent { get; set; } + public string? CustomBundleConfigFile { get; set; } + public bool EnableNativeRuntimeLinking { get; set; } + + bool _Debug { + get { + return string.Equals (Debug, "true", StringComparison.OrdinalIgnoreCase); + } + } + + public override bool RunTask () + { + var doc = AndroidAppManifest.Load (Manifest, MonoAndroidHelper.SupportedVersions); + int minApiVersion = doc.MinSdkVersion == null ? 4 : (int) doc.MinSdkVersion; + // We need to include any special assemblies in the Assemblies list + var mainFileName = Path.GetFileName (MainAssembly); + + using (var pkgmgr = MemoryStreamPool.Shared.CreateStreamWriter ()) { + pkgmgr.WriteLine ("package mono;"); + + // Write all the user assemblies + pkgmgr.WriteLine ("public class MonoPackageManager_Resources {"); + pkgmgr.WriteLine ("\tpublic static String[] Assemblies = new String[]{"); + + pkgmgr.WriteLine ("\t\t/* We need to ensure that \"{0}\" comes first in this list. */", mainFileName); + pkgmgr.WriteLine ("\t\t\"" + mainFileName + "\","); + foreach (var assembly in ResolvedUserAssemblies) { + if (string.Compare (Path.GetFileName (assembly.ItemSpec), mainFileName, StringComparison.OrdinalIgnoreCase) == 0) + continue; + pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly.ItemSpec) + "\","); + } + + // Write the assembly dependencies + pkgmgr.WriteLine ("\t};"); + pkgmgr.WriteLine ("\tpublic static String[] Dependencies = new String[]{"); + + //foreach (var assembly in assemblies.Except (args.Assemblies)) { + // if (args.SharedRuntime && !Toolbox.IsInSharedRuntime (assembly)) + // pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly) + "\","); + //} + + pkgmgr.WriteLine ("\t};"); + + pkgmgr.WriteLine ("}"); + pkgmgr.Flush (); + + // Only copy to the real location if the contents actually changed + var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); + + Files.CopyIfStreamChanged (pkgmgr.BaseStream, dest); + } + + AddEnvironment (); + + return !Log.HasLoggedErrors; + } + + static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) => MonoAndroidHelper.AbiToTargetArch (abi); + + static readonly string[] defaultLogLevel = {"MONO_LOG_LEVEL", "info"}; + static readonly string[] defaultMonoDebug = {"MONO_DEBUG", "gen-compact-seq-points"}; + static readonly string[] defaultHttpMessageHandler = {"XA_HTTP_CLIENT_HANDLER_TYPE", "System.Net.Http.HttpClientHandler, System.Net.Http"}; + static readonly string[] defaultTlsProvider = {"XA_TLS_PROVIDER", "btls"}; + + void AddEnvironment () + { + bool usesMonoAOT = false; + var environmentVariables = new Dictionary (StringComparer.Ordinal); + var systemProperties = new Dictionary (StringComparer.Ordinal); + + if (!Enum.TryParse (PackageNamingPolicy, out PackageNamingPolicy pnp)) { + pnp = PackageNamingPolicyEnum.LowercaseCrc64; + } + + AotMode aotMode = AotMode.None; + if (!string.IsNullOrEmpty (AndroidAotMode) && Aot.GetAndroidAotMode (AndroidAotMode, out aotMode) && aotMode != AotMode.None) { + usesMonoAOT = true; + } + + SequencePointsMode sequencePointsMode; + if (!Aot.TryGetSequencePointsMode (AndroidSequencePointsMode, out sequencePointsMode)) + sequencePointsMode = SequencePointsMode.None; + + // Even though environment files were potentially parsed in GenerateJavaStubs, we need to do it here again because we might have additional environment + // files (generated by us) which weren't present by the time GeneratJavaStubs ran. + var environmentParser = new EnvironmentFilesParser { + BrokenExceptionTransitions = false, + UsesAssemblyPreload = EnablePreloadAssembliesDefault, + }; + environmentParser.Parse (Environments, sequencePointsMode, Log); + + foreach (string line in environmentParser.EnvironmentVariableLines) { + AddEnvironmentVariableLine (line); + } + + if (_Debug && !environmentParser.HaveLogLevel) { + AddEnvironmentVariable (defaultLogLevel[0], defaultLogLevel[1]); + } + + if (sequencePointsMode != SequencePointsMode.None && !environmentParser.HaveMonoDebug) { + AddEnvironmentVariable (defaultMonoDebug[0], defaultMonoDebug[1]); + } + + if (!environmentParser.HaveHttpMessageHandler) { + if (HttpClientHandlerType == null) + AddEnvironmentVariable (defaultHttpMessageHandler[0], defaultHttpMessageHandler[1]); + else + AddEnvironmentVariable ("XA_HTTP_CLIENT_HANDLER_TYPE", HttpClientHandlerType.Trim ()); + } + + if (!environmentParser.HaveMonoGCParams) { + if (EnableSGenConcurrent) + AddEnvironmentVariable ("MONO_GC_PARAMS", "major=marksweep-conc"); + else + AddEnvironmentVariable ("MONO_GC_PARAMS", "major=marksweep"); + } + + global::Android.Runtime.BoundExceptionType boundExceptionType; + if (String.IsNullOrEmpty (BoundExceptionType) || String.Compare (BoundExceptionType, "System", StringComparison.OrdinalIgnoreCase) == 0) { + boundExceptionType = global::Android.Runtime.BoundExceptionType.System; + } else if (String.Compare (BoundExceptionType, "Java", StringComparison.OrdinalIgnoreCase) == 0) { + boundExceptionType = global::Android.Runtime.BoundExceptionType.Java; + } else { + throw new InvalidOperationException ($"Unsupported BoundExceptionType value '{BoundExceptionType}'"); + } + + int assemblyNameWidth = 0; + Encoding assemblyNameEncoding = Encoding.UTF8; + + Action updateNameWidth = (ITaskItem assembly) => { + if (UseAssemblyStore) { + return; + } + + string assemblyName = Path.GetFileName (assembly.ItemSpec); + int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; + if (nameBytes > assemblyNameWidth) { + assemblyNameWidth = nameBytes; + } + }; + + int assemblyCount = 0; + bool enableMarshalMethods = EnableMarshalMethods; + HashSet archAssemblyNames = null; + HashSet uniqueAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); + Action updateAssemblyCount = (ITaskItem assembly) => { + string? culture = MonoAndroidHelper.GetAssemblyCulture (assembly); + string fileName = Path.GetFileName (assembly.ItemSpec); + string assemblyName; + + if (String.IsNullOrEmpty (culture)) { + assemblyName = fileName; + } else { + assemblyName = $"{culture}/{fileName}"; + } + + if (!uniqueAssemblyNames.Contains (assemblyName)) { + uniqueAssemblyNames.Add (assemblyName); + } + + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); + + if (!archAssemblyNames.Contains (assemblyName)) { + assemblyCount++; + archAssemblyNames.Add (assemblyName); + } + }; + + if (SatelliteAssemblies != null) { + foreach (ITaskItem assembly in SatelliteAssemblies) { + updateNameWidth (assembly); + updateAssemblyCount (assembly); + } + } + + int android_runtime_jnienv_class_token = -1; + int jnienv_initialize_method_token = -1; + int jnienv_registerjninatives_method_token = -1; + foreach (var assembly in ResolvedAssemblies) { + updateNameWidth (assembly); + updateAssemblyCount (assembly); + + if (android_runtime_jnienv_class_token != -1) { continue; pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly.ItemSpec) + "\","); } @@ -52,10 +285,162 @@ public override bool RunTask () pkgmgr.WriteLine ("}"); pkgmgr.Flush (); - // Only copy to the real location if the contents actually changed - var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); + ConcurrentDictionary? nativeCodeGenStates = null; + if (enableMarshalMethods || EnableNativeRuntimeLinking) { + nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( + MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), + RegisteredTaskObjectLifetime.Build + ); + } + + bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); + var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); + LLVMIR.LlvmIrComposer appConfigAsmGen; + + if (TargetsCLR) { + Dictionary? runtimeProperties = RuntimePropertiesParser.ParseConfig (ProjectRuntimeConfigFilePath); + appConfigAsmGen = new ApplicationConfigNativeAssemblyGeneratorCLR (environmentVariables, systemProperties, runtimeProperties, Log) { + UsesAssemblyPreload = environmentParser.UsesAssemblyPreload, + AndroidPackageName = AndroidPackageName, + PackageNamingPolicy = pnp, + JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent, + NumberOfAssembliesInApk = assemblyCount, + BundledAssemblyNameWidth = assemblyNameWidth, + NativeLibraries = uniqueNativeLibraries, + AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, + JNIEnvInitializeToken = jnienv_initialize_method_token, + JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, + JniRemappingReplacementTypeCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementTypeCount, + JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, + MarshalMethodsEnabled = EnableMarshalMethods, + ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, + IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), + }; + } else { + appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { + UsesMonoAOT = usesMonoAOT, + UsesMonoLLVM = EnableLLVM, + UsesAssemblyPreload = environmentParser.UsesAssemblyPreload, + MonoAOTMode = aotMode.ToString ().ToLowerInvariant (), + AotEnableLazyLoad = AndroidAotEnableLazyLoad, + AndroidPackageName = AndroidPackageName, + BrokenExceptionTransitions = environmentParser.BrokenExceptionTransitions, + PackageNamingPolicy = pnp, + BoundExceptionType = boundExceptionType, + JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent, + HaveRuntimeConfigBlob = haveRuntimeConfigBlob, + NumberOfAssembliesInApk = assemblyCount, + BundledAssemblyNameWidth = assemblyNameWidth, + MonoComponents = (MonoComponent)monoComponents, + NativeLibraries = uniqueNativeLibraries, + HaveAssemblyStore = UseAssemblyStore, + AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, + JNIEnvInitializeToken = jnienv_initialize_method_token, + JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, + JniRemappingReplacementTypeCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementTypeCount, + JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, + MarshalMethodsEnabled = EnableMarshalMethods, + ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, + IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), + }; + } + LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); - Files.CopyIfStreamChanged (pkgmgr.BaseStream, dest); + foreach (string abi in SupportedAbis) { + string targetAbi = abi.ToLowerInvariant (); + string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}"); + string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); + string? pinvokePreserveBaseAsmFilePath = EnableNativeRuntimeLinking ? Path.Combine (EnvironmentOutputDirectory, $"pinvoke_preserve.{targetAbi}") : null; + string environmentLlFilePath = $"{environmentBaseAsmFilePath}.ll"; + string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; + string? pinvokePreserveLlFilePath = pinvokePreserveBaseAsmFilePath != null ? $"{pinvokePreserveBaseAsmFilePath}.ll" : null; + AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); + + using var appConfigWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + appConfigAsmGen.Generate (appConfigModule, targetArch, appConfigWriter, environmentLlFilePath); + } catch { + throw; + } finally { + appConfigWriter.Flush (); + Files.CopyIfStreamChanged (appConfigWriter.BaseStream, environmentLlFilePath); + } + + MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; + if (enableMarshalMethods) { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + Log, + assemblyCount, + uniqueAssemblyNames, + EnsureCodeGenState (targetArch), + EnableManagedMarshalMethodsLookup + ); + } else { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + Log, + targetArch, + assemblyCount, + uniqueAssemblyNames + ); + } + + if (EnableNativeRuntimeLinking) { + var pinvokePreserveGen = new PreservePinvokesNativeAssemblyGenerator (Log, EnsureCodeGenState (targetArch), MonoComponents); + LLVMIR.LlvmIrModule pinvokePreserveModule = pinvokePreserveGen.Construct (); + using var pinvokePreserveWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + pinvokePreserveGen.Generate (pinvokePreserveModule, targetArch, pinvokePreserveWriter, pinvokePreserveLlFilePath); + } catch { + throw; + } finally { + pinvokePreserveWriter.Flush (); + Files.CopyIfStreamChanged (pinvokePreserveWriter.BaseStream, pinvokePreserveLlFilePath); + } + } + + LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); + using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, marshalMethodsWriter, marshalMethodsLlFilePath); + } catch { + throw; + } finally { + marshalMethodsWriter.Flush (); + Files.CopyIfStreamChanged (marshalMethodsWriter.BaseStream, marshalMethodsLlFilePath); + } + } + + NativeCodeGenStateObject EnsureCodeGenState (AndroidTargetArch targetArch) + { + if (nativeCodeGenStates == null || !nativeCodeGenStates.States.TryGetValue (targetArch, out NativeCodeGenStateObject? state)) { + throw new InvalidOperationException ($"Internal error: missing native code generation state for architecture '{targetArch}'"); + } + + return state; + } + + void AddEnvironmentVariable (string name, string value) + { + if (Char.IsUpper(name [0]) || !Char.IsLetter(name [0])) + environmentVariables [ValidAssemblerString (name)] = ValidAssemblerString (value); + else + systemProperties [ValidAssemblerString (name)] = ValidAssemblerString (value); + } + + void AddEnvironmentVariableLine (string l) + { + string line = l?.Trim (); + if (String.IsNullOrEmpty (line) || line [0] == '#') + return; + + string[] nv = line.Split (new char[]{'='}, 2); + AddEnvironmentVariable (nv[0].Trim (), nv.Length < 2 ? String.Empty : nv[1].Trim ()); + } + + string ValidAssemblerString (string s) + { + return s.Replace ("\"", "\\\""); + } } return !Log.HasLoggedErrors; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs new file mode 100644 index 00000000000..0f6587035f0 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -0,0 +1,131 @@ +using System; +using System.IO; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class GetNativeRuntimeComponents : AndroidTask +{ + public override string TaskPrefix => "GNRC"; + + public ITaskItem[] MonoComponents { get; set; } + + [Required] + public ITaskItem[] ResolvedNativeArchives { get; set; } + + [Required] + public ITaskItem[] ResolvedNativeObjectFiles { get; set; } + + [Output] + public ITaskItem[] NativeArchives { get; set; } + + [Output] + public ITaskItem[] RequiredLibraries { get; set; } + + [Output] + public ITaskItem[] LinkStartFiles { get; set; } + + [Output] + public ITaskItem[] LinkEndFiles { get; set; } + + // TODO: more research, for now it seems `--export-dynamic-symbol=name` options generated from + // this array don't work as expected. + [Output] + public ITaskItem[] NativeSymbolsToExport { get; set; } + + public override bool RunTask () + { + var components = new NativeRuntimeComponents (MonoComponents); + var uniqueAbis = new HashSet (StringComparer.OrdinalIgnoreCase); + var archives = new List (); + var symbolsToExport = new List (); + + foreach (NativeRuntimeComponents.Archive archiveItem in components.KnownArchives) { + if (!archiveItem.Include) { + continue; + } + MakeArchiveItem (archiveItem, archives, uniqueAbis); + if (archiveItem.SymbolsToPreserve == null || archiveItem.SymbolsToPreserve.Count == 0) { + continue; + } + + foreach (string symbolName in archiveItem.SymbolsToPreserve) { + MakeLibItem (symbolName, symbolsToExport, uniqueAbis); + } + } + NativeArchives = archives.ToArray (); + NativeSymbolsToExport = symbolsToExport.ToArray (); + + var items = new List (); + foreach (string lib in components.NativeLibraries) { + MakeLibItem (lib, items, uniqueAbis); + } + RequiredLibraries = items.ToArray (); + + items = new List (); + foreach (string startFile in components.LinkStartFiles) { + MakeFileItem ("_NativeLinkStartFiles", startFile, ResolvedNativeObjectFiles, items, uniqueAbis); + } + LinkStartFiles = items.ToArray (); + + items = new List (); + foreach (string endFile in components.LinkEndFiles) { + MakeFileItem ("_NativeLinkEndFiles", endFile, ResolvedNativeObjectFiles, items, uniqueAbis); + } + LinkEndFiles = items.ToArray (); + + return !Log.HasLoggedErrors; + } + + void MakeLibItem (string libName, List libraries, HashSet uniqueAbis) + { + foreach (string abi in uniqueAbis) { + var item = new TaskItem (libName); + item.SetMetadata (KnownMetadata.Abi, abi); + libraries.Add (item); + } + } + + void MakeFileItem (string msbuildItemName, string fileName, ITaskItem[] inputItems, List outputItems, HashSet uniqueAbis) + { + foreach (ITaskItem item in inputItems) { + string name = Path.GetFileName (item.ItemSpec); + if (String.Compare (name, fileName, StringComparison.OrdinalIgnoreCase) == 0) { + outputItems.Add (DoMakeItem (msbuildItemName, item, uniqueAbis)); + } + } + } + + void MakeArchiveItem (NativeRuntimeComponents.Archive archive, List archives, HashSet uniqueAbis) + { + foreach (ITaskItem resolvedArchive in ResolvedNativeArchives) { + string fileName = Path.GetFileName (resolvedArchive.ItemSpec); + if (String.Compare (fileName, archive.Name, StringComparison.OrdinalIgnoreCase) != 0) { + continue; + } + + ITaskItem newItem = DoMakeItem ("_ResolvedNativeArchive", resolvedArchive, uniqueAbis); + newItem.SetMetadata (KnownMetadata.NativeLinkWholeArchive, archive.WholeArchive.ToString ()); + if (archive.DontExportSymbols) { + newItem.SetMetadata (KnownMetadata.NativeDontExportSymbols, "true"); + } + archives.Add (newItem); + } + } + + ITaskItem DoMakeItem (string msbuildItemName, ITaskItem sourceItem, HashSet uniqueAbis) + { + var ret = new TaskItem (sourceItem.ItemSpec); + string rid = sourceItem.GetRequiredMetadata (msbuildItemName, KnownMetadata.RuntimeIdentifier, Log); + string abi = MonoAndroidHelper.RidToAbi (rid); + uniqueAbis.Add (abi); + ret.SetMetadata (KnownMetadata.Abi, abi); + ret.SetMetadata (KnownMetadata.RuntimeIdentifier, rid); + + return ret; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index f77f5a37581..70fbc2c6dea 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -78,6 +78,66 @@ IEnumerable GetLinkerConfigs () ZipAlignmentPages = ZipAlignmentPages, }; + string stripSymbolsArg = DebugBuild ? String.Empty : " -s"; + + string ld = Path.Combine (AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, "ld")); + var targetLinkerArgs = new List (); + foreach (var kvp in abis) { + string abi = kvp.Key; + InputFiles inputs = kvp.Value; + + targetLinkerArgs.Clear (); + string elf_arch; + uint maxPageSize; + switch (abi) { + case "armeabi-v7a": + targetLinkerArgs.Add ("-X"); + elf_arch = "armelf_linux_eabi"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); + break; + + case "arm64": + case "arm64-v8a": + case "aarch64": + targetLinkerArgs.Add ("--fix-cortex-a53-843419"); + elf_arch = "aarch64linux"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); + break; + + case "x86": + elf_arch = "elf_i386"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); + break; + + case "x86_64": + elf_arch = "elf_x86_64"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); + break; + + default: + throw new NotSupportedException ($"Unsupported Android target architecture ABI: {abi}"); + } + + targetLinkerArgs.Add ("-m"); + targetLinkerArgs.Add (elf_arch); + + foreach (string file in inputs.ObjectFiles) { + targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (file)); + } + + targetLinkerArgs.Add ("-o"); + targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (inputs.OutputSharedLibrary)); + + if (inputs.ExtraLibraries != null) { + foreach (string lib in inputs.ExtraLibraries) { + targetLinkerArgs.Add (lib); + } + } + + targetLinkerArgs.Add ("-z"); + targetLinkerArgs.Add ($"max-page-size={maxPageSize}"); + + string targetArgs = String.Join (" ", targetLinkerArgs); yield return new Config { Linker = linker, LinkItems = GatherFilesForABI (item, abi, ObjectFiles), @@ -106,5 +166,17 @@ List GetItemsForABI (string abi, ITaskItem[] items) return ret; } + + void OnOutputData (string linkerName, object sender, DataReceivedEventArgs e) + { + if (e.Data != null) + LogMessage ($"[{linkerName} stdout] {e.Data}"); + } + + void OnErrorData (string linkerName, object sender, DataReceivedEventArgs e) + { + if (e.Data != null) + LogMessage ($"[{linkerName} stderr] {e.Data}"); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs new file mode 100644 index 00000000000..034066c77b6 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class LinkNativeRuntime : AsyncTask +{ + public override string TaskPrefix => "LNR"; + + public ITaskItem[] MonoComponents { get; set; } + + [Required] + public string AndroidBinUtilsDirectory { get; set; } + + [Required] + public string IntermediateOutputPath { get; set; } + + [Required] + public ITaskItem[] LinkLibraries { get; set; } + + [Required] + public ITaskItem[] NativeArchives { get; set; } + + [Required] + public ITaskItem[] NativeObjectFiles { get; set; } + + [Required] + public ITaskItem[] NativeLinkStartFiles { get; set; } + + [Required] + public ITaskItem[] NativeLinkEndFiles { get; set; } + + [Required] + public ITaskItem[] NativeSymbolsToExport { get; set; } + + [Required] + public ITaskItem[] OutputRuntimes { get; set; } + + [Required] + public ITaskItem[] SupportedAbis { get; set; } + + public bool SaveDebugSymbols { get; set; } + + public override System.Threading.Tasks.Task RunTaskAsync () + { + return this.WhenAll (SupportedAbis, LinkRuntime); + } + + void LinkRuntime (ITaskItem abiItem) + { + string abi = abiItem.ItemSpec; + Log.LogDebugMessage ($"LinkRuntime ({abi})"); + ITaskItem outputRuntime = GetFirstAbiItem (OutputRuntimes, "_UnifiedNativeRuntime", abi); + string soname = Path.GetFileNameWithoutExtension (outputRuntime.ItemSpec); + if (soname.StartsWith ("lib", StringComparison.OrdinalIgnoreCase)) { + soname = soname.Substring (3); + } + + var linker = new NativeLinker (Log, abi, soname, AndroidBinUtilsDirectory, IntermediateOutputPath, CancellationToken, Cancel) { + SaveDebugSymbols = SaveDebugSymbols, + }; + linker.Link ( + outputRuntime, + GetAbiItems (NativeObjectFiles, "_NativeAssemblyTarget", abi), + GetAbiItems (NativeArchives, "_SelectedNativeArchive", abi), + GetAbiItems (LinkLibraries, "_RequiredLinkLibraries", abi), + GetAbiItems (NativeLinkStartFiles, "_NativeLinkStartFiles", abi), + GetAbiItems (NativeLinkEndFiles, "_NativeLinkEndFiles", abi), + GetAbiItems (NativeSymbolsToExport, "_NativeSymbolsToExport", abi) + ); + } + + List GetAbiItems (ITaskItem[] source, string itemName, string abi) + { + var ret = new List (); + + foreach (ITaskItem item in source) { + if (AbiMatches (abi, item, itemName)) { + ret.Add (item); + } + } + + return ret; + } + + ITaskItem GetFirstAbiItem (ITaskItem[] source, string itemName, string abi) + { + foreach (ITaskItem item in source) { + if (AbiMatches (abi, item, itemName)) { + return item; + } + } + + throw new InvalidOperationException ($"Internal error: item '{itemName}' for ABI '{abi}' not found"); + } + + bool AbiMatches (string abi, ITaskItem item, string itemName) + { + return String.Compare (abi, item.GetRequiredMetadata (itemName, "Abi", Log), StringComparison.OrdinalIgnoreCase) == 0; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index cad5439df21..52d87df5565 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -16,6 +16,7 @@ public class PrepareAbiItems : AndroidTask const string CompressedAssembliesBase = "compressed_assemblies"; const string JniRemappingBase = "jni_remap"; const string MarshalMethodsBase = "marshal_methods"; + const string PinvokePreserveBase = "pinvoke_preserve"; public override string TaskPrefix => "PAI"; @@ -53,6 +54,8 @@ public override bool RunTask () baseName = JniRemappingBase; } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = MarshalMethodsBase; + } else if (String.Compare ("runtime_linking", Mode, StringComparison.OrdinalIgnoreCase) == 0) { + baseName = PinvokePreserveBase; } else { Log.LogError ($"Unknown mode: {Mode}"); return false; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs index 8267d0dde17..b752096d9fa 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs @@ -22,6 +22,16 @@ public class ProcessNativeLibraries : AndroidTask "libxamarin-debug-app-helper", }; + // Please keep the list sorted. Any new runtime libraries that are added upstream need to be mentioned here. + static readonly HashSet KnownRuntimeNativeLibrariesCLR = new (StringComparer.OrdinalIgnoreCase) { + "libSystem.Globalization.Native.so", + "libSystem.IO.Compression.Native.so", + "libSystem.Native.so", + "libSystem.Security.Cryptography.Native.Android.so", + "libclrjit.so", + "libcorclr.so", + }; + /// /// Assumed to be .so files only /// @@ -30,6 +40,7 @@ public class ProcessNativeLibraries : AndroidTask public string []? ExcludedLibraries { get; set; } public bool IncludeDebugSymbols { get; set; } + public bool NativeRuntimeLinking { get; set; } [Output] public ITaskItem []? OutputLibraries { get; set; } @@ -69,6 +80,11 @@ public override bool RunTask () } if (fileName.StartsWith ("libmono-android", StringComparison.Ordinal) || fileName.StartsWith ("libnet-android", StringComparison.Ordinal)) { + if (NativeRuntimeLinking) { + // We don't need the precompiled runtime, it will be linked during application build + continue; + } + if (fileName.EndsWith (".debug", StringComparison.Ordinal)) { if (!IncludeDebugSymbols) continue; @@ -90,12 +106,32 @@ public override bool RunTask () } } - output.Add (library); + if (!IgnoreLibraryWhenLinkingRuntime (library)) { + output.Add (library); + } } OutputLibraries = output.ToArray (); return !Log.HasLoggedErrors; } + + bool IgnoreLibraryWhenLinkingRuntime (ITaskItem libItem) + { + if (!NativeRuntimeLinking) { + return false; + } + + // We ignore all the shared libraries coming from the runtime packages, as they are all linked into our runtime and + // need not be packaged. + string packageId = libItem.GetMetadata ("NuGetPackageId"); + if (packageId.StartsWith ("Microsoft.NETCore.App.Runtime.Mono.android-", StringComparison.OrdinalIgnoreCase) || + packageId.StartsWith ("Microsoft.NETCore.App.Runtime.CoreCLR.android-", StringComparison.OrdinalIgnoreCase)) { + return true; + } + + // Should `NuGetPackageId` be empty, we check the libs by name, as the last resort. + return KnownRuntimeNativeLibrariesCLR.Contains (Path.GetFileName (libItem.ItemSpec)); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs index 5bb5246b936..0918aba990d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs @@ -3,9 +3,7 @@ namespace Xamarin.Android.Tasks; static class KnownMetadata { public const string Abi = "Abi"; - public const string NativeDontExportSymbols = "DontExportSymbols"; - public const string NativeLinkItemSet = "NativeLinkItemSet"; public const string NativeLinkWholeArchive = "LinkWholeArchive"; - public const string NativeSharedLibrary = "NativeSharedLibrary"; public const string RuntimeIdentifier = "RuntimeIdentifier"; + public const string NativeDontExportSymbols = "DontExportSymbols"; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index 6a0785dfd94..4314d17f812 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -30,6 +30,13 @@ class NativeCodeGenState /// public List AllJavaTypes { get; } + /// + /// Contains information about p/invokes used by the managed assemblies included in the + /// application. Will be **null** unless native runtime linking at application build time + /// is enabled. + /// + public List? PinvokeInfos { get; set; } + public List JavaTypesForJCW { get; } public XAAssemblyResolver Resolver { get; } public TypeDefinitionCache TypeCache { get; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 8fed3e87554..8e8b5b2ec29 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -15,6 +15,7 @@ class NativeLinker { static readonly List standardArgs = new () { "--shared", + "--allow-shlib-undefined", // TODO: need to enable zstd in binutils build // "--compress-debug-sections=zstd", // TODO: test the commented-out flags @@ -25,12 +26,11 @@ class NativeLinker "-z relro", "-z noexecstack", "-z max-page-size=16384", - "-z now", // we need it for security reasons (without it PLT can be overwritten) "--enable-new-dtags", "--build-id=sha1", "--warn-shared-textrel", "--fatal-warnings", - "--no-rosegment", + "--no-rosegment" }; readonly List extraArgs = new (); @@ -42,17 +42,11 @@ class NativeLinker readonly CancellationToken? cancellationToken; readonly Action? cancelTask; - public bool StripDebugSymbols { get; set; } = true; - public bool SaveDebugSymbols { get; set; } = true; - public bool AllowUndefinedSymbols { get; set; } = false; - public bool UseNdkLibraries { get; set; } = false; - public bool TargetsCLR { get; set; } - public string? NdkRootPath { get; set; } - public string? NdkApiLevel { get; set; } - public int ZipAlignmentPages { get; set; } = AndroidZipAlign.DefaultZipAlignment64Bit; + public bool StripDebugSymbols { get; set; } + public bool SaveDebugSymbols { get; set; } public NativeLinker (TaskLoggingHelper log, string abi, string soname, string binutilsDir, string intermediateDir, - IEnumerable runtimePackLibDirs, CancellationToken? cancellationToken = null, Action? cancelTask = null) + CancellationToken? cancellationToken = null, Action? cancelTask = null) { this.log = log; this.abi = abi; @@ -66,12 +60,10 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-soname {soname}"); string? elfArch = null; - uint maxPageSize; switch (abi) { case "armeabi-v7a": extraArgs.Add ("-X"); elfArch = "armelf_linux_eabi"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); break; case "arm64": @@ -79,17 +71,14 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi case "aarch64": extraArgs.Add ("--fix-cortex-a53-843419"); elfArch = "aarch64linux"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); break; case "x86": elfArch = "elf_i386"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); break; case "x86_64": elfArch = "elf_x86_64"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); break; default: @@ -100,41 +89,24 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-m {elfArch}"); } - extraArgs.Add ($"-z max-page-size={maxPageSize}"); + string runtimeNativeLibsDir = MonoAndroidHelper.GetNativeLibsRootDirectoryPath (binutilsDir); + string runtimeNativeLibStubsDir = MonoAndroidHelper.GetLibstubsRootDirectoryPath (binutilsDir); + string RID = MonoAndroidHelper.AbiToRid (abi); + string libStubsPath = Path.Combine (runtimeNativeLibStubsDir, RID); + string runtimeLibsDir = Path.Combine (runtimeNativeLibsDir, RID); - string? nativeLibsDir = MonoAndroidHelper.GetRuntimePackNativeLibDir (MonoAndroidHelper.AbiToTargetArch (abi), runtimePackLibDirs); - extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (nativeLibsDir)}"); + extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (libStubsPath)}"); + extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (runtimeLibsDir)}"); } - /// - /// A helper method to create a task item that refers to a native library, for the purpose of linking - /// it into another library. `baseLibraryName` must be just the "stem" of the library name (e.g. `c` or `dl`) - /// without any paths, prefixes or extensions. The returned item will then be turned by the - /// method into the `-l` parameter passed to the native linker. - /// - public static ITaskItem MakeLibraryItem (string baseLibraryName, string abi) + public bool Link (ITaskItem outputLibraryPath, List objectFiles, List archives, List libraries, + List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) { - ITaskItem libItem = new TaskItem (baseLibraryName); - libItem.SetMetadata (KnownMetadata.Abi, abi); - libItem.SetMetadata (KnownMetadata.NativeSharedLibrary, "true"); - return libItem; - } - - public bool Link (ITaskItem outputLibraryPath, List linkItems, List? linkStartFiles = null, List? linkEndFiles = null, ICollection? exportDynamicSymbols = null) - { - if (UseNdkLibraries) { - if (String.IsNullOrEmpty (NdkRootPath)) { - throw new InvalidOperationException ("Internal error: request to use NDK libraries, but NDK root not specified."); - } - - if (String.IsNullOrEmpty (NdkApiLevel)) { - throw new InvalidOperationException ("Internal error: request to use NDK libraries, but NDK API level not specified."); - } - } - log.LogDebugMessage ($"Linking: {outputLibraryPath}"); EnsureCorrectAbi (outputLibraryPath); - EnsureCorrectAbi (linkItems); + EnsureCorrectAbi (objectFiles); + EnsureCorrectAbi (archives); + EnsureCorrectAbi (libraries); EnsureCorrectAbi (linkStartFiles); EnsureCorrectAbi (linkEndFiles); @@ -147,21 +119,6 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List linkItems, List (); WriteFilesToResponseFile (sw, linkStartFiles); - WriteFilesToResponseFile (sw, linkItems); + WriteFilesToResponseFile (sw, objectFiles); + WriteFilesToResponseFile (sw, archives); if (exportDynamicSymbols != null && exportDynamicSymbols.Count > 0) { foreach (ITaskItem symbolItem in exportDynamicSymbols) { @@ -185,11 +143,15 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List { - $"@{MonoAndroidHelper.QuoteFileNameArgument (respFilePath)}", + $"@{respFilePath}", "-o", MonoAndroidHelper.QuoteFileNameArgument (outputLibraryPath.ItemSpec) }; @@ -204,14 +166,11 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List? files) + void WriteFilesToResponseFile (StreamWriter sw, List files) { - if (files == null) { - return; - } - foreach (ITaskItem file in files) { bool wholeArchive = IncludeWholeArchive (file); @@ -221,10 +180,7 @@ void WriteFilesToResponseFile (StreamWriter sw, List? files) if (wholeArchive) { sw.Write ("--whole-archive "); - } else if (IsNativeSharedLibrary (file)) { - sw.Write ("-l"); } - sw.Write (MonoAndroidHelper.QuoteFileNameArgument (file.ItemSpec)); // string abi = file.GetMetadata ("Abi") ?? String.Empty; // string destDir = Path.Combine ("/tmp/t", abi); @@ -239,7 +195,6 @@ void WriteFilesToResponseFile (StreamWriter sw, List? files) bool IncludeWholeArchive (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeLinkWholeArchive); bool ExcludeFromExports (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeDontExportSymbols); - bool IsNativeSharedLibrary (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeSharedLibrary); bool ParseBooleanMetadata (ITaskItem item, string metadata) { @@ -253,15 +208,6 @@ bool ParseBooleanMetadata (ITaskItem item, string metadata) } } - string GetAbiNdkRootDir () - { - // Let it throw if invalid - int apiLevel = Int32.Parse (NdkApiLevel); - NdkTools ndk = NdkTools.Create (NdkRootPath, logErrors: true, log: log); - - return ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, MonoAndroidHelper.AbiToTargetArch (abi), apiLevel); - } - void EnsureCorrectAbi (ITaskItem item) { // The exception is just a precaution, since the items passed to us should have already been checked @@ -273,12 +219,8 @@ void EnsureCorrectAbi (ITaskItem item) throw new InvalidOperationException ($"Internal error: '{item}' ABI ('{itemAbi}') doesn't have the expected value '{abi}'"); } - void EnsureCorrectAbi (List? items) + void EnsureCorrectAbi (List items) { - if (items == null) { - return; - } - foreach (ITaskItem item in items) { EnsureCorrectAbi (item); } @@ -307,7 +249,6 @@ bool ExtractDebugSymbols (ITaskItem outputSharedLibrary) stdoutLines.Clear (); stderrLines.Clear (); - args.Clear (); args.Add ("--strip-debug"); args.Add ("--strip-unneeded"); @@ -356,31 +297,61 @@ bool RunLinker (List args, ITaskItem outputSharedLibrary) bool RunCommand (string label, string binaryPath, List args, List stdoutLines, List stderrLines) { + using var stdout_completed = new ManualResetEvent (false); + using var stderr_completed = new ManualResetEvent (false); + var psi = new ProcessStartInfo () { + FileName = binaryPath, + Arguments = String.Join (" ", args), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + }; + string binaryName = Path.GetFileName (ld); - return MonoAndroidHelper.RunProcess ( - label, - binaryPath, - String.Join (" ", args), - log, - onOutput: (s, e) => { - if (e.Data == null) { - return; - } + log.LogDebugMessage ($"[{label}] {psi.FileName} {psi.Arguments}"); + + using var proc = new Process (); + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) { OnOutputData (binaryName, s, e); stdoutLines.Add (e.Data); - }, - onError: (s, e) => { - if (e.Data == null) { - return; - } + } else { + stdout_completed.Set (); + } + }; + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) { OnErrorData (binaryName, s, e); stderrLines.Add (e.Data); - }, - cancellationToken, - cancelTask, - logWarningOnFailure: false - ) == 0; + } else { + stderr_completed.Set (); + } + }; + + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + cancellationToken?.Register (() => { try { proc.Kill (); } catch (Exception) { } }); + proc.WaitForExit (); + + if (psi.RedirectStandardError) { + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (psi.RedirectStandardOutput) { + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (proc.ExitCode != 0) { + cancelTask?.Invoke (); + return false; + } + + return true; } void OnOutputData (string linkerName, object sender, DataReceivedEventArgs e) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs new file mode 100644 index 00000000000..b75e80d3977 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +class NativeRuntimeComponents +{ + internal class Archive + { + public readonly string Name; + public readonly string JniOnLoadName; + public bool Include => shouldInclude (this); + public readonly bool WholeArchive; + public bool DontExportSymbols { get; set; } + public HashSet? SymbolsToPreserve { get; set; } + + Func shouldInclude; + + public Archive (string name, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null) + { + Name = name; + shouldInclude = include == null ? ((Archive arch) => true) : include; + WholeArchive = wholeArchive; + JniOnLoadName = jniOnLoadName; + } + } + + internal class MonoComponentArchive : Archive + { + public readonly string ComponentName; + + public MonoComponentArchive (string name, string componentName, Func include) + : base (name, include) + { + ComponentName = componentName; + DontExportSymbols = true; + } + } + + sealed class ClangBuiltinsArchive : Archive + { + public ClangBuiltinsArchive (string clangAbi) + : base ($"libclang_rt.builtins-{clangAbi}-android.a") + {} + } + + class AndroidArchive : Archive + { + public AndroidArchive (string name) + : base (name, wholeArchive: false) + {} + } + + sealed class BclArchive : Archive + { + public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName = null) + : base (name, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName) + { + DontExportSymbols = true; + } + } + + readonly ITaskItem[] monoComponents; + + public readonly List KnownArchives; + public readonly List NativeLibraries; + public readonly List LinkStartFiles; + public readonly List LinkEndFiles; + + public NativeRuntimeComponents (ITaskItem[] monoComponents) + { + this.monoComponents = monoComponents; + KnownArchives = new () { + // Mono components + new MonoComponentArchive ("libmono-component-debugger-static.a", "debugger", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-debugger-stub-static.a", "debugger", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-diagnostics_tracing-static.a", "diagnostics_tracing", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-diagnostics_tracing-stub-static.a", "diagnostics_tracing", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-hot_reload-static.a", "hot_reload", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-hot_reload-stub-static.a", "hot_reload", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-marshal-ilgen-static.a", "marshal-ilgen", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-marshal-ilgen-stub-static.a", "marshal-ilgen", IncludeIfMonoComponentAbsent), + + // MonoVM runtime + BCL + new Archive ("libmonosgen-2.0.a") { + DontExportSymbols = true, + }, + new BclArchive ("libSystem.Globalization.Native.a"), + new BclArchive ("libSystem.IO.Compression.Native.a"), + new BclArchive ("libSystem.Native.a"), + new BclArchive ("libSystem.Security.Cryptography.Native.Android.a", jniOnLoadName: "AndroidCryptoNative_InitLibraryOnLoad") { + SymbolsToPreserve = new (StringComparer.Ordinal) { + // This isn't referenced directly by any code in libSystem.Security.Cryptography.Native.Android. It is instead + // referenced by the Java code shipped with the component (`DotnetProxyTrustManager`), as a native Java method: + // + // static native boolean verifyRemoteCertificate(long sslStreamProxyHandle); + // + // Therefore we must reference it explicitly + "Java_net_dot_android_crypto_DotnetProxyTrustManager_verifyRemoteCertificate" + }, + + // For now, we have to export all the symbols from this archive because we need the above `Java_net*` symbol to be + // externally visible, and the linker's `--exclude-libs` flag works on the archive (.a) level. + // + // TODO: use `llvm-ar` to extract the relevant object file and link it separately? + DontExportSymbols = false, + }, + + // .NET for Android + new AndroidArchive ("libpinvoke-override-dynamic-release.a"), + new AndroidArchive ("libruntime-base-release.a"), + new AndroidArchive ("libxa-java-interop-release.a"), + new AndroidArchive ("libxa-lz4-release.a"), + new AndroidArchive ("libxa-shared-bits-release.a"), + new AndroidArchive ("libmono-android.release-static-release.a"), + + // LLVM clang built-ins archives + new ClangBuiltinsArchive ("aarch64"), + new ClangBuiltinsArchive ("arm"), + new ClangBuiltinsArchive ("i686"), + new ClangBuiltinsArchive ("x86_64"), + + // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released + new Archive ("libunwind.a") { + DontExportSymbols = true, + }, + }; + + // Just the base names of libraries to link into the unified runtime. Must have all the dependencies of all the static archives we + // link into the final library. + NativeLibraries = new () { + "c", + "dl", + "m", + "z", + "log", + + // Atomic is a static library in clang, need to investigate if it's really needed +// "atomic", + }; + + // Files that will be linked before any other object/archive/library files + LinkStartFiles = new () { + "crtbegin_so.o", + }; + + // Files that will be linked after any other object/archive/library files + LinkEndFiles = new () { + "crtend_so.o", + }; + } + + bool MonoComponentExists (Archive archive) + { + if (monoComponents.Length == 0) { + return false; + } + + var mcArchive = archive as MonoComponentArchive; + if (mcArchive == null) { + throw new ArgumentException (nameof (archive), "Must be an instance of MonoComponentArchive"); + } + + foreach (ITaskItem item in monoComponents) { + if (String.Compare (item.ItemSpec, mcArchive.ComponentName, StringComparison.OrdinalIgnoreCase) == 0) { + return true; + } + } + + return false; + } + + bool IncludeIfMonoComponentAbsent (Archive archive) + { + return !MonoComponentExists (archive); + } + + bool IncludeIfMonoComponentPresent (Archive archive) + { + return MonoComponentExists (archive); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs new file mode 100644 index 00000000000..bbce0e6acdb --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +using Mono.Cecil; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class PinvokeScanner +{ + public sealed class PinvokeEntryInfo + { + public readonly string LibraryName; + public readonly string EntryName; + + public PinvokeEntryInfo (MethodDefinition method) + { + LibraryName = method.PInvokeInfo.Module.Name; + EntryName = method.PInvokeInfo.EntryPoint; + } + } + + readonly TaskLoggingHelper log; + + public PinvokeScanner (TaskLoggingHelper log) + { + this.log = log; + } + + public List Scan (AndroidTargetArch targetArch, XAAssemblyResolver resolver, ICollection frameworkAssemblies) + { + var pinvokes = new List (); + var pinvokeCache = new HashSet (StringComparer.Ordinal); + + foreach (ITaskItem fasm in frameworkAssemblies) { + string asmName = Path.GetFileNameWithoutExtension (fasm.ItemSpec); + AssemblyDefinition? asmdef = resolver.Resolve (asmName); + if (asmdef == null) { + log.LogWarning ($"Failed to resolve assembly '{fasm.ItemSpec}' for target architecture {targetArch}"); + continue; + } + Scan (targetArch, asmdef, pinvokeCache, pinvokes); + } + + return pinvokes; + } + + void Scan (AndroidTargetArch targetArch, AssemblyDefinition assembly, HashSet pinvokeCache, List pinvokes) + { + log.LogDebugMessage ($"[p/invoke][{targetArch}] Scanning assembly {assembly}"); + foreach (ModuleDefinition module in assembly.Modules) { + if (!module.HasTypes) { + continue; + } + + foreach (TypeDefinition type in module.Types) { + Scan (targetArch, type, pinvokeCache, pinvokes); + } + } + } + + void Scan (AndroidTargetArch targetArch, TypeDefinition type, HashSet pinvokeCache, List pinvokes) + { + if (type.HasNestedTypes) { + foreach (TypeDefinition nestedType in type.NestedTypes) { + Scan (targetArch, nestedType, pinvokeCache, pinvokes); + } + } + + if (!type.HasMethods) { + return; + } + + log.LogDebugMessage ($"[p/invoke][{targetArch}] Scanning type '{type}'"); + foreach (MethodDefinition method in type.Methods) { + if (!method.HasPInvokeInfo) { + continue; + } + + var pinfo = new PinvokeEntryInfo (method); + string key = $"{pinfo.LibraryName}/{pinfo.EntryName}"; + if (pinvokeCache.Contains (key)) { + continue; + } + + log.LogDebugMessage ($" [{targetArch}] p/invoke method: {pinfo.LibraryName}/{pinfo.EntryName}"); + pinvokeCache.Add (key); + pinvokes.Add (pinfo); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs new file mode 100644 index 00000000000..4517518ce72 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class PreservePinvokesNativeAssemblyGenerator : LlvmIrComposer +{ + sealed class PInvoke + { + public readonly LlvmIrFunction NativeFunction; + public readonly PinvokeScanner.PinvokeEntryInfo Info; + public readonly ulong Hash; + + public PInvoke (LlvmIrModule module, PinvokeScanner.PinvokeEntryInfo pinfo, bool is64Bit) + { + Info = pinfo; + Hash = MonoAndroidHelper.GetXxHash (pinfo.EntryName, is64Bit); + + // All the p/invoke functions use the same dummy signature. The only thing we care about is + // a way to reference to the symbol at build time so that we can return pointer to it. For + // that all we need is a known name, signature doesn't matter to us. + var funcSig = new LlvmIrFunctionSignature (name: pinfo.EntryName, returnType: typeof(void)); + NativeFunction = module.DeclareExternalFunction (funcSig); + } + } + + sealed class Component + { + public readonly string Name; + public readonly ulong NameHash; + public readonly List PInvokes; + public bool Is64Bit; + + public Component (string name, bool is64Bit) + { + Name = name; + NameHash = MonoAndroidHelper.GetXxHash (name, is64Bit); + PInvokes = new (); + Is64Bit = is64Bit; + } + + public void Add (LlvmIrModule module, PinvokeScanner.PinvokeEntryInfo pinfo) + { + PInvokes.Add (new PInvoke (module, pinfo, Is64Bit)); + } + + public void Sort () + { + PInvokes.Sort ((PInvoke a, PInvoke b) => a.Hash.CompareTo (b.Hash)); + } + } + + sealed class ConstructionState + { + public LlvmIrFunction Func; + public LlvmIrFunctionLabelItem ReturnLabel; + public LlvmIrFunctionParameter EntryPointHashParam; + public LlvmIrInstructions.Phi Phi; + public bool Is64Bit; + } + + // Maps a component name after ridding it of the `lib` prefix and the extension to a "canonical" + // name of a library, as used in `[DllImport]` attributes. + readonly Dictionary libraryNameMap = new (StringComparer.Ordinal) { + { "xa-java-interop", "java-interop" }, + { "mono-android.release-static", String.Empty }, + { "mono-android.release", String.Empty }, + }; + + readonly NativeCodeGenState state; + readonly ITaskItem[] monoComponents; + + public PreservePinvokesNativeAssemblyGenerator (TaskLoggingHelper log, NativeCodeGenState codeGenState, ITaskItem[] monoComponents) + : base (log) + { + if (codeGenState.PinvokeInfos == null) { + throw new InvalidOperationException ($"Internal error: {nameof (codeGenState)} `{nameof (codeGenState.PinvokeInfos)}` property is `null`"); + } + + this.state = codeGenState; + this.monoComponents = monoComponents; + } + + protected override void Construct (LlvmIrModule module) + { + Log.LogDebugMessage ($"[{state.TargetArch}] Constructing p/invoke preserve code"); + List pinvokeInfos = state.PinvokeInfos!; + if (pinvokeInfos.Count == 0) { + // This is a very unlikely scenario, but we will work just fine. The module that this generator produces will merely result + // in an empty (but valid) .ll file and an "empty" object file to link into the shared library. + return; + } + + Log.LogDebugMessage (" Looking for enabled native components"); + var componentNames = new HashSet (StringComparer.Ordinal); + var jniOnLoadNames = new HashSet (StringComparer.Ordinal); + var symbolsToExplicitlyPreserve = new HashSet (); + var nativeComponents = new NativeRuntimeComponents (monoComponents); + foreach (NativeRuntimeComponents.Archive archiveItem in nativeComponents.KnownArchives) { + if (!archiveItem.Include) { + continue; + } + + Log.LogDebugMessage ($" {archiveItem.Name}"); + componentNames.Add (archiveItem.Name); + if (!String.IsNullOrEmpty (archiveItem.JniOnLoadName)) { + jniOnLoadNames.Add (archiveItem.JniOnLoadName); + } + + if (archiveItem.SymbolsToPreserve == null || archiveItem.SymbolsToPreserve.Count == 0) { + continue; + } + + foreach (string symbolName in archiveItem.SymbolsToPreserve) { + symbolsToExplicitlyPreserve.Add (new LlvmIrGlobalVariableReference (symbolName)); + DeclareDummyFunction (module, symbolName); + } + } + + if (componentNames.Count == 0) { + Log.LogDebugMessage ("No native framework components are included in the build, not scanning for p/invoke usage"); + return; + } + + module.AddGlobalVariable ("__jni_on_load_handler_count", (uint)jniOnLoadNames.Count, LlvmIrVariableOptions.GlobalConstant); + var jniOnLoadPointers = new List (); + foreach (string name in jniOnLoadNames) { + jniOnLoadPointers.Add (new LlvmIrGlobalVariableReference (name)); + DeclareDummyFunction (module, name); + } + module.AddGlobalVariable ("__jni_on_load_handlers", jniOnLoadPointers, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__jni_on_load_handler_names", jniOnLoadNames, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__explicitly_preserved_symbols", symbolsToExplicitlyPreserve, LlvmIrVariableOptions.GlobalConstant); + + bool is64Bit = state.TargetArch switch { + AndroidTargetArch.Arm64 => true, + AndroidTargetArch.X86_64 => true, + AndroidTargetArch.Arm => false, + AndroidTargetArch.X86 => false, + _ => throw new NotSupportedException ($"Architecture {state.TargetArch} is not supported here") + }; + + Log.LogDebugMessage (" Checking discovered p/invokes against the list of components"); + var preservedPerComponent = new Dictionary (StringComparer.OrdinalIgnoreCase); + var processedCache = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (PinvokeScanner.PinvokeEntryInfo pinfo in pinvokeInfos) { + Log.LogDebugMessage ($" p/invoke: {pinfo.EntryName} in {pinfo.LibraryName}"); + string key = $"{pinfo.LibraryName}/${pinfo.EntryName}"; + if (processedCache.Contains (key)) { + Log.LogDebugMessage ($" already processed"); + continue; + } + + processedCache.Add (key); + if (!MustPreserve (pinfo, componentNames)) { + Log.LogDebugMessage (" no need to preserve"); + continue; + } + Log.LogDebugMessage (" must be preserved"); + + if (!preservedPerComponent.TryGetValue (pinfo.LibraryName, out Component? component)) { + component = new Component (pinfo.LibraryName, is64Bit); + preservedPerComponent.Add (component.Name, component); + } + component.Add (module, pinfo); + } + + var components = new List (preservedPerComponent.Values); + if (is64Bit) { + AddFindPinvoke (module, components, is64Bit); + } else { + AddFindPinvoke (module, components, is64Bit); + } + } + + void AddFindPinvoke (LlvmIrModule module, List components, bool is64Bit) where T: struct + { + var hashType = is64Bit ? typeof (ulong) : typeof (uint); + var parameters = new List { + new LlvmIrFunctionParameter (hashType, "library_name_hash") { + NoUndef = true, + }, + + new LlvmIrFunctionParameter (hashType, "entrypoint_hash") { + NoUndef = true, + }, + + new LlvmIrFunctionParameter (typeof(IntPtr), "known_library") { + Align = 1, // it's a reference to C++ `bool` + Dereferenceable = 1, + IsCplusPlusReference = true, + NoCapture = true, + NonNull = true, + NoUndef = true, + WriteOnly = true, + }, + }; + + var sig = new LlvmIrFunctionSignature ( + name: "find_pinvoke", + returnType: typeof(IntPtr), + parameters: parameters, + new LlvmIrFunctionSignature.ReturnTypeAttributes { + NoUndef = true, + } + ); + + var func = new LlvmIrFunction (sig, MakeFindPinvokeAttributeSet (module)) { + CallingConvention = LlvmIrCallingConvention.Fastcc, + Visibility = LlvmIrVisibility.Hidden, + }; + LlvmIrLocalVariable retval = func.CreateLocalVariable (typeof(IntPtr), "retval"); + var state = new ConstructionState { + Func = func, + ReturnLabel = new LlvmIrFunctionLabelItem ("return"), + EntryPointHashParam = parameters[1], + Phi = new LlvmIrInstructions.Phi (retval), + Is64Bit = is64Bit, + }; + module.Add (state.Func); + state.Func.Body.Add (new LlvmIrFunctionLabelItem ("entry")); + + var libraryNameSwitchEpilog = new LlvmIrFunctionLabelItem ("libNameSW.epilog"); + var componentSwitch = new LlvmIrInstructions.Switch (parameters[0], libraryNameSwitchEpilog, "sw.libname"); + + state.Func.Body.Add (componentSwitch); + state.Phi.AddNode (libraryNameSwitchEpilog, null); + + components.Sort ((Component a, Component b) => a.NameHash.CompareTo (b.NameHash)); + Log.LogDebugMessage (" Components to be preserved:"); + uint componentID = 1; + + foreach (Component component in components) { + Log.LogDebugMessage ($" {component.Name} (hash: 0x{component.NameHash:x}; {component.PInvokes.Count} p/invoke(s))"); + + string comment = $" {component.Name} (p/invoke count: {component.PInvokes.Count})"; + LlvmIrFunctionLabelItem componentLabel = AddSwitchItem (componentSwitch, component.NameHash, is64Bit, comment, null); + + func.Body.Add (componentLabel, comment); + AddPInvokeSwitch (state, componentLabel, component, componentID++); + } + + func.Body.Add (libraryNameSwitchEpilog); + + var setKnownLib = new LlvmIrInstructions.Store (false, parameters[2]); + func.Body.Add (setKnownLib); + AddReturnBranch (func, state.ReturnLabel); + + func.Body.Add (state.ReturnLabel); + func.Body.Add (state.Phi); + func.Body.Add (new LlvmIrInstructions.Ret (typeof (IntPtr), retval)); + } + + void AddPInvokeSwitch (ConstructionState state, LlvmIrFunctionLabelItem componentLabel, Component component, uint id) where T: struct + { + var pinvokeSwitchEpilog = new LlvmIrFunctionLabelItem ($"pinvokeSW.epilog.{id}"); + state.Phi.AddNode (pinvokeSwitchEpilog, null); + + var pinvokeSwitch = new LlvmIrInstructions.Switch (state.EntryPointHashParam, pinvokeSwitchEpilog, $"sw.pinvoke.{id}"); + state.Func.Body.Add (pinvokeSwitch); + + component.Sort (); + bool first = true; + foreach (PInvoke pi in component.PInvokes) { + string pinvokeName = pi.NativeFunction.Signature.Name; + string comment = $" {pinvokeName}"; + LlvmIrFunctionLabelItem pinvokeLabel = AddSwitchItem (pinvokeSwitch, pi.Hash, state.Is64Bit, comment, first ? state.ReturnLabel : null); + + // First item of every component switch block "reuses" the block's label + if (first) { + first = false; + } else { + state.Func.Body.Add (pinvokeLabel, comment); + AddReturnBranch (state.Func, state.ReturnLabel); + } + + state.Phi.AddNode (pinvokeLabel == state.ReturnLabel ? componentLabel : pinvokeLabel, new LlvmIrGlobalVariableReference (pinvokeName)); + } + + state.Func.Body.Add (pinvokeSwitchEpilog); + AddReturnBranch (state.Func, state.ReturnLabel); + } + + void AddReturnBranch (LlvmIrFunction func, LlvmIrFunctionLabelItem returnLabel) + { + var branch = new LlvmIrInstructions.Br (returnLabel); + func.Body.Add (branch); + } + + LlvmIrFunctionLabelItem AddSwitchItem (LlvmIrInstructions.Switch sw, ulong hash, bool is64Bit, string? comment, LlvmIrFunctionLabelItem? label) where T: struct + { + if (is64Bit) { + return sw.Add ((T)(object)hash, dest: label, comment: comment); + } + return sw.Add ((T)(object)(uint)hash, dest: label, comment: comment); + } + + LlvmIrFunctionAttributeSet MakeFindPinvokeAttributeSet (LlvmIrModule module) + { + var attrSet = new LlvmIrFunctionAttributeSet { + new MustprogressFunctionAttribute (), + new NofreeFunctionAttribute (), + new NorecurseFunctionAttribute (), + new NosyncFunctionAttribute (), + new NounwindFunctionAttribute (), + new WillreturnFunctionAttribute (), + new MemoryFunctionAttribute { + Default = MemoryAttributeAccessKind.Write, + Argmem = MemoryAttributeAccessKind.None, + InaccessibleMem = MemoryAttributeAccessKind.None, + }, + new UwtableFunctionAttribute (), + new NoTrappingMathFunctionAttribute (true), + }; + + return module.AddAttributeSet (attrSet); + } + + // Returns `true` for all p/invokes that we know are part of our set of components, otherwise returns `false`. + // Returning `false` merely means that the p/invoke isn't in any of BCL or our code and therefore we shouldn't + // care. It doesn't mean the p/invoke will be removed in any way. + bool MustPreserve (PinvokeScanner.PinvokeEntryInfo pinfo, ICollection components) + { + if (String.Compare ("xa-internal-api", pinfo.LibraryName, StringComparison.Ordinal) == 0) { + return true; + } + + foreach (string component in components) { + // The most common pattern for the BCL - file name without extension + string componentName = Path.GetFileNameWithoutExtension (component); + if (Matches (pinfo.LibraryName, componentName)) { + return true; + } + + // If it starts with `lib`, drop the prefix + if (componentName.StartsWith ("lib", StringComparison.Ordinal)) { + if (Matches (pinfo.LibraryName, componentName.Substring (3))) { + return true; + } + } + + // Might require mapping of component name to a canonical one + if (libraryNameMap.TryGetValue (componentName, out string? mappedComponentName) && !String.IsNullOrEmpty (mappedComponentName)) { + if (Matches (pinfo.LibraryName, mappedComponentName)) { + return true; + } + } + + // Try full file name, as the last resort + if (Matches (pinfo.LibraryName, Path.GetFileName (component))) { + return true; + } + } + + return false; + + bool Matches (string libraryName, string componentName) + { + return String.Compare (libraryName, componentName, StringComparison.Ordinal) == 0; + } + } + + static void DeclareDummyFunction (LlvmIrModule module, string name) + { + // Just a dummy declaration, we don't care about the arguments + var funcSig = new LlvmIrFunctionSignature (name, returnType: typeof(void)); + var _ = module.DeclareExternalFunction (funcSig); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 6421833e60e..5ff83e67d9f 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -365,6 +365,9 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidUseManagedMarshalMethodsLookup Condition=" '$(_AndroidUseManagedMarshalMethodsLookup)' == '' and '$(_AndroidUseMarshalMethods)' == 'True' and '$(_AndroidRuntime)' != 'MonoVM' ">True <_AndroidUseManagedMarshalMethodsLookup Condition=" '$(_AndroidUseManagedMarshalMethodsLookup)' == '' ">False + + + <_AndroidEnableNativeRuntimeLinking Condition=" '$(_AndroidEnableNativeRuntimeLinking)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' And '$(_AndroidRuntime)' == 'CoreCLR'">true @@ -1613,7 +1616,8 @@ because xbuild doesn't support framework reference assemblies. FrameworkDirectories="$(_XATargetFrameworkDirectories);$(_XATargetFrameworkDirectories)Facades" SupportedAbis="@(_BuildTargetAbis)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" - IntermediateOutputDirectory="$(IntermediateOutputPath)"> + IntermediateOutputDirectory="$(IntermediateOutputPath)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)"> + + + @@ -1880,9 +1892,40 @@ because xbuild doesn't support framework reference assemblies. + ResolvedAssemblies="@(_ResolvedAssemblies)" + ResolvedUserAssemblies="@(_ResolvedUserAssemblies)" + SatelliteAssemblies="@(_AndroidResolvedSatellitePaths)" + IntermediateOutputDirectory="$(IntermediateOutputPath)" + NativeLibraries="@(AndroidNativeLibrary);@(EmbeddedNativeLibrary);@(FrameworkNativeLibrary)" + MonoComponents="@(_MonoComponent)" + MainAssembly="$(TargetPath)" + OutputDirectory="$(_AndroidIntermediateJavaSourceDirectory)mono" + EnvironmentOutputDirectory="$(IntermediateOutputPath)android" + TargetFrameworkVersion="$(TargetFrameworkVersion)" + Manifest="$(IntermediateOutputPath)android\AndroidManifest.xml" + Environments="@(_EnvironmentFiles)" + AndroidAotMode="$(AndroidAotMode)" + AndroidAotEnableLazyLoad="$(AndroidAotEnableLazyLoad)" + EnableLLVM="$(EnableLLVM)" + HttpClientHandlerType="$(AndroidHttpClientHandlerType)" + TlsProvider="$(AndroidTlsProvider)" + Debug="$(AndroidIncludeDebugSymbols)" + AndroidSequencePointsMode="$(_SequencePointsMode)" + EnableSGenConcurrent="$(AndroidEnableSGenConcurrent)" + SupportedAbis="@(_BuildTargetAbis)" + AndroidPackageName="$(_AndroidPackage)" + EnablePreloadAssembliesDefault="$(_AndroidEnablePreloadAssembliesDefault)" + PackageNamingPolicy="$(AndroidPackageNamingPolicy)" + BoundExceptionType="$(AndroidBoundExceptionType)" + RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" + UseAssemblyStore="$(AndroidUseAssemblyStore)" + EnableMarshalMethods="$(_AndroidUseMarshalMethods)" + EnableManagedMarshalMethodsLookup="$(_AndroidUseManagedMarshalMethodsLookup)" + CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" + TargetsCLR="$(_AndroidUseCLR)" + ProjectRuntimeConfigFilePath="$(ProjectRuntimeConfigFilePath)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)" + > + + @@ -2040,6 +2085,7 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; _CreateApplicationSharedLibraries; + _LinkNativeRuntime; _GetMonoPlatformJarPath; _GetLibraryImports; _SetProguardMappingFileProperty; @@ -2151,6 +2197,9 @@ because xbuild doesn't support framework reference assemblies. <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AndroidRemapAssemblySource.abi) + <_NativeAssemblyTarget Include="@(_RuntimeLinkingAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_RuntimeLinkingAssemblySource.abi) + @@ -2170,10 +2219,10 @@ because xbuild doesn't support framework reference assemblies. + Outputs="@(_ApplicationSharedLibrary)" + Condition=" '$(_AndroidEnableNativeRuntimeLinking)' != 'true' "> Date: Thu, 20 Mar 2025 11:39:51 +0100 Subject: [PATCH 04/41] Builds now --- .../Tasks/LinkNativeRuntime.cs | 5 ++++- .../Utilities/NativeLinker.cs | 12 +++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs index 034066c77b6..f14a6e14448 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs @@ -45,6 +45,9 @@ public class LinkNativeRuntime : AsyncTask [Required] public ITaskItem[] SupportedAbis { get; set; } + [Required] + public ITaskItem[] RuntimePackLibraryDirectories { get; set; } = Array.Empty (); + public bool SaveDebugSymbols { get; set; } public override System.Threading.Tasks.Task RunTaskAsync () @@ -62,7 +65,7 @@ void LinkRuntime (ITaskItem abiItem) soname = soname.Substring (3); } - var linker = new NativeLinker (Log, abi, soname, AndroidBinUtilsDirectory, IntermediateOutputPath, CancellationToken, Cancel) { + var linker = new NativeLinker (Log, abi, soname, AndroidBinUtilsDirectory, IntermediateOutputPath, RuntimePackLibraryDirectories, CancellationToken, Cancel) { SaveDebugSymbols = SaveDebugSymbols, }; linker.Link ( diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 8e8b5b2ec29..0279f195456 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -46,7 +46,7 @@ class NativeLinker public bool SaveDebugSymbols { get; set; } public NativeLinker (TaskLoggingHelper log, string abi, string soname, string binutilsDir, string intermediateDir, - CancellationToken? cancellationToken = null, Action? cancelTask = null) + IEnumerable runtimePackLibDirs, CancellationToken? cancellationToken = null, Action? cancelTask = null) { this.log = log; this.abi = abi; @@ -89,14 +89,8 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-m {elfArch}"); } - string runtimeNativeLibsDir = MonoAndroidHelper.GetNativeLibsRootDirectoryPath (binutilsDir); - string runtimeNativeLibStubsDir = MonoAndroidHelper.GetLibstubsRootDirectoryPath (binutilsDir); - string RID = MonoAndroidHelper.AbiToRid (abi); - string libStubsPath = Path.Combine (runtimeNativeLibStubsDir, RID); - string runtimeLibsDir = Path.Combine (runtimeNativeLibsDir, RID); - - extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (libStubsPath)}"); - extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (runtimeLibsDir)}"); + string nativeLibsDir = MonoAndroidHelper.GetRuntimePackNativeLibDir (MonoAndroidHelper.AbiToTargetArch (abi), runtimePackLibDirs); + extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (nativeLibsDir)}"); } public bool Link (ITaskItem outputLibraryPath, List objectFiles, List archives, List libraries, From 0bd1fbfee9465bf2c0b11128e320d825d8777ada Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Mar 2025 11:46:59 +0100 Subject: [PATCH 05/41] More archives to package --- build-tools/create-packs/Microsoft.Android.Runtime.proj | 7 +++++-- .../targets/Microsoft.Android.Sdk.NativeRuntime.targets | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 14ea8c29ff8..a199bbd5bd8 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -55,9 +55,12 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets index a84b2d87d55..6ce45118048 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets @@ -54,7 +54,8 @@ Contains code to build and link the native runtime at application build time. LinkLibraries="@(_RequiredLinkLibraries)" OutputRuntimes="@(_UnifiedNativeRuntime)" SupportedAbis="@(_BuildTargetAbis)" - SaveDebugSymbols="true" /> + SaveDebugSymbols="true" + RuntimePackLibraryDirectories="@(_RuntimePackLibraryDirectory)" /> From 321f43634746c8d7815ac6415fac41bb19cdd377 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Mar 2025 16:58:59 +0100 Subject: [PATCH 06/41] Builds now --- .../Microsoft.Android.Runtime.proj | 10 +- .../scripts/generate-pinvoke-tables.sh | 2 +- src/native/CMakeLists.txt | 1 + src/native/clr/host/CMakeLists.txt | 4 +- .../clr/include/host/pinvoke-override-impl.hh | 157 ++++++++++++++++++ .../clr/include/host/pinvoke-override.hh | 154 ++--------------- .../clr/pinvoke-override/CMakeLists.txt | 92 ++++++++++ src/native/clr/pinvoke-override/dynamic.cc | 104 ++++++++++++ .../generate-pinvoke-tables.cc | 0 .../pinvoke-tables.include | 0 .../precompiled.cc} | 9 +- 11 files changed, 381 insertions(+), 152 deletions(-) create mode 100644 src/native/clr/include/host/pinvoke-override-impl.hh create mode 100644 src/native/clr/pinvoke-override/CMakeLists.txt create mode 100644 src/native/clr/pinvoke-override/dynamic.cc rename src/native/clr/{host => pinvoke-override}/generate-pinvoke-tables.cc (100%) rename src/native/clr/{host => pinvoke-override}/pinvoke-tables.include (100%) rename src/native/clr/{host/pinvoke-override.cc => pinvoke-override/precompiled.cc} (93%) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index a199bbd5bd8..a0c5aa8cbd3 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -75,11 +75,11 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - - - - - + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libmono-android.debug.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libmono-android.release.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-debug-app-helper.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-native-tracing.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> diff --git a/build-tools/scripts/generate-pinvoke-tables.sh b/build-tools/scripts/generate-pinvoke-tables.sh index dbf70afd352..ca359228ce3 100755 --- a/build-tools/scripts/generate-pinvoke-tables.sh +++ b/build-tools/scripts/generate-pinvoke-tables.sh @@ -5,7 +5,7 @@ HOST="$(uname | tr A-Z a-z)" NATIVE_DIR="${MY_DIR}/../../src/native" MONODROID_SOURCE_DIR="${NATIVE_DIR}/mono/pinvoke-override" MONODROID_INCLUDE_DIR="${NATIVE_DIR}/mono/shared" -CLR_SOURCE_DIR="${NATIVE_DIR}/clr/host" +CLR_SOURCE_DIR="${NATIVE_DIR}/clr/pinvoke-override" CLR_INCLUDE_DIR="${NATIVE_DIR}/clr/include/shared" GENERATOR_SOURCE="generate-pinvoke-tables.cc" GENERATOR_BINARY="generate-pinvoke-tables" diff --git a/src/native/CMakeLists.txt b/src/native/CMakeLists.txt index 77fccf62f5c..e383328fa46 100644 --- a/src/native/CMakeLists.txt +++ b/src/native/CMakeLists.txt @@ -619,6 +619,7 @@ else() endif() if (IS_CLR_RUNTIME) + add_subdirectory(clr/pinvoke-override) add_subdirectory(clr/host) add_subdirectory(clr/startup) endif() diff --git a/src/native/clr/host/CMakeLists.txt b/src/native/clr/host/CMakeLists.txt index 6cb67f06158..1fb9332c4cc 100644 --- a/src/native/clr/host/CMakeLists.txt +++ b/src/native/clr/host/CMakeLists.txt @@ -33,8 +33,6 @@ set(XAMARIN_MONODROID_SOURCES internal-pinvokes.cc jni-remapping.cc os-bridge.cc - pinvoke-override.cc - runtime-util.cc typemap.cc xamarin_getifaddrs.cc ) @@ -145,7 +143,7 @@ macro(lib_target_options TARGET_NAME) xa::xamarin-startup xa::runtime-base xa::java-interop -# xa::pinvoke-override-precompiled + xa::pinvoke-override-precompiled xa::lz4 -llog -lcoreclr diff --git a/src/native/clr/include/host/pinvoke-override-impl.hh b/src/native/clr/include/host/pinvoke-override-impl.hh new file mode 100644 index 00000000000..1a12fc02f9b --- /dev/null +++ b/src/native/clr/include/host/pinvoke-override-impl.hh @@ -0,0 +1,157 @@ +#pragma once + +#if !defined(PINVOKE_OVERRIDE_INLINE) +#error The PINVOKE_OVERRIDE_INLINE macro must be defined before including this header file +#endif + +#include +#include + +#include "pinvoke-override.hh" +#include "../runtime-base/logger.hh" +#include "../runtime-base/monodroid-dl.hh" +#include "../runtime-base/startup-aware-lock.hh" + +namespace xamarin::android { + PINVOKE_OVERRIDE_INLINE + auto PinvokeOverride::load_library_symbol (const char *library_name, const char *symbol_name, void **dso_handle) noexcept -> void* + { + void *lib_handle = dso_handle == nullptr ? nullptr : *dso_handle; + + if (lib_handle == nullptr) { + // We're being called as part of the p/invoke mechanism, we don't need to look in the AOT cache + constexpr bool PREFER_AOT_CACHE = false; + lib_handle = MonodroidDl::monodroid_dlopen (library_name, microsoft::java_interop::JAVA_INTEROP_LIB_LOAD_LOCALLY, PREFER_AOT_CACHE); + if (lib_handle == nullptr) { + log_warn (LOG_ASSEMBLY, "Shared library '{}' not loaded, p/invoke '{}' may fail", optional_string (library_name), optional_string (symbol_name)); + return nullptr; + } + + if (dso_handle != nullptr) { + void *expected_null = nullptr; + if (!__atomic_compare_exchange (dso_handle, &expected_null, &lib_handle, false /* weak */, __ATOMIC_ACQUIRE /* success_memorder */, __ATOMIC_RELAXED /* xxxfailure_memorder */)) { + log_debug (LOG_ASSEMBLY, "Library '{}' handle already cached by another thread", optional_string (library_name)); + } + } + } + + void *entry_handle = MonodroidDl::monodroid_dlsym (lib_handle, symbol_name); + if (entry_handle == nullptr) { + log_warn (LOG_ASSEMBLY, "Symbol '{}' not found in shared library '{}', p/invoke may fail", optional_string (library_name), optional_string (symbol_name)); + return nullptr; + } + + return entry_handle; + } + + PINVOKE_OVERRIDE_INLINE + auto PinvokeOverride::load_library_entry (std::string const& library_name, std::string const& entrypoint_name, pinvoke_api_map_ptr api_map) noexcept -> void* + { + // Make sure some other thread hasn't just added the entry + auto iter = api_map->find (entrypoint_name); + if (iter != api_map->end () && iter->second != nullptr) { + return iter->second; + } + + void *entry_handle = load_library_symbol (library_name.c_str (), entrypoint_name.c_str ()); + if (entry_handle == nullptr) { + // error already logged + return nullptr; + } + + log_debug (LOG_ASSEMBLY, "Caching p/invoke entry {} @ {}", library_name, entrypoint_name); + (*api_map)[entrypoint_name] = entry_handle; + return entry_handle; + } + + PINVOKE_OVERRIDE_INLINE + void PinvokeOverride::load_library_entry (const char *library_name, const char *entrypoint_name, PinvokeEntry &entry, void **dso_handle) noexcept + { + void *entry_handle = load_library_symbol (library_name, entrypoint_name, dso_handle); + void *expected_null = nullptr; + + bool already_loaded = !__atomic_compare_exchange ( + /* ptr */ &entry.func, + /* expected */ &expected_null, + /* desired */ &entry_handle, + /* weak */ false, + /* success_memorder */ __ATOMIC_ACQUIRE, + /* failure_memorder */ __ATOMIC_RELAXED + ); + + if (already_loaded) { + log_debug (LOG_ASSEMBLY, "Entry '{}' from library '{}' already loaded by another thread", entrypoint_name, library_name); + } + } + + PINVOKE_OVERRIDE_INLINE + auto PinvokeOverride::fetch_or_create_pinvoke_map_entry (std::string const& library_name, std::string const& entrypoint_name, hash_t entrypoint_name_hash, pinvoke_api_map_ptr api_map, bool need_lock) noexcept -> void* + { + auto iter = api_map->find (entrypoint_name, entrypoint_name_hash); + if (iter != api_map->end () && iter->second != nullptr) { + return iter->second; + } + + if (!need_lock) { + return load_library_entry (library_name, entrypoint_name, api_map); + } + + StartupAwareLock lock (pinvoke_map_write_lock); + return load_library_entry (library_name, entrypoint_name, api_map); + } + + PINVOKE_OVERRIDE_INLINE + auto PinvokeOverride::find_pinvoke_address (hash_t hash, const PinvokeEntry *entries, size_t entry_count) noexcept -> PinvokeEntry* + { + while (entry_count > 0uz) { + const size_t mid = entry_count / 2uz; + const PinvokeEntry *const ret = entries + mid; + const std::strong_ordering result = hash <=> ret->hash; + + if (result < 0) { + entry_count = mid; + } else if (result > 0) { + entries = ret + 1; + entry_count -= mid + 1uz; + } else { + return const_cast(ret); + } + } + + return nullptr; + } + + PINVOKE_OVERRIDE_INLINE + auto PinvokeOverride::handle_other_pinvoke_request (const char *library_name, hash_t library_name_hash, const char *entrypoint_name, hash_t entrypoint_name_hash) noexcept -> void* + { + std::string lib_name {library_name}; + std::string entry_name {entrypoint_name}; + + auto iter = other_pinvoke_map.find (lib_name, library_name_hash); + void *handle = nullptr; + if (iter == other_pinvoke_map.end ()) { + StartupAwareLock lock (pinvoke_map_write_lock); + + pinvoke_api_map_ptr lib_map; + // Make sure some other thread hasn't just added the map + iter = other_pinvoke_map.find (lib_name, library_name_hash); + if (iter == other_pinvoke_map.end () || iter->second == nullptr) { + lib_map = new pinvoke_api_map (1); + other_pinvoke_map[lib_name] = lib_map; + } else { + lib_map = iter->second; + } + + handle = fetch_or_create_pinvoke_map_entry (lib_name, entry_name, entrypoint_name_hash, lib_map, /* need_lock */ false); + } else { + if (iter->second == nullptr) [[unlikely]] { + log_warn (LOG_ASSEMBLY, "Internal error: null entry in p/invoke map for key '{}'", optional_string (library_name)); + return nullptr; // fall back to `monodroid_dlopen` + } + + handle = fetch_or_create_pinvoke_map_entry (lib_name, entry_name, entrypoint_name_hash, iter->second, /* need_lock */ true); + } + + return handle; + } +} diff --git a/src/native/clr/include/host/pinvoke-override.hh b/src/native/clr/include/host/pinvoke-override.hh index e1b7afebd95..de89dce1c5f 100644 --- a/src/native/clr/include/host/pinvoke-override.hh +++ b/src/native/clr/include/host/pinvoke-override.hh @@ -3,6 +3,8 @@ #include #include +#include + // NDEBUG causes robin_map.h not to include which, in turn, prevents indirect inclusion of . // conflicts with our std::mutex definition in cppcompat.hh #if !defined (NDEBUG) @@ -71,153 +73,25 @@ namespace xamarin::android { static inline constexpr pinvoke_library_map::size_type LIBRARY_MAP_INITIAL_BUCKET_COUNT = 1uz; public: - [[gnu::always_inline]] - static auto load_library_symbol (const char *library_name, const char *symbol_name, void **dso_handle = nullptr) noexcept -> void* - { - void *lib_handle = dso_handle == nullptr ? nullptr : *dso_handle; - - if (lib_handle == nullptr) { - // We're being called as part of the p/invoke mechanism, we don't need to look in the AOT cache - constexpr bool PREFER_AOT_CACHE = false; - lib_handle = MonodroidDl::monodroid_dlopen (library_name, microsoft::java_interop::JAVA_INTEROP_LIB_LOAD_LOCALLY, PREFER_AOT_CACHE); - if (lib_handle == nullptr) { - log_warn (LOG_ASSEMBLY, "Shared library '{}' not loaded, p/invoke '{}' may fail", optional_string (library_name), optional_string (symbol_name)); - return nullptr; - } - - if (dso_handle != nullptr) { - void *expected_null = nullptr; - if (!__atomic_compare_exchange (dso_handle, &expected_null, &lib_handle, false /* weak */, __ATOMIC_ACQUIRE /* success_memorder */, __ATOMIC_RELAXED /* xxxfailure_memorder */)) { - log_debug (LOG_ASSEMBLY, "Library '{}' handle already cached by another thread", optional_string (library_name)); - } - } - } - - void *entry_handle = MonodroidDl::monodroid_dlsym (lib_handle, symbol_name); - if (entry_handle == nullptr) { - log_warn (LOG_ASSEMBLY, "Symbol '{}' not found in shared library '{}', p/invoke may fail", optional_string (library_name), optional_string (symbol_name)); - return nullptr; - } - - return entry_handle; - } - - static auto load_library_entry (std::string const& library_name, std::string const& entrypoint_name, pinvoke_api_map_ptr api_map) noexcept -> void* - { - // Make sure some other thread hasn't just added the entry - auto iter = api_map->find (entrypoint_name); - if (iter != api_map->end () && iter->second != nullptr) { - return iter->second; - } - - void *entry_handle = load_library_symbol (library_name.c_str (), entrypoint_name.c_str ()); - if (entry_handle == nullptr) { - // error already logged - return nullptr; - } - - log_debug (LOG_ASSEMBLY, "Caching p/invoke entry {} @ {}", library_name, entrypoint_name); - (*api_map)[entrypoint_name] = entry_handle; - return entry_handle; - } - - static void load_library_entry (const char *library_name, const char *entrypoint_name, PinvokeEntry &entry, void **dso_handle) noexcept - { - void *entry_handle = load_library_symbol (library_name, entrypoint_name, dso_handle); - void *expected_null = nullptr; - - bool already_loaded = !__atomic_compare_exchange ( - /* ptr */ &entry.func, - /* expected */ &expected_null, - /* desired */ &entry_handle, - /* weak */ false, - /* success_memorder */ __ATOMIC_ACQUIRE, - /* failure_memorder */ __ATOMIC_RELAXED - ); - - if (already_loaded) { - log_debug (LOG_ASSEMBLY, "Entry '{}' from library '{}' already loaded by another thread", entrypoint_name, library_name); - } - } - - static auto fetch_or_create_pinvoke_map_entry (std::string const& library_name, std::string const& entrypoint_name, hash_t entrypoint_name_hash, pinvoke_api_map_ptr api_map, bool need_lock) noexcept -> void* - { - auto iter = api_map->find (entrypoint_name, entrypoint_name_hash); - if (iter != api_map->end () && iter->second != nullptr) { - return iter->second; - } - - if (!need_lock) { - return load_library_entry (library_name, entrypoint_name, api_map); - } - - StartupAwareLock lock (pinvoke_map_write_lock); - return load_library_entry (library_name, entrypoint_name, api_map); - } - - [[gnu::always_inline]] - static auto find_pinvoke_address (hash_t hash, const PinvokeEntry *entries, size_t entry_count) noexcept -> PinvokeEntry* - { - while (entry_count > 0uz) { - const size_t mid = entry_count / 2uz; - const PinvokeEntry *const ret = entries + mid; - const std::strong_ordering result = hash <=> ret->hash; - - if (result < 0) { - entry_count = mid; - } else if (result > 0) { - entries = ret + 1; - entry_count -= mid + 1uz; - } else { - return const_cast(ret); - } - } - - return nullptr; - } - - [[gnu::always_inline, gnu::flatten]] - static auto handle_other_pinvoke_request (const char *library_name, hash_t library_name_hash, const char *entrypoint_name, hash_t entrypoint_name_hash) noexcept -> void* - { - std::string lib_name {library_name}; - std::string entry_name {entrypoint_name}; - - auto iter = other_pinvoke_map.find (lib_name, library_name_hash); - void *handle = nullptr; - if (iter == other_pinvoke_map.end ()) { - StartupAwareLock lock (pinvoke_map_write_lock); - - pinvoke_api_map_ptr lib_map; - // Make sure some other thread hasn't just added the map - iter = other_pinvoke_map.find (lib_name, library_name_hash); - if (iter == other_pinvoke_map.end () || iter->second == nullptr) { - lib_map = new pinvoke_api_map (1); - other_pinvoke_map[lib_name] = lib_map; - } else { - lib_map = iter->second; - } - - handle = fetch_or_create_pinvoke_map_entry (lib_name, entry_name, entrypoint_name_hash, lib_map, /* need_lock */ false); - } else { - if (iter->second == nullptr) [[unlikely]] { - log_warn (LOG_ASSEMBLY, "Internal error: null entry in p/invoke map for key '{}'", optional_string (library_name)); - return nullptr; // fall back to `monodroid_dlopen` - } - - handle = fetch_or_create_pinvoke_map_entry (lib_name, entry_name, entrypoint_name_hash, iter->second, /* need_lock */ true); - } - - return handle; - } - + static auto load_library_symbol (const char *library_name, const char *symbol_name, void **dso_handle = nullptr) noexcept -> void*; + static auto load_library_entry (std::string const& library_name, std::string const& entrypoint_name, pinvoke_api_map_ptr api_map) noexcept -> void*; + static void load_library_entry (const char *library_name, const char *entrypoint_name, PinvokeEntry &entry, void **dso_handle) noexcept; + static auto fetch_or_create_pinvoke_map_entry (std::string const& library_name, std::string const& entrypoint_name, hash_t entrypoint_name_hash, pinvoke_api_map_ptr api_map, bool need_lock) noexcept -> void*; + static auto find_pinvoke_address (hash_t hash, const PinvokeEntry *entries, size_t entry_count) noexcept -> PinvokeEntry*; + static auto handle_other_pinvoke_request (const char *library_name, hash_t library_name_hash, const char *entrypoint_name, hash_t entrypoint_name_hash) noexcept -> void*; + + static void handle_jni_on_load (JavaVM *vm, void *reserved) noexcept; static auto monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) noexcept -> void*; private: static inline std::mutex pinvoke_map_write_lock{}; - static inline pinvoke_library_map other_pinvoke_map{}; + static inline pinvoke_library_map other_pinvoke_map { PinvokeOverride::LIBRARY_MAP_INITIAL_BUCKET_COUNT }; + +#if defined(PRECOMPILED) static inline void *system_native_library_handle = nullptr; static inline void *system_security_cryptography_native_android_library_handle = nullptr; static inline void *system_io_compression_native_library_handle = nullptr; static inline void *system_globalization_native_library_handle = nullptr; +#endif }; } diff --git a/src/native/clr/pinvoke-override/CMakeLists.txt b/src/native/clr/pinvoke-override/CMakeLists.txt new file mode 100644 index 00000000000..b1f35a16e2f --- /dev/null +++ b/src/native/clr/pinvoke-override/CMakeLists.txt @@ -0,0 +1,92 @@ +set(LIB_NAME_PRECOMPILED pinvoke-override-precompiled) +set(LIB_ALIAS_PRECOMPILED xa::pinvoke-override-precompiled) + +set(LIB_NAME_DYNAMIC pinvoke-override-dynamic) +set(LIB_ALIAS_DYNAMIC xa::pinvoke-override-dynamic) + +set(XA_PINVOKE_OVERRIDE_PRECOMPILED_SOURCES + precompiled.cc +) +add_clang_check_sources("${XA_PINVOKE_OVERRIDE_PRECOMPILED_SOURCES}") + +set(XA_PINVOKE_OVERRIDE_DYNAMIC_SOURCES + dynamic.cc +) +add_clang_check_sources("${XA_PINVOKE_OVERRIDE_DYNAMIC_SOURCES}") + +list(APPEND POTENTIAL_LOCAL_COMPILER_ARGS + -ffunction-sections + -fdata-sections +) + +xa_check_c_args(PINVOKE_OVERRIDE_CXX_ARGS "${POTENTIAL_LOCAL_COMPILER_ARGS}") + +macro(create_library _libname _alias _sources) + add_library( + ${_libname} + STATIC + ${_sources} + ) + + add_library(${_alias} ALIAS ${_libname}) + + set_static_library_suffix(${_libname}) + + target_compile_definitions( + ${_libname} + PRIVATE + TSL_NO_EXCEPTIONS + ) + + target_compile_options( + ${_libname} + PRIVATE + ${XA_COMMON_CXX_ARGS} + ${PINVOKE_OVERRIDE_CXX_ARGS} + ) + + target_include_directories( + ${_libname} + PRIVATE + ${ROBIN_MAP_DIR}/include + ) + + target_include_directories( + ${_libname} + PUBLIC + "$" + ) + + target_include_directories( + ${_libname} + SYSTEM PRIVATE + ${SYSROOT_CXX_INCLUDE_DIR} + ${RUNTIME_INCLUDE_DIR} + ) + + target_link_libraries( + ${_libname} + PRIVATE + ${SHARED_LIB_NAME} + xa::xamarin-app + xa::runtime-base + ) + + xa_add_compile_definitions(${_libname}) + xa_add_include_directories(${_libname}) +endmacro() + +create_library(${LIB_NAME_PRECOMPILED} ${LIB_ALIAS_PRECOMPILED} ${XA_PINVOKE_OVERRIDE_PRECOMPILED_SOURCES}) +create_library(${LIB_NAME_DYNAMIC} ${LIB_ALIAS_DYNAMIC} ${XA_PINVOKE_OVERRIDE_DYNAMIC_SOURCES}) + +target_compile_definitions( + ${LIB_NAME_PRECOMPILED} + PRIVATE + PRECOMPILED +) + +set_target_properties( + ${LIB_NAME_PRECOMPILED} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" +) diff --git a/src/native/clr/pinvoke-override/dynamic.cc b/src/native/clr/pinvoke-override/dynamic.cc new file mode 100644 index 00000000000..6c860aa440c --- /dev/null +++ b/src/native/clr/pinvoke-override/dynamic.cc @@ -0,0 +1,104 @@ +#include + +#include + +#define PINVOKE_OVERRIDE_INLINE [[gnu::noinline]] +#include + +using namespace xamarin::android; + +using JniOnLoadHandler = jint (*) (JavaVM *vm, void *reserved); + +// +// These external functions are generated during application build (see obj/${CONFIGURATION}/${RID}/android/pinvoke_preserve.*.ll) +// +extern "C" { + void* find_pinvoke (hash_t library_name_hash, hash_t entrypoint_hash, bool &known_library); + + extern const uint32_t __jni_on_load_handler_count; + extern const JniOnLoadHandler __jni_on_load_handlers[]; + extern const char* __jni_on_load_handler_names[]; + extern const void* __explicitly_preserved_symbols[]; +} + +[[gnu::flatten]] +auto PinvokeOverride::monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) noexcept -> void* +{ + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + log_debug (LOG_ASSEMBLY, "library_name == '{}'; entrypoint_name == '{}'"sv, optional_string (library_name), optional_string (entrypoint_name)); + + if (library_name == nullptr || entrypoint_name == nullptr) [[unlikely]] { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Both library name ('{}') and entry point name ('{}') must be specified"sv, + optional_string (library_name), + optional_string (entrypoint_name) + ) + ); + } + + hash_t library_name_hash = xxhash::hash (library_name, strlen (library_name)); + hash_t entrypoint_hash = xxhash::hash (entrypoint_name, strlen (entrypoint_name)); + log_debug (LOG_ASSEMBLY, "library_name_hash == 0x{:x}; entrypoint_hash == 0x{:x}"sv, library_name_hash, entrypoint_hash); + + bool known_library = true; + void *pinvoke_ptr = find_pinvoke (library_name_hash, entrypoint_hash, known_library); + if (pinvoke_ptr != nullptr) [[likely]] { + log_debug (LOG_ASSEMBLY, "pinvoke_ptr == {:p}"sv, pinvoke_ptr); + return pinvoke_ptr; + } + + if (known_library) [[unlikely]] { + log_debug (LOG_ASSEMBLY, "Lookup in a known library == internal"sv); + // Should "never" happen. It seems we have a known library hash (of one that's linked into the dynamically + // built DSO) but an unknown symbol hash. The symbol **probably** doesn't exist (was most likely linked out if + // the find* functions didn't know its hash), but we cannot be sure of that so we'll try to load it. + pinvoke_ptr = dlsym (RTLD_DEFAULT, entrypoint_name); + if (pinvoke_ptr == nullptr) { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Unable to load p/invoke entry '{}/{}' from the unified runtime DSO"sv, + optional_string (library_name), + optional_string (entrypoint_name) + ) + ); + } + + return pinvoke_ptr; + } + + log_debug (LOG_ASSEMBLY, "p/invoke not from a known library, slow path taken."sv); + pinvoke_ptr = handle_other_pinvoke_request (library_name, library_name_hash, entrypoint_name, entrypoint_hash);; + log_debug (LOG_ASSEMBLY, "foreign library pinvoke_ptr == {:p}"sv, pinvoke_ptr); + return pinvoke_ptr; +} + +void PinvokeOverride::handle_jni_on_load (JavaVM *vm, void *reserved) noexcept +{ + if (__jni_on_load_handler_count == 0) { + return; + } + + for (uint32_t i = 0; i < __jni_on_load_handler_count; i++) { + __jni_on_load_handlers[i] (vm, reserved); + } + + // This is just to reference the generated array, all we need from it is to be there + // TODO: see if there's an attribute we can use to make the linker keep the symbol instead. + // void *first_ptr = __explicitly_preserved_symbols; + // if (first_ptr == nullptr) { + // // This will never actually be logged, since by the time this function is called we haven't initialized + // // logging categories yet. It's here just to have some code in the if statement body. + // log_debug (LOG_ASSEMBLY, "No explicitly preserved symbols"); + // } +} + +const void* Host::clr_pinvoke_override (const char *library_name, const char *entry_point_name) noexcept +{ + log_debug (LOG_ASSEMBLY, "[dynamic] clr_pinvoke_override (\"{}\", \"{}\")"sv, optional_string (library_name), optional_string (entry_point_name)); + void *ret = PinvokeOverride::monodroid_pinvoke_override (library_name, entry_point_name); + log_debug (LOG_DEFAULT, "[dynamic] p/invoke {}found", ret == nullptr ? "not"sv : ""sv); + return ret; +} diff --git a/src/native/clr/host/generate-pinvoke-tables.cc b/src/native/clr/pinvoke-override/generate-pinvoke-tables.cc similarity index 100% rename from src/native/clr/host/generate-pinvoke-tables.cc rename to src/native/clr/pinvoke-override/generate-pinvoke-tables.cc diff --git a/src/native/clr/host/pinvoke-tables.include b/src/native/clr/pinvoke-override/pinvoke-tables.include similarity index 100% rename from src/native/clr/host/pinvoke-tables.include rename to src/native/clr/pinvoke-override/pinvoke-tables.include diff --git a/src/native/clr/host/pinvoke-override.cc b/src/native/clr/pinvoke-override/precompiled.cc similarity index 93% rename from src/native/clr/host/pinvoke-override.cc rename to src/native/clr/pinvoke-override/precompiled.cc index d1467e4eda1..a6e3b037852 100644 --- a/src/native/clr/host/pinvoke-override.cc +++ b/src/native/clr/pinvoke-override/precompiled.cc @@ -1,5 +1,7 @@ +#define PINVOKE_OVERRIDE_INLINE [[gnu::always_inline]] + #include -#include +#include #include using namespace xamarin::android; @@ -98,8 +100,9 @@ auto PinvokeOverride::monodroid_pinvoke_override (const char *library_name, cons const void* Host::clr_pinvoke_override (const char *library_name, const char *entry_point_name) noexcept { - log_debug (LOG_ASSEMBLY, "clr_pinvoke_override (\"{}\", \"{}\")", library_name, entry_point_name); + log_debug (LOG_ASSEMBLY, "[precompiled] clr_pinvoke_override (\"{}\", \"{}\")", library_name, entry_point_name); void *ret = PinvokeOverride::monodroid_pinvoke_override (library_name, entry_point_name); - log_debug (LOG_DEFAULT, "p/invoke {}found", ret == nullptr ? "not"sv : ""sv); + log_debug (LOG_DEFAULT, "[precompiled] p/invoke {}found", ret == nullptr ? "not"sv : ""sv); return ret; } + From 6961f10a69f5ad49dbe0a0e726ea3e0f98e7cca7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Mar 2025 17:11:59 +0100 Subject: [PATCH 07/41] Nope. Not here. --- .../targets/Microsoft.Android.Sdk.NativeRuntime.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets index 6ce45118048..59b62403a5b 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets @@ -38,7 +38,7 @@ Contains code to build and link the native runtime at application build time. From 0fc60cad5e1a4bd61b8c9ac38d5a6f48242276a9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Mar 2025 18:15:59 +0100 Subject: [PATCH 08/41] Symbol preservation tweaks --- .../Utilities/NativeRuntimeComponents.cs | 54 +------------- ...PreservePinvokesNativeAssemblyGenerator.cs | 74 ++++++++++++------- 2 files changed, 50 insertions(+), 78 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index b75e80d3977..1fa709fbffe 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -27,18 +27,6 @@ public Archive (string name, Func? include = null, bool wholeArch } } - internal class MonoComponentArchive : Archive - { - public readonly string ComponentName; - - public MonoComponentArchive (string name, string componentName, Func include) - : base (name, include) - { - ComponentName = componentName; - DontExportSymbols = true; - } - } - sealed class ClangBuiltinsArchive : Archive { public ClangBuiltinsArchive (string clangAbi) @@ -73,17 +61,7 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) { this.monoComponents = monoComponents; KnownArchives = new () { - // Mono components - new MonoComponentArchive ("libmono-component-debugger-static.a", "debugger", IncludeIfMonoComponentPresent), - new MonoComponentArchive ("libmono-component-debugger-stub-static.a", "debugger", IncludeIfMonoComponentAbsent), - new MonoComponentArchive ("libmono-component-diagnostics_tracing-static.a", "diagnostics_tracing", IncludeIfMonoComponentPresent), - new MonoComponentArchive ("libmono-component-diagnostics_tracing-stub-static.a", "diagnostics_tracing", IncludeIfMonoComponentAbsent), - new MonoComponentArchive ("libmono-component-hot_reload-static.a", "hot_reload", IncludeIfMonoComponentPresent), - new MonoComponentArchive ("libmono-component-hot_reload-stub-static.a", "hot_reload", IncludeIfMonoComponentAbsent), - new MonoComponentArchive ("libmono-component-marshal-ilgen-static.a", "marshal-ilgen", IncludeIfMonoComponentPresent), - new MonoComponentArchive ("libmono-component-marshal-ilgen-stub-static.a", "marshal-ilgen", IncludeIfMonoComponentAbsent), - - // MonoVM runtime + BCL + // CoreCLR runtime + BCL new Archive ("libmonosgen-2.0.a") { DontExportSymbols = true, }, @@ -151,34 +129,4 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) "crtend_so.o", }; } - - bool MonoComponentExists (Archive archive) - { - if (monoComponents.Length == 0) { - return false; - } - - var mcArchive = archive as MonoComponentArchive; - if (mcArchive == null) { - throw new ArgumentException (nameof (archive), "Must be an instance of MonoComponentArchive"); - } - - foreach (ITaskItem item in monoComponents) { - if (String.Compare (item.ItemSpec, mcArchive.ComponentName, StringComparison.OrdinalIgnoreCase) == 0) { - return true; - } - } - - return false; - } - - bool IncludeIfMonoComponentAbsent (Archive archive) - { - return !MonoComponentExists (archive); - } - - bool IncludeIfMonoComponentPresent (Archive archive) - { - return MonoComponentExists (archive); - } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs index 4517518ce72..2739cd2a0ad 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs @@ -101,8 +101,8 @@ protected override void Construct (LlvmIrModule module) Log.LogDebugMessage (" Looking for enabled native components"); var componentNames = new HashSet (StringComparer.Ordinal); - var jniOnLoadNames = new HashSet (StringComparer.Ordinal); - var symbolsToExplicitlyPreserve = new HashSet (); + var componentLoadHandlers = new Dictionary (StringComparer.Ordinal); + var componentPreservedSymbols = new Dictionary> (StringComparer.Ordinal); var nativeComponents = new NativeRuntimeComponents (monoComponents); foreach (NativeRuntimeComponents.Archive archiveItem in nativeComponents.KnownArchives) { if (!archiveItem.Include) { @@ -112,17 +112,18 @@ protected override void Construct (LlvmIrModule module) Log.LogDebugMessage ($" {archiveItem.Name}"); componentNames.Add (archiveItem.Name); if (!String.IsNullOrEmpty (archiveItem.JniOnLoadName)) { - jniOnLoadNames.Add (archiveItem.JniOnLoadName); + componentLoadHandlers.Add (archiveItem.Name, archiveItem.JniOnLoadName); } if (archiveItem.SymbolsToPreserve == null || archiveItem.SymbolsToPreserve.Count == 0) { continue; } + var preservedSymbols = new HashSet (); foreach (string symbolName in archiveItem.SymbolsToPreserve) { - symbolsToExplicitlyPreserve.Add (new LlvmIrGlobalVariableReference (symbolName)); - DeclareDummyFunction (module, symbolName); + preservedSymbols.Add (new LlvmIrGlobalVariableReference (symbolName)); } + componentPreservedSymbols.Add (archiveItem.Name, preservedSymbols); } if (componentNames.Count == 0) { @@ -130,16 +131,6 @@ protected override void Construct (LlvmIrModule module) return; } - module.AddGlobalVariable ("__jni_on_load_handler_count", (uint)jniOnLoadNames.Count, LlvmIrVariableOptions.GlobalConstant); - var jniOnLoadPointers = new List (); - foreach (string name in jniOnLoadNames) { - jniOnLoadPointers.Add (new LlvmIrGlobalVariableReference (name)); - DeclareDummyFunction (module, name); - } - module.AddGlobalVariable ("__jni_on_load_handlers", jniOnLoadPointers, LlvmIrVariableOptions.GlobalConstant); - module.AddGlobalVariable ("__jni_on_load_handler_names", jniOnLoadNames, LlvmIrVariableOptions.GlobalConstant); - module.AddGlobalVariable ("__explicitly_preserved_symbols", symbolsToExplicitlyPreserve, LlvmIrVariableOptions.GlobalConstant); - bool is64Bit = state.TargetArch switch { AndroidTargetArch.Arm64 => true, AndroidTargetArch.X86_64 => true, @@ -151,6 +142,11 @@ protected override void Construct (LlvmIrModule module) Log.LogDebugMessage (" Checking discovered p/invokes against the list of components"); var preservedPerComponent = new Dictionary (StringComparer.OrdinalIgnoreCase); var processedCache = new HashSet (StringComparer.OrdinalIgnoreCase); + var jniOnLoadNames = new HashSet (StringComparer.Ordinal); + bool haveLoadHandlers = componentLoadHandlers.Count > 0; + bool havePreservedSymbols = componentPreservedSymbols.Count > 0; + + var symbolsToExplicitlyPreserve = new HashSet (); foreach (PinvokeScanner.PinvokeEntryInfo pinfo in pinvokeInfos) { Log.LogDebugMessage ($" p/invoke: {pinfo.EntryName} in {pinfo.LibraryName}"); @@ -161,12 +157,29 @@ protected override void Construct (LlvmIrModule module) } processedCache.Add (key); - if (!MustPreserve (pinfo, componentNames)) { + (bool preserve, string? componentName) = MustPreserve (pinfo, componentNames); + if (!preserve) { Log.LogDebugMessage (" no need to preserve"); continue; } Log.LogDebugMessage (" must be preserved"); + if (!String.IsNullOrEmpty (componentName)) { + if (haveLoadHandlers && componentLoadHandlers.TryGetValue (componentName, out string jniOnLoadName)) { + if (jniOnLoadNames.Add (jniOnLoadName)) { + Log.LogDebugMessage ($" component '{componentName}' registers a load handler '{jniOnLoadName}'"); + } + } + + if (havePreservedSymbols && componentPreservedSymbols.TryGetValue (componentName, out HashSet preservedSymbols)) { + foreach (LlvmIrGlobalVariableReference vref in preservedSymbols) { + DeclareDummyFunction (module, vref); + symbolsToExplicitlyPreserve.Add (vref); + } + componentPreservedSymbols.Remove (componentName); + } + } + if (!preservedPerComponent.TryGetValue (pinfo.LibraryName, out Component? component)) { component = new Component (pinfo.LibraryName, is64Bit); preservedPerComponent.Add (component.Name, component); @@ -174,6 +187,17 @@ protected override void Construct (LlvmIrModule module) component.Add (module, pinfo); } + module.AddGlobalVariable ("__jni_on_load_handler_count", (uint)jniOnLoadNames.Count, LlvmIrVariableOptions.GlobalConstant); + var jniOnLoadPointers = new List (); + foreach (string name in jniOnLoadNames) { + var symref = new LlvmIrGlobalVariableReference (name); + jniOnLoadPointers.Add (symref); + DeclareDummyFunction (module, symref); + } + module.AddGlobalVariable ("__jni_on_load_handlers", jniOnLoadPointers, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__jni_on_load_handler_names", jniOnLoadNames, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__explicitly_preserved_symbols", symbolsToExplicitlyPreserve, LlvmIrVariableOptions.GlobalConstant); + var components = new List (preservedPerComponent.Values); if (is64Bit) { AddFindPinvoke (module, components, is64Bit); @@ -328,40 +352,40 @@ LlvmIrFunctionAttributeSet MakeFindPinvokeAttributeSet (LlvmIrModule module) // Returns `true` for all p/invokes that we know are part of our set of components, otherwise returns `false`. // Returning `false` merely means that the p/invoke isn't in any of BCL or our code and therefore we shouldn't // care. It doesn't mean the p/invoke will be removed in any way. - bool MustPreserve (PinvokeScanner.PinvokeEntryInfo pinfo, ICollection components) + (bool preserve, string? componentName) MustPreserve (PinvokeScanner.PinvokeEntryInfo pinfo, ICollection components) { if (String.Compare ("xa-internal-api", pinfo.LibraryName, StringComparison.Ordinal) == 0) { - return true; + return (true, null); } foreach (string component in components) { // The most common pattern for the BCL - file name without extension string componentName = Path.GetFileNameWithoutExtension (component); if (Matches (pinfo.LibraryName, componentName)) { - return true; + return (true, componentName); } // If it starts with `lib`, drop the prefix if (componentName.StartsWith ("lib", StringComparison.Ordinal)) { if (Matches (pinfo.LibraryName, componentName.Substring (3))) { - return true; + return (true, componentName); } } // Might require mapping of component name to a canonical one if (libraryNameMap.TryGetValue (componentName, out string? mappedComponentName) && !String.IsNullOrEmpty (mappedComponentName)) { if (Matches (pinfo.LibraryName, mappedComponentName)) { - return true; + return (true, componentName); } } // Try full file name, as the last resort if (Matches (pinfo.LibraryName, Path.GetFileName (component))) { - return true; + return (true, componentName); } } - return false; + return (false, null); bool Matches (string libraryName, string componentName) { @@ -369,10 +393,10 @@ bool Matches (string libraryName, string componentName) } } - static void DeclareDummyFunction (LlvmIrModule module, string name) + static void DeclareDummyFunction (LlvmIrModule module, LlvmIrGlobalVariableReference symref) { // Just a dummy declaration, we don't care about the arguments - var funcSig = new LlvmIrFunctionSignature (name, returnType: typeof(void)); + var funcSig = new LlvmIrFunctionSignature (symref.Name, returnType: typeof(void)); var _ = module.DeclareExternalFunction (funcSig); } } From a9538f7e31421d7d0ee2e45d0919f389b7860bd8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Mar 2025 19:35:51 +0100 Subject: [PATCH 09/41] Better. Next step, dotnet/runtime libs (not in runtime pack, alas) --- .../Microsoft.Android.Runtime.proj | 20 ++++++++++--------- .../Tasks/GetNativeRuntimeComponents.cs | 4 ++++ .../Utilities/NativeRuntimeComponents.cs | 9 +++++---- .../Xamarin.Android.Common.targets | 3 ++- src/native/clr/host/CMakeLists.txt | 2 -- src/native/clr/runtime-base/CMakeLists.txt | 2 ++ .../internal-pinvokes.cc | 0 .../{host => runtime-base}/jni-remapping.cc | 0 8 files changed, 24 insertions(+), 16 deletions(-) rename src/native/clr/{host => runtime-base}/internal-pinvokes.cc (100%) rename src/native/clr/{host => runtime-base}/jni-remapping.cc (100%) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index a0c5aa8cbd3..085a084c6a0 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -51,22 +51,24 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release-static-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release-static-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index 0f6587035f0..06ca2eaa208 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -44,8 +44,11 @@ public override bool RunTask () var archives = new List (); var symbolsToExport = new List (); + Log.LogDebugMessage ($"Generating list of native files for linking"); foreach (NativeRuntimeComponents.Archive archiveItem in components.KnownArchives) { + Log.LogDebugMessage ($" archive '{archiveItem.Name}'"); if (!archiveItem.Include) { + Log.LogDebugMessage (" will not be included"); continue; } MakeArchiveItem (archiveItem, archives, uniqueAbis); @@ -108,6 +111,7 @@ void MakeArchiveItem (NativeRuntimeComponents.Archive archive, List a continue; } + Log.LogDebugMessage ($" creating msbuild item for archive '{archive.Name}'"); ITaskItem newItem = DoMakeItem ("_ResolvedNativeArchive", resolvedArchive, uniqueAbis); newItem.SetMetadata (KnownMetadata.NativeLinkWholeArchive, archive.WholeArchive.ToString ()); if (archive.DontExportSymbols) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 1fa709fbffe..7544700fa92 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -36,8 +36,8 @@ public ClangBuiltinsArchive (string clangAbi) class AndroidArchive : Archive { - public AndroidArchive (string name) - : base (name, wholeArchive: false) + public AndroidArchive (string name, bool wholeArchive = false) + : base (name, wholeArchive: wholeArchive) {} } @@ -87,12 +87,13 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) }, // .NET for Android - new AndroidArchive ("libpinvoke-override-dynamic-release.a"), + new AndroidArchive ("libnet-android.release-static-release.a", wholeArchive: true), + new AndroidArchive ("libpinvoke-override-dynamic-release.a", wholeArchive: true), new AndroidArchive ("libruntime-base-release.a"), new AndroidArchive ("libxa-java-interop-release.a"), new AndroidArchive ("libxa-lz4-release.a"), new AndroidArchive ("libxa-shared-bits-release.a"), - new AndroidArchive ("libmono-android.release-static-release.a"), + new AndroidArchive ("libxamarin-startup-release.a"), // LLVM clang built-ins archives new ClangBuiltinsArchive ("aarch64"), diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 5ff83e67d9f..3944e07d0ff 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -367,7 +367,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidUseManagedMarshalMethodsLookup Condition=" '$(_AndroidUseManagedMarshalMethodsLookup)' == '' ">False - <_AndroidEnableNativeRuntimeLinking Condition=" '$(_AndroidEnableNativeRuntimeLinking)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' And '$(_AndroidRuntime)' == 'CoreCLR'">true + <_AndroidEnableNativeRuntimeLinking Condition=" '$(_AndroidRuntime)' != 'CoreCLR' ">false + <_AndroidEnableNativeRuntimeLinking Condition=" '$(_AndroidEnableNativeRuntimeLinking)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' And '$(_AndroidRuntime)' == 'CoreCLR' ">true diff --git a/src/native/clr/host/CMakeLists.txt b/src/native/clr/host/CMakeLists.txt index 1fb9332c4cc..ad81c83637e 100644 --- a/src/native/clr/host/CMakeLists.txt +++ b/src/native/clr/host/CMakeLists.txt @@ -30,8 +30,6 @@ set(XAMARIN_MONODROID_SOURCES host.cc host-jni.cc host-util.cc - internal-pinvokes.cc - jni-remapping.cc os-bridge.cc typemap.cc xamarin_getifaddrs.cc diff --git a/src/native/clr/runtime-base/CMakeLists.txt b/src/native/clr/runtime-base/CMakeLists.txt index 97930c69e14..2fad890b2b8 100644 --- a/src/native/clr/runtime-base/CMakeLists.txt +++ b/src/native/clr/runtime-base/CMakeLists.txt @@ -52,6 +52,8 @@ set(LIB_ALIAS xa::runtime-base) set(XA_RUNTIME_BASE_SOURCES android-system.cc cpu-arch-detect.cc + internal-pinvokes.cc + jni-remapping.cc logger.cc util.cc ) diff --git a/src/native/clr/host/internal-pinvokes.cc b/src/native/clr/runtime-base/internal-pinvokes.cc similarity index 100% rename from src/native/clr/host/internal-pinvokes.cc rename to src/native/clr/runtime-base/internal-pinvokes.cc diff --git a/src/native/clr/host/jni-remapping.cc b/src/native/clr/runtime-base/jni-remapping.cc similarity index 100% rename from src/native/clr/host/jni-remapping.cc rename to src/native/clr/runtime-base/jni-remapping.cc From 64c8d2310e1533f7657c4b2d608e35ab928a2032 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 21 Mar 2025 20:07:31 +0100 Subject: [PATCH 10/41] Some progress, still ways to go. --- .../Microsoft.Android.Runtime.proj | 2 + .../xaprepare/Steps/Step_Android_SDK_NDK.cs | 13 ++- ...icrosoft.Android.Sdk.NativeRuntime.targets | 3 +- .../Tasks/GetNativeRuntimeComponents.cs | 91 +++++++++++++++++++ .../Utilities/NativeRuntimeComponents.cs | 40 +++++++- .../include/runtime-base/internal-pinvokes.hh | 17 +--- 6 files changed, 139 insertions(+), 27 deletions(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 085a084c6a0..6efe0b98f15 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -72,6 +72,8 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++_static.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++_static.a" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++abi.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++abi.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-aarch64-android.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-aarch64-android.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-x86_64-android.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-x86_64-android.a" /> diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index 17181c8f567..3b8da990135 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -198,14 +198,10 @@ bool CopyRedistributableFiles (Context context) foreach (var kvp in Configurables.Defaults.AndroidToolchainPrefixes) { string abi = kvp.Key; - - string crtFilesPath = Path.Combine ( - Configurables.Paths.AndroidToolchainSysrootLibDirectory, - kvp.Value, - BuildAndroidPlatforms.NdkMinimumAPI.ToString (CultureInfo.InvariantCulture) - ); - + string abiDir = Path.Combine (Configurables.Paths.AndroidToolchainSysrootLibDirectory, kvp.Value); + string crtFilesPath = Path.Combine (abiDir, BuildAndroidPlatforms.NdkMinimumAPI.ToString (CultureInfo.InvariantCulture)); string clangArch = Configurables.Defaults.AbiToClangArch[abi]; + CopyFile (abi, crtFilesPath, "crtbegin_so.o"); CopyFile (abi, crtFilesPath, "crtend_so.o"); CopyFile (abi, clangLibPath, $"libclang_rt.builtins-{clangArch}-android.a"); @@ -215,6 +211,9 @@ bool CopyRedistributableFiles (Context context) clangArch = "i386"; } + CopyFile (abi, abiDir, "libc++_static.a"); + CopyFile (abi, abiDir, "libc++abi.a"); + // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released CopyFile (abi, Path.Combine (clangLibPath, clangArch), "libunwind.a"); } diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets index 59b62403a5b..a547c93b7b6 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets @@ -28,7 +28,8 @@ Contains code to build and link the native runtime at application build time. + ResolvedNativeArchives="@(_ResolvedNativeArchive)" + HackLocalClrRepoPath="$(_HackLocalClrRuntimePath)"> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index 06ca2eaa208..fb8a7e9f471 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -20,6 +20,9 @@ public class GetNativeRuntimeComponents : AndroidTask [Required] public ITaskItem[] ResolvedNativeObjectFiles { get; set; } + [Required] + public string HackLocalClrRepoPath { get; set; } + [Output] public ITaskItem[] NativeArchives { get; set; } @@ -60,6 +63,28 @@ public override bool RunTask () MakeLibItem (symbolName, symbolsToExport, uniqueAbis); } } + + // HACK! START: until CoreCLR runtime pack has the necessary .a archives + var discoveredItemNames = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (ITaskItem item in archives) { + discoveredItemNames.Add (Path.GetFileName (item.ItemSpec)); + } + + Log.LogWarning ("[HACK] Looking for native archives which require CoreCLR hack"); + foreach (NativeRuntimeComponents.Archive archiveItem in components.KnownArchives) { + if (!archiveItem.Include || !archiveItem.NeedsClrHack) { + continue; + } + + Log.LogDebugMessage ($" [HACK] archive {archiveItem.Name}"); + if (discoveredItemNames.Contains (archiveItem.Name)) { + Log.LogDebugMessage (" [HACK] already found elsewhere"); + continue; + } + HackMakeArchiveItem (archiveItem, archives, uniqueAbis); + } + // HACK! END + NativeArchives = archives.ToArray (); NativeSymbolsToExport = symbolsToExport.ToArray (); @@ -132,4 +157,70 @@ ITaskItem DoMakeItem (string msbuildItemName, ITaskItem sourceItem, HashSet archives, HashSet uniqueAbis) + { + var relativeArtifactPaths = new List<(string path, string abi)> (); + string archiveName = Path.GetFileName (archive.Name); + string commonClrObjDir = Path.Combine ("artifacts", "obj", "coreclr"); + + if (IsArchive ("libcoreclr.a")) { + archiveName = "libcoreclr_static.a"; + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "dlls", "mscoree", "coreclr")); + } else if (IsArchive ("libcoreclrpal.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "src")); + } else if (IsArchive ("libminipal.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "shared_minipal")); + } else if (IsArchive ("libcoreclrminipal.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "minipal", "Unix")); + } else if (IsArchive ("libgc_pal.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "gc", "unix")); + } else if (IsArchive ("libeventprovider.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "src", "eventprovider", "dummyprovider")); + } else if (IsArchive ("libnativeresourcestring.a")) { + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "nativeresources")); + } else { + foreach (string abi in uniqueAbis) { + string clrArch = GetClrArch (abi); + relativeArtifactPaths.Add ((Path.Combine ("artifacts", "bin", $"microsoft.netcore.app.runtime.android-{clrArch}", "Release", "runtimes", $"android-{clrArch}", "native"), abi)); + } + } + + foreach ((string relPath, string abi) in relativeArtifactPaths) { + string filePath = Path.Combine (HackLocalClrRepoPath, relPath, archiveName); + if (!File.Exists (filePath)) { + Log.LogWarning ($" [HACK] file {filePath} not found"); + continue; + } + Log.LogWarning ($" [HACK] adding runtime component '{filePath}'"); + var tempItem = new TaskItem (filePath); + tempItem.SetMetadata (KnownMetadata.Abi, abi); + tempItem.SetMetadata (KnownMetadata.RuntimeIdentifier, MonoAndroidHelper.AbiToRid (abi)); + ITaskItem newItem = DoMakeItem ("_ResolvedNativeArchive", tempItem, uniqueAbis); + newItem.SetMetadata (KnownMetadata.NativeLinkWholeArchive, archive.WholeArchive.ToString ()); + if (archive.DontExportSymbols) { + newItem.SetMetadata (KnownMetadata.NativeDontExportSymbols, "true"); + } + archives.Add (newItem); + } + + string GetClrArch (string abi) + { + return abi switch { + "arm64-v8a" => "arm64", + "x86_64" => "x64", + _ => throw new NotSupportedException ($"ABI {abi} is not supported for CoreCLR") + }; + } + + void MakeRelativeArtifactPaths (Func create) + { + foreach (string abi in uniqueAbis) { + string clrArch = GetClrArch (abi); + relativeArtifactPaths.Add ((create (clrArch), abi)); + } + } + + bool IsArchive (string name) => String.Compare (name, archiveName, StringComparison.OrdinalIgnoreCase) == 0; + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 7544700fa92..e2684889dfb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -15,15 +15,17 @@ internal class Archive public readonly bool WholeArchive; public bool DontExportSymbols { get; set; } public HashSet? SymbolsToPreserve { get; set; } + public readonly bool NeedsClrHack; Func shouldInclude; - public Archive (string name, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null) + public Archive (string name, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null, bool needsClrHack = false) { Name = name; shouldInclude = include == null ? ((Archive arch) => true) : include; WholeArchive = wholeArchive; JniOnLoadName = jniOnLoadName; + NeedsClrHack = needsClrHack; } } @@ -44,7 +46,7 @@ public AndroidArchive (string name, bool wholeArchive = false) sealed class BclArchive : Archive { public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName = null) - : base (name, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName) + : base (name, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName, needsClrHack: true) { DontExportSymbols = true; } @@ -57,16 +59,40 @@ public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName public readonly List LinkStartFiles; public readonly List LinkEndFiles; + // LINK_LIBRARIES = pal/src/eventprovider/dummyprovider/libeventprovider.a -llog nativeresources/libnativeresourcestring.a shared_minipal/libminipal.a -ldl -latomic -lm public NativeRuntimeComponents (ITaskItem[] monoComponents) { this.monoComponents = monoComponents; KnownArchives = new () { // CoreCLR runtime + BCL - new Archive ("libmonosgen-2.0.a") { + new Archive ("libcoreclr.a", needsClrHack: true) { DontExportSymbols = true, }, + new Archive ("libcoreclrminipal.a", needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libgc_pal.a", needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libcoreclrpal.a", wholeArchive: true, needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libeventprovider.a", needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libnativeresourcestring.a", needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libminipal.a", needsClrHack: true) { + DontExportSymbols = true, + }, + new Archive ("libbrotlicommon.a", needsClrHack: true), + new Archive ("libbrotlidec.a", needsClrHack: true), + new Archive ("libbrotlienc.a", needsClrHack: true), + new BclArchive ("libSystem.Globalization.Native.a"), new BclArchive ("libSystem.IO.Compression.Native.a"), + new BclArchive ("libSystem.IO.Ports.Native.a"), new BclArchive ("libSystem.Native.a"), new BclArchive ("libSystem.Security.Cryptography.Native.Android.a", jniOnLoadName: "AndroidCryptoNative_InitLibraryOnLoad") { SymbolsToPreserve = new (StringComparer.Ordinal) { @@ -101,6 +127,14 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) new ClangBuiltinsArchive ("i686"), new ClangBuiltinsArchive ("x86_64"), + // C++ standard library + new Archive ("libc++_static.a") { + DontExportSymbols = true, + }, + new Archive ("libc++abi.a") { + DontExportSymbols = true, + }, + // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released new Archive ("libunwind.a") { DontExportSymbols = true, diff --git a/src/native/clr/include/runtime-base/internal-pinvokes.hh b/src/native/clr/include/runtime-base/internal-pinvokes.hh index a3c69c6c6e7..d14210e8b8d 100644 --- a/src/native/clr/include/runtime-base/internal-pinvokes.hh +++ b/src/native/clr/include/runtime-base/internal-pinvokes.hh @@ -19,20 +19,5 @@ extern "C" { void monodroid_free (void *ptr) noexcept; const char* _monodroid_lookup_replacement_type (const char *jniSimpleReference); const JniRemappingReplacementMethod* _monodroid_lookup_replacement_method_info (const char *jniSourceType, const char *jniMethodName, const char *jniMethodSignature); - xamarin::android::managed_timing_sequence* monodroid_timing_start (const char *message); - void monodroid_timing_stop (xamarin::android::managed_timing_sequence *sequence, const char *message); - - void _monodroid_weak_gref_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); - int _monodroid_weak_gref_get (); - int _monodroid_max_gref_get (); - void _monodroid_weak_gref_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); - - void _monodroid_lref_log_new (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); - void _monodroid_lref_log_delete (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); - void _monodroid_gc_wait_for_bridge_processing (); - void* _monodroid_timezone_get_default_id (); - void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); - - int _monodroid_getifaddrs (struct ifaddrs **ifap); - void _monodroid_freeifaddrs (struct ifaddrs *ifa); } + From 4260d5aa6112cba37c667bf265b1b24559db31af Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 24 Mar 2025 16:53:47 +0100 Subject: [PATCH 11/41] Enable a bunch of p/invokes --- src/native/clr/host/CMakeLists.txt | 2 ++ .../{runtime-base => host}/internal-pinvokes.cc | 0 .../include/runtime-base/internal-pinvokes.hh | 17 ++++++++++++++++- src/native/clr/runtime-base/CMakeLists.txt | 1 - 4 files changed, 18 insertions(+), 2 deletions(-) rename src/native/clr/{runtime-base => host}/internal-pinvokes.cc (100%) diff --git a/src/native/clr/host/CMakeLists.txt b/src/native/clr/host/CMakeLists.txt index ad81c83637e..4c580916cc5 100644 --- a/src/native/clr/host/CMakeLists.txt +++ b/src/native/clr/host/CMakeLists.txt @@ -30,7 +30,9 @@ set(XAMARIN_MONODROID_SOURCES host.cc host-jni.cc host-util.cc + internal-pinvokes.cc os-bridge.cc + runtime-util.cc typemap.cc xamarin_getifaddrs.cc ) diff --git a/src/native/clr/runtime-base/internal-pinvokes.cc b/src/native/clr/host/internal-pinvokes.cc similarity index 100% rename from src/native/clr/runtime-base/internal-pinvokes.cc rename to src/native/clr/host/internal-pinvokes.cc diff --git a/src/native/clr/include/runtime-base/internal-pinvokes.hh b/src/native/clr/include/runtime-base/internal-pinvokes.hh index d14210e8b8d..a3c69c6c6e7 100644 --- a/src/native/clr/include/runtime-base/internal-pinvokes.hh +++ b/src/native/clr/include/runtime-base/internal-pinvokes.hh @@ -19,5 +19,20 @@ extern "C" { void monodroid_free (void *ptr) noexcept; const char* _monodroid_lookup_replacement_type (const char *jniSimpleReference); const JniRemappingReplacementMethod* _monodroid_lookup_replacement_method_info (const char *jniSourceType, const char *jniMethodName, const char *jniMethodSignature); -} + xamarin::android::managed_timing_sequence* monodroid_timing_start (const char *message); + void monodroid_timing_stop (xamarin::android::managed_timing_sequence *sequence, const char *message); + + void _monodroid_weak_gref_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); + int _monodroid_weak_gref_get (); + int _monodroid_max_gref_get (); + void _monodroid_weak_gref_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_lref_log_new (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_lref_log_delete (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_gc_wait_for_bridge_processing (); + void* _monodroid_timezone_get_default_id (); + void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); + + int _monodroid_getifaddrs (struct ifaddrs **ifap); + void _monodroid_freeifaddrs (struct ifaddrs *ifa); +} diff --git a/src/native/clr/runtime-base/CMakeLists.txt b/src/native/clr/runtime-base/CMakeLists.txt index 2fad890b2b8..ad37baf9bd9 100644 --- a/src/native/clr/runtime-base/CMakeLists.txt +++ b/src/native/clr/runtime-base/CMakeLists.txt @@ -52,7 +52,6 @@ set(LIB_ALIAS xa::runtime-base) set(XA_RUNTIME_BASE_SOURCES android-system.cc cpu-arch-detect.cc - internal-pinvokes.cc jni-remapping.cc logger.cc util.cc From 284063eb49b1fe936db3923a161190490e128867 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 24 Mar 2025 16:54:20 +0100 Subject: [PATCH 12/41] Fix archive path --- .../Tasks/GetNativeRuntimeComponents.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index fb8a7e9f471..4f78d85bf27 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -178,7 +178,7 @@ void HackMakeArchiveItem (NativeRuntimeComponents.Archive archive, List Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "src", "eventprovider", "dummyprovider")); } else if (IsArchive ("libnativeresourcestring.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "nativeresources")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "nativeresources")); } else { foreach (string abi in uniqueAbis) { string clrArch = GetClrArch (abi); From 12cf7f0294fa7b054f609a9a1d7680d481176787 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 25 Mar 2025 16:52:21 +0100 Subject: [PATCH 13/41] Make clr hack use configurable build config --- .../Tasks/GetNativeRuntimeComponents.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index 4f78d85bf27..300295adb5b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -163,26 +163,27 @@ void HackMakeArchiveItem (NativeRuntimeComponents.Archive archive, List (); string archiveName = Path.GetFileName (archive.Name); string commonClrObjDir = Path.Combine ("artifacts", "obj", "coreclr"); + const string config = "Release"; // or Debug if (IsArchive ("libcoreclr.a")) { archiveName = "libcoreclr_static.a"; - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "dlls", "mscoree", "coreclr")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "dlls", "mscoree", "coreclr")); } else if (IsArchive ("libcoreclrpal.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "src")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "pal", "src")); } else if (IsArchive ("libminipal.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "shared_minipal")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "shared_minipal")); } else if (IsArchive ("libcoreclrminipal.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "minipal", "Unix")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "minipal", "Unix")); } else if (IsArchive ("libgc_pal.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "gc", "unix")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "gc", "unix")); } else if (IsArchive ("libeventprovider.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "pal", "src", "eventprovider", "dummyprovider")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "pal", "src", "eventprovider", "dummyprovider")); } else if (IsArchive ("libnativeresourcestring.a")) { - MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.Release", "nativeresources")); + MakeRelativeArtifactPaths ((string clrArch) => Path.Combine (commonClrObjDir, $"android.{clrArch}.{config}", "nativeresources")); } else { foreach (string abi in uniqueAbis) { string clrArch = GetClrArch (abi); - relativeArtifactPaths.Add ((Path.Combine ("artifacts", "bin", $"microsoft.netcore.app.runtime.android-{clrArch}", "Release", "runtimes", $"android-{clrArch}", "native"), abi)); + relativeArtifactPaths.Add ((Path.Combine ("artifacts", "bin", $"microsoft.netcore.app.runtime.android-{clrArch}", config, "runtimes", $"android-{clrArch}", "native"), abi)); } } From 22b092098bb65c34551ffd4a4c121ada70a2e6d0 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 26 Mar 2025 20:24:03 +0100 Subject: [PATCH 14/41] Package actual NDK libs instead of stubs The NativeLinker helper and the LinkNativeRuntime tasks need to be modified slightly to allow more control over order of link items on the linker command line. --- Configuration.props | 1 + .../Microsoft.Android.Runtime.proj | 29 +++++++------ .../Application/Properties.Defaults.cs.in | 2 +- .../xaprepare/Steps/Step_Android_SDK_NDK.cs | 32 ++++++++++---- ...icrosoft.Android.Sdk.NativeRuntime.targets | 8 +++- .../Tasks/GetNativeRuntimeComponents.cs | 2 +- .../Tasks/LinkNativeRuntime.cs | 10 ++++- .../Utilities/NativeLinker.cs | 43 ++++++++++++++++--- .../Utilities/NativeRuntimeComponents.cs | 6 +-- src/native/CMakeLists.txt | 1 + src/native/native.targets | 11 +++++ 11 files changed, 111 insertions(+), 34 deletions(-) diff --git a/Configuration.props b/Configuration.props index 99e44109931..b8c74df5bc6 100644 --- a/Configuration.props +++ b/Configuration.props @@ -233,6 +233,7 @@ <_MonoRuntimeFlavorDirName>mono <_CLRRuntimeFlavorDirName>clr + <_RuntimeRedistDirName>redist diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 6efe0b98f15..64a156288c6 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -48,6 +48,10 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. $(IntermediateOutputPath)$(AndroidRID)\RuntimeList.xml <_RuntimeFlavorDirName Condition=" '$(AndroidRuntime)' == 'CoreCLR' ">$(_CLRRuntimeFlavorDirName) <_RuntimeFlavorDirName Condition=" '$(AndroidRuntime)' == 'Mono' Or '$(AndroidRuntime)' == '' ">$(_MonoRuntimeFlavorDirName) + <_ClangArch Condition=" '$(AndroidRID)' == 'android-arm64' ">aarch64 + <_ClangArch Condition=" '$(AndroidRID)' == 'android-arm' ">arm + <_ClangArch Condition=" '$(AndroidRID)' == 'android-x64' ">x86_64 + <_ClangArch Condition=" '$(AndroidRID)' == 'android-x86' ">i686 @@ -70,12 +74,12 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a" /> <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtbegin_so.o" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\crtend_so.o" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++_static.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++_static.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++abi.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc++abi.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-aarch64-android.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-aarch64-android.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-x86_64-android.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libclang_rt.builtins-x86_64-android.a" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\crtbegin_so.o" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\crtend_so.o" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc++_static.a" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc++abi.a" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libclang_rt.builtins-$(_ClangArch)-android.a" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libunwind.a" /> @@ -87,12 +91,13 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - - - - - - + + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libdl.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\liblog.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libm.so" /> + <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libz.so" /> + <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libarchive-dso-stub.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libarchive-dso-stub.so" /> diff --git a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in index 661ac9bf4d3..760bc66d865 100644 --- a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in +++ b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in @@ -58,7 +58,7 @@ namespace Xamarin.Android.Prepare properties.Add (KnownProperties.Pkg7Zip_CommandLine, StripQuotes (@"@Pkg7-Zip_CommandLine@")); properties.Add (KnownProperties.PkgXamarin_LibZipSharp, StripQuotes (@"@PkgXamarin_LibZipSharp@")); properties.Add (KnownProperties.ProductVersion, StripQuotes ("@ProductVersion@")); - properties.Add (KnownProperties.RuntimeRedistDirName, StripQuotes ("@_RuntimeRedistDirName@")); + properties.Add (KnownProperties.RuntimeRedistDirName, StripQuotes ("@_RuntimeRedistDirName@")); properties.Add (KnownProperties.XABuildToolsFolder, StripQuotes (@"@XABuildToolsFolder@")); properties.Add (KnownProperties.XABuildToolsVersion, StripQuotes ("@XABuildToolsVersion@")); properties.Add (KnownProperties.XABuildToolsPackagePrefixMacOS, StripQuotes ("@XABuildToolsPackagePrefixMacOS@")); diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index 3b8da990135..956744edb87 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -24,6 +24,8 @@ sealed class AndroidPackage #nullable enable static readonly string[] CRTFiles = { + "crtbegin_so.o", + "crtend_so.o", "libc.so", "libdl.so", "liblog.so", @@ -31,6 +33,15 @@ sealed class AndroidPackage "libz.so", }; + static readonly string[] CPPAbiFiles = { + "libc++_static.a", + "libc++abi.a", + }; + + static readonly string[] ClangArchFiles = { + "libunwind.a", + }; + bool RefreshSdk = false; bool RefreshNdk = false; AndroidToolchainComponentType DependencyTypeToInstall = AndroidToolchainComponentType.All; @@ -202,20 +213,25 @@ bool CopyRedistributableFiles (Context context) string crtFilesPath = Path.Combine (abiDir, BuildAndroidPlatforms.NdkMinimumAPI.ToString (CultureInfo.InvariantCulture)); string clangArch = Configurables.Defaults.AbiToClangArch[abi]; - CopyFile (abi, crtFilesPath, "crtbegin_so.o"); - CopyFile (abi, crtFilesPath, "crtend_so.o"); + foreach (string file in CRTFiles) { + CopyFile (abi, crtFilesPath, file); + } + + foreach (string file in CPPAbiFiles) { + CopyFile (abi, abiDir, file); + } + CopyFile (abi, clangLibPath, $"libclang_rt.builtins-{clangArch}-android.a"); // Yay, consistency if (String.Compare (clangArch, "i686", StringComparison.Ordinal) == 0) { clangArch = "i386"; } + string clangArchLibPath = Path.Combine (clangLibPath, clangArch); - CopyFile (abi, abiDir, "libc++_static.a"); - CopyFile (abi, abiDir, "libc++abi.a"); - - // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released - CopyFile (abi, Path.Combine (clangLibPath, clangArch), "libunwind.a"); + foreach (string file in ClangArchFiles) { + CopyFile (abi, clangArchLibPath, file); + } } return true; @@ -226,7 +242,7 @@ void CopyFile (string abi, string sourceDir, string fileName) string rid = Configurables.Defaults.AbiToRID [abi]; string outputDir = Path.Combine ( context.Properties.GetRequiredValue (KnownProperties.NativeRuntimeOutputRootDir), - context.Properties.GetRequiredValue (KnownProperties.CLRRuntimeFlavorDirName), + context.Properties.GetRequiredValue (KnownProperties.RuntimeRedistDirName), rid ); diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets index a547c93b7b6..cede002e0cd 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets @@ -43,9 +43,14 @@ Contains code to build and link the native runtime at application build time. Inputs="@(_NativeAssemblyTarget);@(_SelectedNativeArchive)" Outputs="@(_UnifiedNativeRuntime)" Condition=" '$(_AndroidEnableNativeRuntimeLinking)' == 'true' "> + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index 300295adb5b..eb6691ec4f6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -163,7 +163,7 @@ void HackMakeArchiveItem (NativeRuntimeComponents.Archive archive, List (); string archiveName = Path.GetFileName (archive.Name); string commonClrObjDir = Path.Combine ("artifacts", "obj", "coreclr"); - const string config = "Release"; // or Debug + const string config = "Debug"; // or Release if (IsArchive ("libcoreclr.a")) { archiveName = "libcoreclr_static.a"; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs index f14a6e14448..c96444c04db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs @@ -18,6 +18,9 @@ public class LinkNativeRuntime : AsyncTask [Required] public string AndroidBinUtilsDirectory { get; set; } + public string? AndroidNdkDirectory { get; set; } + public string? AndroidApiLevel { get; set; } + [Required] public string IntermediateOutputPath { get; set; } @@ -48,7 +51,8 @@ public class LinkNativeRuntime : AsyncTask [Required] public ITaskItem[] RuntimePackLibraryDirectories { get; set; } = Array.Empty (); - public bool SaveDebugSymbols { get; set; } + public bool SaveDebugSymbols { get; set; } = true; + public bool StripDebugSymbols { get; set; } = true; public override System.Threading.Tasks.Task RunTaskAsync () { @@ -66,7 +70,11 @@ void LinkRuntime (ITaskItem abiItem) } var linker = new NativeLinker (Log, abi, soname, AndroidBinUtilsDirectory, IntermediateOutputPath, RuntimePackLibraryDirectories, CancellationToken, Cancel) { + StripDebugSymbols = StripDebugSymbols, SaveDebugSymbols = SaveDebugSymbols, + UseNdkLibraries = true, + NdkRootPath = AndroidNdkDirectory, + NdkApiLevel = AndroidApiLevel, }; linker.Link ( outputRuntime, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 0279f195456..c68e1c42dd6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -15,7 +15,6 @@ class NativeLinker { static readonly List standardArgs = new () { "--shared", - "--allow-shlib-undefined", // TODO: need to enable zstd in binutils build // "--compress-debug-sections=zstd", // TODO: test the commented-out flags @@ -42,8 +41,12 @@ class NativeLinker readonly CancellationToken? cancellationToken; readonly Action? cancelTask; - public bool StripDebugSymbols { get; set; } - public bool SaveDebugSymbols { get; set; } + public bool StripDebugSymbols { get; set; } = true; + public bool SaveDebugSymbols { get; set; } = true; + public bool AllowUndefinedSymbols { get; set; } = false; + public bool UseNdkLibraries { get; set; } = false; + public string? NdkRootPath { get; set; } + public string? NdkApiLevel { get; set; } public NativeLinker (TaskLoggingHelper log, string abi, string soname, string binutilsDir, string intermediateDir, IEnumerable runtimePackLibDirs, CancellationToken? cancellationToken = null, Action? cancelTask = null) @@ -96,6 +99,16 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi public bool Link (ITaskItem outputLibraryPath, List objectFiles, List archives, List libraries, List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) { + if (UseNdkLibraries) { + if (String.IsNullOrEmpty (NdkRootPath)) { + throw new InvalidOperationException ("Internal error: request to use NDK libraries, but NDK root not specified."); + } + + if (String.IsNullOrEmpty (NdkApiLevel)) { + throw new InvalidOperationException ("Internal error: request to use NDK libraries, but NDK API level not specified."); + } + } + log.LogDebugMessage ($"Linking: {outputLibraryPath}"); EnsureCorrectAbi (outputLibraryPath); EnsureCorrectAbi (objectFiles); @@ -113,6 +126,17 @@ public bool Link (ITaskItem outputLibraryPath, List objectFiles, List sw.WriteLine (arg); } + if (AllowUndefinedSymbols) { + sw.WriteLine ("--allow-shlib-undefined"); + } else { + sw.WriteLine ("--no-undefined"); + } + + // This MUST go before extra args, since the NDK library path must take precedence over the path in extra args set in the ctor + if (UseNdkLibraries) { + sw.WriteLine ($"-L {MonoAndroidHelper.QuoteFileNameArgument (GetAbiNdkRootDir ())}"); + } + foreach (string arg in extraArgs) { sw.WriteLine (arg); } @@ -160,8 +184,7 @@ public bool Link (ITaskItem outputLibraryPath, List objectFiles, List return ret; } - ret = ExtractDebugSymbols (outputLibraryPath); - return ret; + return ExtractDebugSymbols (outputLibraryPath); void WriteFilesToResponseFile (StreamWriter sw, List files) { @@ -202,6 +225,15 @@ bool ParseBooleanMetadata (ITaskItem item, string metadata) } } + string GetAbiNdkRootDir () + { + // Let it throw if invalid + int apiLevel = Int32.Parse (NdkApiLevel); + NdkTools ndk = NdkTools.Create (NdkRootPath, logErrors: true, log: log); + + return ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, MonoAndroidHelper.AbiToTargetArch (abi), apiLevel); + } + void EnsureCorrectAbi (ITaskItem item) { // The exception is just a precaution, since the items passed to us should have already been checked @@ -243,6 +275,7 @@ bool ExtractDebugSymbols (ITaskItem outputSharedLibrary) stdoutLines.Clear (); stderrLines.Clear (); + args.Clear (); args.Add ("--strip-debug"); args.Add ("--strip-unneeded"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index e2684889dfb..40a7d64009c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -135,7 +135,6 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) DontExportSymbols = true, }, - // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released new Archive ("libunwind.a") { DontExportSymbols = true, }, @@ -146,12 +145,9 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) NativeLibraries = new () { "c", "dl", + "log", "m", "z", - "log", - - // Atomic is a static library in clang, need to investigate if it's really needed -// "atomic", }; // Files that will be linked before any other object/archive/library files diff --git a/src/native/CMakeLists.txt b/src/native/CMakeLists.txt index e383328fa46..dd03112bb84 100644 --- a/src/native/CMakeLists.txt +++ b/src/native/CMakeLists.txt @@ -592,6 +592,7 @@ if(BUILD_ARCHIVE_DSO_STUB) add_subdirectory(common/archive-dso-stub) else() add_subdirectory(common/java-interop) +# add_subdirectory(common/libstub) add_subdirectory(common/libunwind) add_subdirectory(common/lz4) diff --git a/src/native/native.targets b/src/native/native.targets index df70d1cab5f..0a8d197adc7 100644 --- a/src/native/native.targets +++ b/src/native/native.targets @@ -309,6 +309,17 @@ + + <_RuntimePackFiles Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\%(AndroidSupportedTargetJitAbi.AndroidRID)\*.o" + AndroidRID="%(AndroidSupportedTargetJitAbi.AndroidRID)" + AndroidRuntime="$(CMakeRuntimeFlavor)" + RuntimePackName="$(_RuntimePackName)" /> + <_RuntimePackFiles Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\%(AndroidSupportedTargetJitAbi.AndroidRID)\*.a" + AndroidRID="%(AndroidSupportedTargetJitAbi.AndroidRID)" + AndroidRuntime="$(CMakeRuntimeFlavor)" + RuntimePackName="$(_RuntimePackName)" /> + + Date: Thu, 27 Mar 2025 12:50:59 +0100 Subject: [PATCH 15/41] [WIP] reshaping linking --- .../Tasks/GetNativeRuntimeComponents.cs | 7 +- .../Tasks/LinkNativeRuntime.cs | 58 +++++++++++- .../Utilities/KnownMetadata.cs | 4 +- .../Utilities/NativeLinker.cs | 20 ++--- .../Utilities/NativeRuntimeComponents.cs | 88 ++++++++++--------- 5 files changed, 119 insertions(+), 58 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs index eb6691ec4f6..30b0568cd78 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -114,6 +114,7 @@ void MakeLibItem (string libName, List libraries, HashSet uni foreach (string abi in uniqueAbis) { var item = new TaskItem (libName); item.SetMetadata (KnownMetadata.Abi, abi); + item.SetMetadata (KnownMetadata.NativeSharedLibrary, "true"); libraries.Add (item); } } @@ -139,6 +140,7 @@ void MakeArchiveItem (NativeRuntimeComponents.Archive archive, List a Log.LogDebugMessage ($" creating msbuild item for archive '{archive.Name}'"); ITaskItem newItem = DoMakeItem ("_ResolvedNativeArchive", resolvedArchive, uniqueAbis); newItem.SetMetadata (KnownMetadata.NativeLinkWholeArchive, archive.WholeArchive.ToString ()); + newItem.SetMetadata (KnownMetadata.NativeLinkItemSet, archive.SetName); if (archive.DontExportSymbols) { newItem.SetMetadata (KnownMetadata.NativeDontExportSymbols, "true"); } @@ -163,7 +165,7 @@ void HackMakeArchiveItem (NativeRuntimeComponents.Archive archive, List (); string archiveName = Path.GetFileName (archive.Name); string commonClrObjDir = Path.Combine ("artifacts", "obj", "coreclr"); - const string config = "Debug"; // or Release + const string config = "Release"; if (IsArchive ("libcoreclr.a")) { archiveName = "libcoreclr_static.a"; @@ -190,7 +192,7 @@ void HackMakeArchiveItem (NativeRuntimeComponents.Archive archive, List items = OrganizeCommandLineItemsCLR (abi); + bool success = linker.Link ( outputRuntime, - GetAbiItems (NativeObjectFiles, "_NativeAssemblyTarget", abi), - GetAbiItems (NativeArchives, "_SelectedNativeArchive", abi), - GetAbiItems (LinkLibraries, "_RequiredLinkLibraries", abi), + items, GetAbiItems (NativeLinkStartFiles, "_NativeLinkStartFiles", abi), GetAbiItems (NativeLinkEndFiles, "_NativeLinkEndFiles", abi), GetAbiItems (NativeSymbolsToExport, "_NativeSymbolsToExport", abi) ); + + if (!success) { + Log.LogError ($"Failed to link native runtime {outputRuntime}"); + } + } + + // Puts object files, static archives in the correct order. This is a bit clumsy, but unfortunately necessary + List OrganizeCommandLineItemsCLR (string abi) + { + // Code farther down the method does NOT check whether a set is present, it assumes that. This is on purpose, to + // let the exception be thrown should a required (and assumed to be present) set be missing. + var sets = new Dictionary> (StringComparer.Ordinal); + foreach (ITaskItem item in GetAbiItems (NativeArchives, "_SelectedNativeArchive", abi)) { + string setName = item.GetRequiredMetadata ("_SelectedNativeArchive", KnownMetadata.NativeLinkItemSet, Log); + if (!sets.TryGetValue (setName, out List? items)) { + items = new List (); + sets.Add (setName, items); + } + + items.Add (item); + } + + var ret = new List (); + + // First go our own object files... + ret.AddRange (GetAbiItems (NativeObjectFiles, "_NativeAssemblyTarget", abi)); + + // ...then go our runtime archives... + ret.AddRange (sets[NativeRuntimeComponents.KnownSets.XamarinAndroidRuntime]); + + // ...followed by CoreCLR components... + ret.AddRange (sets[NativeRuntimeComponents.KnownSets.CoreClrRuntime]); + + // ...and after them the BCL PAL libraries... + ret.AddRange (sets[NativeRuntimeComponents.KnownSets.BCL]); + + // ...and then the C/C++ runtime libraries + var systemLibs = new Dictionary (StringComparer.Ordinal); + foreach (ITaskItem item in GetAbiItems (LinkLibraries, "_RequiredLinkLibraries", abi)) { + systemLibs.Add (Path.GetFileName (item.ItemSpec), item); + } + + ret.Add (systemLibs["log"]); + ret.AddRange (sets[NativeRuntimeComponents.KnownSets.CplusPlusRuntime]); + ret.Add (systemLibs["z"]); + ret.Add (systemLibs["m"]); + ret.Add (systemLibs["dl"]); + ret.Add (systemLibs["c"]); + + return ret; } List GetAbiItems (ITaskItem[] source, string itemName, string abi) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs index 0918aba990d..5bb5246b936 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs @@ -3,7 +3,9 @@ namespace Xamarin.Android.Tasks; static class KnownMetadata { public const string Abi = "Abi"; + public const string NativeDontExportSymbols = "DontExportSymbols"; + public const string NativeLinkItemSet = "NativeLinkItemSet"; public const string NativeLinkWholeArchive = "LinkWholeArchive"; + public const string NativeSharedLibrary = "NativeSharedLibrary"; public const string RuntimeIdentifier = "RuntimeIdentifier"; - public const string NativeDontExportSymbols = "DontExportSymbols"; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index c68e1c42dd6..23c5cf1f1bf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -96,8 +96,7 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (nativeLibsDir)}"); } - public bool Link (ITaskItem outputLibraryPath, List objectFiles, List archives, List libraries, - List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) + public bool Link (ITaskItem outputLibraryPath, List linkItems, List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) { if (UseNdkLibraries) { if (String.IsNullOrEmpty (NdkRootPath)) { @@ -111,9 +110,7 @@ public bool Link (ITaskItem outputLibraryPath, List objectFiles, List log.LogDebugMessage ($"Linking: {outputLibraryPath}"); EnsureCorrectAbi (outputLibraryPath); - EnsureCorrectAbi (objectFiles); - EnsureCorrectAbi (archives); - EnsureCorrectAbi (libraries); + EnsureCorrectAbi (linkItems); EnsureCorrectAbi (linkStartFiles); EnsureCorrectAbi (linkEndFiles); @@ -147,8 +144,7 @@ public bool Link (ITaskItem outputLibraryPath, List objectFiles, List var excludeExportsLibs = new List (); WriteFilesToResponseFile (sw, linkStartFiles); - WriteFilesToResponseFile (sw, objectFiles); - WriteFilesToResponseFile (sw, archives); + WriteFilesToResponseFile (sw, linkItems); if (exportDynamicSymbols != null && exportDynamicSymbols.Count > 0) { foreach (ITaskItem symbolItem in exportDynamicSymbols) { @@ -161,10 +157,6 @@ public bool Link (ITaskItem outputLibraryPath, List objectFiles, List sw.WriteLine ($"--exclude-libs={libs}"); } - foreach (ITaskItem libItem in libraries) { - sw.WriteLine ($"-l{libItem.ItemSpec}"); - } - WriteFilesToResponseFile (sw, linkEndFiles); sw.Flush (); @@ -197,7 +189,10 @@ void WriteFilesToResponseFile (StreamWriter sw, List files) if (wholeArchive) { sw.Write ("--whole-archive "); + } else if (IsNativeSharedLibrary (file)) { + sw.Write ("-l"); } + sw.Write (MonoAndroidHelper.QuoteFileNameArgument (file.ItemSpec)); // string abi = file.GetMetadata ("Abi") ?? String.Empty; // string destDir = Path.Combine ("/tmp/t", abi); @@ -212,6 +207,7 @@ void WriteFilesToResponseFile (StreamWriter sw, List files) bool IncludeWholeArchive (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeLinkWholeArchive); bool ExcludeFromExports (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeDontExportSymbols); + bool IsNativeSharedLibrary (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeSharedLibrary); bool ParseBooleanMetadata (ITaskItem item, string metadata) { @@ -381,6 +377,8 @@ bool RunCommand (string label, string binaryPath, List args, List? SymbolsToPreserve { get; set; } + public string SetName { get; } + public readonly bool NeedsClrHack; Func shouldInclude; - public Archive (string name, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null, bool needsClrHack = false) + public Archive (string name, string setName, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null, bool needsClrHack = false) { Name = name; + SetName = setName; shouldInclude = include == null ? ((Archive arch) => true) : include; WholeArchive = wholeArchive; JniOnLoadName = jniOnLoadName; @@ -32,21 +43,39 @@ public Archive (string name, Func? include = null, bool wholeArch sealed class ClangBuiltinsArchive : Archive { public ClangBuiltinsArchive (string clangAbi) - : base ($"libclang_rt.builtins-{clangAbi}-android.a") + : base ($"libclang_rt.builtins-{clangAbi}-android.a", KnownSets.CplusPlusRuntime) {} } class AndroidArchive : Archive { public AndroidArchive (string name, bool wholeArchive = false) - : base (name, wholeArchive: wholeArchive) + : base (name, KnownSets.XamarinAndroidRuntime, wholeArchive: wholeArchive) {} } sealed class BclArchive : Archive { public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName = null) - : base (name, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName, needsClrHack: true) + : base (name, KnownSets.BCL, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName, needsClrHack: true) + { + DontExportSymbols = true; + } + } + + sealed class ClrArchive : Archive + { + public ClrArchive (string name, bool wholeArchive = false) + : base (name, KnownSets.CoreClrRuntime, wholeArchive: wholeArchive, needsClrHack: true) + { + DontExportSymbols = true; + } + } + + sealed class CplusPlusArchive : Archive + { + public CplusPlusArchive (string name) + : base (name, KnownSets.CplusPlusRuntime) { DontExportSymbols = true; } @@ -59,36 +88,21 @@ public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName public readonly List LinkStartFiles; public readonly List LinkEndFiles; - // LINK_LIBRARIES = pal/src/eventprovider/dummyprovider/libeventprovider.a -llog nativeresources/libnativeresourcestring.a shared_minipal/libminipal.a -ldl -latomic -lm public NativeRuntimeComponents (ITaskItem[] monoComponents) { this.monoComponents = monoComponents; KnownArchives = new () { // CoreCLR runtime + BCL - new Archive ("libcoreclr.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libcoreclrminipal.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libgc_pal.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libcoreclrpal.a", wholeArchive: true, needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libeventprovider.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libnativeresourcestring.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libminipal.a", needsClrHack: true) { - DontExportSymbols = true, - }, - new Archive ("libbrotlicommon.a", needsClrHack: true), - new Archive ("libbrotlidec.a", needsClrHack: true), - new Archive ("libbrotlienc.a", needsClrHack: true), + new ClrArchive ("libcoreclr.a"), + new ClrArchive ("libcoreclrminipal.a"), + new ClrArchive ("libgc_pal.a"), + new ClrArchive ("libcoreclrpal.a", wholeArchive: true), + new ClrArchive ("libeventprovider.a"), + new ClrArchive ("libnativeresourcestring.a"), + new ClrArchive ("libminipal.a"), + new ClrArchive ("libbrotlicommon.a"), + new ClrArchive ("libbrotlidec.a"), + new ClrArchive ("libbrotlienc.a"), new BclArchive ("libSystem.Globalization.Native.a"), new BclArchive ("libSystem.IO.Compression.Native.a"), @@ -121,23 +135,17 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) new AndroidArchive ("libxa-shared-bits-release.a"), new AndroidArchive ("libxamarin-startup-release.a"), + // C++ standard library + new CplusPlusArchive ("libc++_static.a"), + new CplusPlusArchive ("libc++abi.a"), + // LLVM clang built-ins archives new ClangBuiltinsArchive ("aarch64"), new ClangBuiltinsArchive ("arm"), new ClangBuiltinsArchive ("i686"), new ClangBuiltinsArchive ("x86_64"), - // C++ standard library - new Archive ("libc++_static.a") { - DontExportSymbols = true, - }, - new Archive ("libc++abi.a") { - DontExportSymbols = true, - }, - - new Archive ("libunwind.a") { - DontExportSymbols = true, - }, + new CplusPlusArchive ("libunwind.a"), // techically it's from clang }; // Just the base names of libraries to link into the unified runtime. Must have all the dependencies of all the static archives we From 36b8d1c94d6813eca98e76ba985e88262f46b578 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 27 Mar 2025 15:22:18 +0100 Subject: [PATCH 16/41] Remove TODO and add a debug message --- src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 23c5cf1f1bf..7091a0dba0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -369,6 +369,7 @@ bool RunCommand (string label, string binaryPath, List args, List args, List Date: Thu, 27 Mar 2025 15:27:15 +0100 Subject: [PATCH 17/41] Strip debug symbols again --- .../targets/Microsoft.Android.Sdk.NativeRuntime.targets | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets index cede002e0cd..cfe78733e4e 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeRuntime.targets @@ -60,8 +60,6 @@ Contains code to build and link the native runtime at application build time. LinkLibraries="@(_RequiredLinkLibraries)" OutputRuntimes="@(_UnifiedNativeRuntime)" SupportedAbis="@(_BuildTargetAbis)" - SaveDebugSymbols="false" - StripDebugSymbols="false" RuntimePackLibraryDirectories="@(_RuntimePackLibraryDirectory)" /> From 860415cb314703290c69b0bcb0a2f1da04538fcd Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 28 Mar 2025 14:43:42 +0100 Subject: [PATCH 18/41] Some must-have flags --- src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs | 4 +++- .../Utilities/NativeRuntimeComponents.cs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 7091a0dba0a..2c51bf1bf8d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -25,11 +25,13 @@ class NativeLinker "-z relro", "-z noexecstack", "-z max-page-size=16384", + "-z now", // we need it for security reasons (without it PLT can be overwritten) "--enable-new-dtags", "--build-id=sha1", "--warn-shared-textrel", "--fatal-warnings", - "--no-rosegment" + "--no-rosegment", + "--eh-frame-hdr", // CoreCLR needs it }; readonly List extraArgs = new (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 79e96117ed5..11b63ff4092 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -96,7 +96,10 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) new ClrArchive ("libcoreclr.a"), new ClrArchive ("libcoreclrminipal.a"), new ClrArchive ("libgc_pal.a"), - new ClrArchive ("libcoreclrpal.a", wholeArchive: true), + + // CoreCLR links this archive whole, but it appears to work fine for us when linked in a normal way. + // Leave the flag commented, just in case we discover something not working right. + new ClrArchive ("libcoreclrpal.a"/*, wholeArchive: true*/), new ClrArchive ("libeventprovider.a"), new ClrArchive ("libnativeresourcestring.a"), new ClrArchive ("libminipal.a"), From 1d7a1c26c66020b4141811061938a4aefcfd80de Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 28 Mar 2025 18:13:53 +0100 Subject: [PATCH 19/41] Fix after rebase From 0c94c9198cadb97fcc82da685c74671be2ffccca Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 28 Mar 2025 18:25:07 +0100 Subject: [PATCH 20/41] Fix indentation --- .../xaprepare/Application/Properties.Defaults.cs.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in index 760bc66d865..6d63358fe13 100644 --- a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in +++ b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in @@ -22,7 +22,7 @@ namespace Xamarin.Android.Prepare properties.Add (KnownProperties.AndroidToolchainDirectory, StripQuotes (@"@AndroidToolchainDirectory@")); properties.Add (KnownProperties.AutoProvision, StripQuotes ("@AutoProvision@")); properties.Add (KnownProperties.AutoProvisionUsesSudo, StripQuotes ("@AutoProvisionUsesSudo@")); - properties.Add (KnownProperties.CLRRuntimeFlavorDirName, StripQuotes ("@_CLRRuntimeFlavorDirName@")); + properties.Add (KnownProperties.CLRRuntimeFlavorDirName, StripQuotes ("@_CLRRuntimeFlavorDirName@")); properties.Add (KnownProperties.CMakePath, StripQuotes (@"@CmakePath@")); properties.Add (KnownProperties.Configuration, StripQuotes ("@Configuration@")); properties.Add (KnownProperties.CommandLineToolsFolder, StripQuotes ("@CommandLineToolsFolder@")); @@ -49,16 +49,16 @@ namespace Xamarin.Android.Prepare properties.Add (KnownProperties.MicrosoftAndroidSdkOutDir, StripQuotes (@"@MicrosoftAndroidSdkOutDir@")); properties.Add (KnownProperties.MonoCecilVersion, StripQuotes ("@MonoCecilVersion@")); properties.Add (KnownProperties.MonoDarwinPackageUrl, StripQuotes ("@MonoDarwinPackageUrl@")); - properties.Add (KnownProperties.MonoRuntimeFlavorDirName, StripQuotes ("@_MonoRuntimeFlavorDirName@")); + properties.Add (KnownProperties.MonoRuntimeFlavorDirName, StripQuotes ("@_MonoRuntimeFlavorDirName@")); properties.Add (KnownProperties.MonoRequiredMinimumVersion, StripQuotes ("@MonoRequiredMinimumVersion@")); properties.Add (KnownProperties.MonoRequiredMaximumVersion, StripQuotes ("@MonoRequiredMaximumVersion@")); properties.Add (KnownProperties.MonoSourceFullPath, StripQuotes (@"@MonoSourceFullPath@")); - properties.Add (KnownProperties.NativeRuntimeOutputRootDir, StripQuotes (@"@NativeRuntimeOutputRootDir@")); + properties.Add (KnownProperties.NativeRuntimeOutputRootDir, StripQuotes (@"@NativeRuntimeOutputRootDir@")); properties.Add (KnownProperties.NinjaPath, StripQuotes (@"@NinjaPath@")); properties.Add (KnownProperties.Pkg7Zip_CommandLine, StripQuotes (@"@Pkg7-Zip_CommandLine@")); properties.Add (KnownProperties.PkgXamarin_LibZipSharp, StripQuotes (@"@PkgXamarin_LibZipSharp@")); properties.Add (KnownProperties.ProductVersion, StripQuotes ("@ProductVersion@")); - properties.Add (KnownProperties.RuntimeRedistDirName, StripQuotes ("@_RuntimeRedistDirName@")); + properties.Add (KnownProperties.RuntimeRedistDirName, StripQuotes ("@_RuntimeRedistDirName@")); properties.Add (KnownProperties.XABuildToolsFolder, StripQuotes (@"@XABuildToolsFolder@")); properties.Add (KnownProperties.XABuildToolsVersion, StripQuotes ("@XABuildToolsVersion@")); properties.Add (KnownProperties.XABuildToolsPackagePrefixMacOS, StripQuotes ("@XABuildToolsPackagePrefixMacOS@")); From 7132aa37436c938bac7daa59787c751d7e8b57c0 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 31 Mar 2025 13:02:06 +0200 Subject: [PATCH 21/41] Use NativeLinker for libxamarin-app.so too --- .../Tasks/LinkApplicationSharedLibraries.cs | 73 +------------------ .../Utilities/NativeLinker.cs | 42 ++++++++++- 2 files changed, 39 insertions(+), 76 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index 70fbc2c6dea..e1f0f81f0cd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -73,71 +73,12 @@ IEnumerable GetLinkerConfigs () RuntimePackLibraryDirectories, CancellationToken ) { + AllowUndefinedSymbols = true, StripDebugSymbols = !DebugBuild, SaveDebugSymbols = !DebugBuild, ZipAlignmentPages = ZipAlignmentPages, }; - string stripSymbolsArg = DebugBuild ? String.Empty : " -s"; - - string ld = Path.Combine (AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, "ld")); - var targetLinkerArgs = new List (); - foreach (var kvp in abis) { - string abi = kvp.Key; - InputFiles inputs = kvp.Value; - - targetLinkerArgs.Clear (); - string elf_arch; - uint maxPageSize; - switch (abi) { - case "armeabi-v7a": - targetLinkerArgs.Add ("-X"); - elf_arch = "armelf_linux_eabi"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); - break; - - case "arm64": - case "arm64-v8a": - case "aarch64": - targetLinkerArgs.Add ("--fix-cortex-a53-843419"); - elf_arch = "aarch64linux"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); - break; - - case "x86": - elf_arch = "elf_i386"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); - break; - - case "x86_64": - elf_arch = "elf_x86_64"; - maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); - break; - - default: - throw new NotSupportedException ($"Unsupported Android target architecture ABI: {abi}"); - } - - targetLinkerArgs.Add ("-m"); - targetLinkerArgs.Add (elf_arch); - - foreach (string file in inputs.ObjectFiles) { - targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (file)); - } - - targetLinkerArgs.Add ("-o"); - targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (inputs.OutputSharedLibrary)); - - if (inputs.ExtraLibraries != null) { - foreach (string lib in inputs.ExtraLibraries) { - targetLinkerArgs.Add (lib); - } - } - - targetLinkerArgs.Add ("-z"); - targetLinkerArgs.Add ($"max-page-size={maxPageSize}"); - - string targetArgs = String.Join (" ", targetLinkerArgs); yield return new Config { Linker = linker, LinkItems = GatherFilesForABI (item, abi, ObjectFiles), @@ -166,17 +107,5 @@ List GetItemsForABI (string abi, ITaskItem[] items) return ret; } - - void OnOutputData (string linkerName, object sender, DataReceivedEventArgs e) - { - if (e.Data != null) - LogMessage ($"[{linkerName} stdout] {e.Data}"); - } - - void OnErrorData (string linkerName, object sender, DataReceivedEventArgs e) - { - if (e.Data != null) - LogMessage ($"[{linkerName} stderr] {e.Data}"); - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 2c51bf1bf8d..a97c2cc9e5a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -31,7 +31,6 @@ class NativeLinker "--warn-shared-textrel", "--fatal-warnings", "--no-rosegment", - "--eh-frame-hdr", // CoreCLR needs it }; readonly List extraArgs = new (); @@ -47,8 +46,10 @@ class NativeLinker public bool SaveDebugSymbols { get; set; } = true; public bool AllowUndefinedSymbols { get; set; } = false; public bool UseNdkLibraries { get; set; } = false; + public bool TargetsCLR { get; set; } public string? NdkRootPath { get; set; } public string? NdkApiLevel { get; set; } + public int ZipAlignmentPages { get; set; } = AndroidZipAlign.DefaultZipAlignment64Bit; public NativeLinker (TaskLoggingHelper log, string abi, string soname, string binutilsDir, string intermediateDir, IEnumerable runtimePackLibDirs, CancellationToken? cancellationToken = null, Action? cancelTask = null) @@ -65,10 +66,12 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-soname {soname}"); string? elfArch = null; + uint maxPageSize; switch (abi) { case "armeabi-v7a": extraArgs.Add ("-X"); elfArch = "armelf_linux_eabi"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); break; case "arm64": @@ -76,14 +79,17 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi case "aarch64": extraArgs.Add ("--fix-cortex-a53-843419"); elfArch = "aarch64linux"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); break; case "x86": elfArch = "elf_i386"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (AndroidZipAlign.ZipAlignment32Bit); break; case "x86_64": elfArch = "elf_x86_64"; + maxPageSize = MonoAndroidHelper.ZipAlignmentToPageSize (ZipAlignmentPages); break; default: @@ -94,11 +100,27 @@ public NativeLinker (TaskLoggingHelper log, string abi, string soname, string bi extraArgs.Add ($"-m {elfArch}"); } + extraArgs.Add ($"-z max-page-size={maxPageSize}"); + string nativeLibsDir = MonoAndroidHelper.GetRuntimePackNativeLibDir (MonoAndroidHelper.AbiToTargetArch (abi), runtimePackLibDirs); extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (nativeLibsDir)}"); } - public bool Link (ITaskItem outputLibraryPath, List linkItems, List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) + /// + /// A helper method to create a task item that refers to a native library, for the purpose of linking + /// it into another library. `baseLibraryName` must be just the "stem" of the library name (e.g. `c` or `dl`) + /// without any paths, prefixes or extensions. The returned item will then be turned by the + /// method into the `-l` parameter passed to the native linker. + /// + public static ITaskItem MakeLibraryItem (string baseLibraryName, string abi) + { + ITaskItem libItem = new TaskItem (baseLibraryName); + libItem.SetMetadata (KnownMetadata.Abi, abi); + libItem.SetMetadata (KnownMetadata.NativeSharedLibrary, "true"); + return libItem; + } + + public bool Link (ITaskItem outputLibraryPath, List linkItems, List? linkStartFiles = null, List? linkEndFiles = null, ICollection? exportDynamicSymbols = null) { if (UseNdkLibraries) { if (String.IsNullOrEmpty (NdkRootPath)) { @@ -131,6 +153,10 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List linkItems, List files) + void WriteFilesToResponseFile (StreamWriter sw, List? files) { + if (files == null) { + return; + } + foreach (ITaskItem file in files) { bool wholeArchive = IncludeWholeArchive (file); @@ -243,8 +273,12 @@ void EnsureCorrectAbi (ITaskItem item) throw new InvalidOperationException ($"Internal error: '{item}' ABI ('{itemAbi}') doesn't have the expected value '{abi}'"); } - void EnsureCorrectAbi (List items) + void EnsureCorrectAbi (List? items) { + if (items == null) { + return; + } + foreach (ITaskItem item in items) { EnsureCorrectAbi (item); } From c90f569c6953b42fe60b103d7db5442a661e6755 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 31 Mar 2025 15:22:18 +0200 Subject: [PATCH 22/41] Use MonoAndroidHelper.RunProcess --- .../Utilities/NativeLinker.cs | 69 +++++-------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index a97c2cc9e5a..d259f4645be 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -356,62 +356,31 @@ bool RunLinker (List args, ITaskItem outputSharedLibrary) bool RunCommand (string label, string binaryPath, List args, List stdoutLines, List stderrLines) { - using var stdout_completed = new ManualResetEvent (false); - using var stderr_completed = new ManualResetEvent (false); - var psi = new ProcessStartInfo () { - FileName = binaryPath, - Arguments = String.Join (" ", args), - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WindowStyle = ProcessWindowStyle.Hidden, - }; - string binaryName = Path.GetFileName (ld); - log.LogDebugMessage ($"[{label}] {psi.FileName} {psi.Arguments}"); - - using var proc = new Process (); - proc.OutputDataReceived += (s, e) => { - if (e.Data != null) { + return MonoAndroidHelper.RunProcess ( + label, + binaryPath, + String.Join (" ", args), + log, + onOutput: (s, e) => { + if (e.Data == null) { + return; + } OnOutputData (binaryName, s, e); stdoutLines.Add (e.Data); - } else { - stdout_completed.Set (); - } - }; + }, + onError: (s, e) => { + if (e.Data != null) { + return; + } - proc.ErrorDataReceived += (s, e) => { - if (e.Data != null) { OnErrorData (binaryName, s, e); stderrLines.Add (e.Data); - } else { - stderr_completed.Set (); - } - }; - - proc.StartInfo = psi; - proc.Start (); - proc.BeginOutputReadLine (); - proc.BeginErrorReadLine (); - cancellationToken?.Register (() => { try { proc.Kill (); } catch (Exception) { } }); - proc.WaitForExit (); - - if (psi.RedirectStandardError) { - stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); - } - - if (psi.RedirectStandardOutput) { - stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); - } - - log.LogDebugMessage ($"[{label}] exit code == {proc.ExitCode}"); - if (proc.ExitCode != 0) { - cancelTask?.Invoke (); - return false; - } - - return true; + }, + cancellationToken, + cancelTask, + logWarningOnFailure: false + ) == 0; } void OnOutputData (string linkerName, object sender, DataReceivedEventArgs e) From b9fb57d673d2542c330af4c9dcdae765a39d6b92 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 1 Apr 2025 09:57:31 +0200 Subject: [PATCH 23/41] Fix typo --- src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index d259f4645be..c88e6bfc292 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -370,7 +370,7 @@ bool RunCommand (string label, string binaryPath, List args, List { - if (e.Data != null) { + if (e.Data == null) { return; } From b0d090baacb1605c938edc28415fc4544b0494d6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 1 Apr 2025 09:59:44 +0200 Subject: [PATCH 24/41] Quote response file path --- src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index c88e6bfc292..7bf370b36e4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -189,7 +189,7 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List { - $"@{respFilePath}", + $"@{MonoAndroidHelper.QuoteFileNameArgument (respFilePath)}", "-o", MonoAndroidHelper.QuoteFileNameArgument (outputLibraryPath.ItemSpec) }; From baba3873fe75185182cadcce0623d0055a8f234e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 1 Apr 2025 10:25:48 +0200 Subject: [PATCH 25/41] Remove library stubs --- src/native/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/native/CMakeLists.txt b/src/native/CMakeLists.txt index dd03112bb84..e383328fa46 100644 --- a/src/native/CMakeLists.txt +++ b/src/native/CMakeLists.txt @@ -592,7 +592,6 @@ if(BUILD_ARCHIVE_DSO_STUB) add_subdirectory(common/archive-dso-stub) else() add_subdirectory(common/java-interop) -# add_subdirectory(common/libstub) add_subdirectory(common/libunwind) add_subdirectory(common/lz4) From 2a7702721e828181709f27386e4a215c46b82d5a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Apr 2025 09:43:18 +0200 Subject: [PATCH 26/41] Fix after rebase --- .../Microsoft.Android.Runtime.proj | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 64a156288c6..b6800147047 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -45,7 +45,6 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. DependsOnTargets="_GetLicense;_GetDefaultPackageVersion" BeforeTargets="GetFilesToPackage" > - $(IntermediateOutputPath)$(AndroidRID)\RuntimeList.xml <_RuntimeFlavorDirName Condition=" '$(AndroidRuntime)' == 'CoreCLR' ">$(_CLRRuntimeFlavorDirName) <_RuntimeFlavorDirName Condition=" '$(AndroidRuntime)' == 'Mono' Or '$(AndroidRuntime)' == '' ">$(_MonoRuntimeFlavorDirName) <_ClangArch Condition=" '$(AndroidRID)' == 'android-arm64' ">aarch64 @@ -55,49 +54,48 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug-static-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.debug.so" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release-static-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release-static-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libnet-android.release.so" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libpinvoke-override-dynamic-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libruntime-base-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-java-interop-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-lz4-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxa-shared-bits-release.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-debug.a" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-startup-release.a" /> + + + + + + + + + + + + + + + + + + - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\crtbegin_so.o" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\crtend_so.o" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc++_static.a" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc++abi.a" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libclang_rt.builtins-$(_ClangArch)-android.a" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libunwind.a" /> + + + + + + - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libmono-android.debug.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libmono-android.release.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-debug-app-helper.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libxamarin-native-tracing.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin-release.a" /> + + + + + - - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libc.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libdl.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\liblog.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libm.so" /> - <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeRedistDirName)\$(AndroidRID)\libz.so" /> - <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libarchive-dso-stub.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libarchive-dso-stub.so" /> + + + + + + From 9a257d215c7b1b3a5799a96f29621facf86dcd7d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Apr 2025 11:37:45 +0200 Subject: [PATCH 27/41] Post-rebase fixlet --- build-tools/create-packs/Microsoft.Android.Runtime.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index b6800147047..8595c5118ae 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -86,7 +86,8 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - + + From e5d053193298319d3e35b8a8588c865211f4a390 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Apr 2025 13:08:14 +0200 Subject: [PATCH 28/41] Fix a typo --- .../Microsoft.Android.Runtime.proj | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 8595c5118ae..7b1d240feb1 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -54,31 +54,31 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + @@ -91,12 +91,12 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. - - - - - - + + + + + + From 01087311e2ffcf9dba8a5df96337b77289bdce0f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Apr 2025 13:19:30 +0200 Subject: [PATCH 29/41] Link `libcoreclrpal.a` whole, just like upstream does --- .../Utilities/NativeRuntimeComponents.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 11b63ff4092..9a92b88751d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -97,9 +97,7 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) new ClrArchive ("libcoreclrminipal.a"), new ClrArchive ("libgc_pal.a"), - // CoreCLR links this archive whole, but it appears to work fine for us when linked in a normal way. - // Leave the flag commented, just in case we discover something not working right. - new ClrArchive ("libcoreclrpal.a"/*, wholeArchive: true*/), + new ClrArchive ("libcoreclrpal.a", wholeArchive: true), new ClrArchive ("libeventprovider.a"), new ClrArchive ("libnativeresourcestring.a"), new ClrArchive ("libminipal.a"), From 0da459c6698a762a6d7a3fd0c708073d2d9abc6e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 22 Apr 2025 12:17:38 +0200 Subject: [PATCH 30/41] Update after the recent timing changes --- build-tools/create-packs/Microsoft.Android.Runtime.proj | 2 ++ .../Utilities/NativeRuntimeComponents.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 7b1d240feb1..53db6c52f87 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -62,6 +62,8 @@ projects that use the Microsoft.Android.Runtimes framework in .NET 6+. + + diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 9a92b88751d..59c503a6b68 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -130,6 +130,7 @@ public NativeRuntimeComponents (ITaskItem[] monoComponents) // .NET for Android new AndroidArchive ("libnet-android.release-static-release.a", wholeArchive: true), new AndroidArchive ("libpinvoke-override-dynamic-release.a", wholeArchive: true), + new AndroidArchive ("libruntime-base-common-release.a"), new AndroidArchive ("libruntime-base-release.a"), new AndroidArchive ("libxa-java-interop-release.a"), new AndroidArchive ("libxa-lz4-release.a"), From 7d5d43e6d24f505f52bd2e764ce10c827713b88e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 24 Apr 2025 09:33:12 +0200 Subject: [PATCH 31/41] Fix after rebase --- .../Tasks/GeneratePackageManagerJava.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 3c0980a3436..5dc669f7696 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -285,10 +285,11 @@ void AddEnvironment () pkgmgr.WriteLine ("}"); pkgmgr.Flush (); - ConcurrentDictionary? nativeCodeGenStates = null; + NativeCodeGenStateCollection? nativeCodeGenStates = null; + if (enableMarshalMethods || EnableNativeRuntimeLinking) { - nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( - MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), + nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal ( + MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), RegisteredTaskObjectLifetime.Build ); } From 6c304f1077b89c26bd1acf20d92643f15339133a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 24 Apr 2025 10:00:00 +0200 Subject: [PATCH 32/41] Post-rebase fixes --- .../Utilities/MarshalMethodCecilAdapter.cs | 5 ++++- .../Utilities/PreservePinvokesNativeAssemblyGenerator.cs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodCecilAdapter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodCecilAdapter.cs index 1fdb3ce1bf2..1378eeba9ba 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodCecilAdapter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodCecilAdapter.cs @@ -28,6 +28,8 @@ class MarshalMethodCecilAdapter var arch = kvp.Key; var state = kvp.Value; var obj = CreateNativeCodeGenState (arch, state); + obj.PinvokeInfos = state.PinvokeInfos; + obj.TargetArch = state.TargetArch; collection.States.Add (arch, obj); } @@ -156,7 +158,8 @@ class NativeCodeGenStateCollection class NativeCodeGenStateObject { public Dictionary> MarshalMethods { get; } = []; - public List<(string JniName, string AssemblyQualifiedName)> ApplicationsAndInstrumentationsToRegister { get; } = []; + public List? PinvokeInfos { get; set; } + public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; } class MarshalMethodEntryObject diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs index 2739cd2a0ad..9f5a40bef78 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs @@ -75,10 +75,10 @@ sealed class ConstructionState { "mono-android.release", String.Empty }, }; - readonly NativeCodeGenState state; + readonly NativeCodeGenStateObject state; readonly ITaskItem[] monoComponents; - public PreservePinvokesNativeAssemblyGenerator (TaskLoggingHelper log, NativeCodeGenState codeGenState, ITaskItem[] monoComponents) + public PreservePinvokesNativeAssemblyGenerator (TaskLoggingHelper log, NativeCodeGenStateObject codeGenState, ITaskItem[] monoComponents) : base (log) { if (codeGenState.PinvokeInfos == null) { From f114167890971f97a819db9a3496bd044a7ef472 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 24 Apr 2025 17:54:23 +0200 Subject: [PATCH 33/41] Add support for -Bsymbolic --- src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs | 1 + src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs index d755403d8c7..d1ca3e4ec23 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs @@ -73,6 +73,7 @@ void LinkRuntime (ITaskItem abiItem) StripDebugSymbols = StripDebugSymbols, SaveDebugSymbols = SaveDebugSymbols, UseNdkLibraries = true, + UseSymbolic = true, NdkRootPath = AndroidNdkDirectory, NdkApiLevel = AndroidApiLevel, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs index 7bf370b36e4..9e9c12d2341 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -47,6 +47,7 @@ class NativeLinker public bool AllowUndefinedSymbols { get; set; } = false; public bool UseNdkLibraries { get; set; } = false; public bool TargetsCLR { get; set; } + public bool UseSymbolic { get; set; } public string? NdkRootPath { get; set; } public string? NdkApiLevel { get; set; } public int ZipAlignmentPages { get; set; } = AndroidZipAlign.DefaultZipAlignment64Bit; @@ -157,6 +158,10 @@ public bool Link (ITaskItem outputLibraryPath, List linkItems, List Date: Fri, 25 Apr 2025 11:46:49 +0200 Subject: [PATCH 34/41] Fix after rebase --- .../Tasks/GeneratePackageManagerJava.cs | 402 +----------------- .../Xamarin.Android.Common.targets | 40 +- 2 files changed, 12 insertions(+), 430 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 5dc669f7696..9d721dcfbda 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -29,244 +29,10 @@ public override bool RunTask () // Write all the user assemblies pkgmgr.WriteLine ("public class MonoPackageManager_Resources {"); pkgmgr.WriteLine ("\tpublic static String[] Assemblies = new String[]{"); - - public ITaskItem[] NativeLibraries { get; set; } - - public ITaskItem[] MonoComponents { get; set; } - - public ITaskItem[] SatelliteAssemblies { get; set; } - - public bool UseAssemblyStore { get; set; } - - [Required] - public string OutputDirectory { get; set; } - - [Required] - public string EnvironmentOutputDirectory { get; set; } - - [Required] - public string IntermediateOutputDirectory { get; set; } = ""; - - [Required] - public string MainAssembly { get; set; } - - [Required] - public string TargetFrameworkVersion { get; set; } - - [Required] - public string Manifest { get; set; } - - [Required] - public string [] SupportedAbis { get; set; } - - [Required] - public string AndroidPackageName { get; set; } - - [Required] - public bool EnablePreloadAssembliesDefault { get; set; } - - [Required] - public bool TargetsCLR { get; set; } - - public bool EnableMarshalMethods { get; set; } - public bool EnableManagedMarshalMethodsLookup { get; set; } - public string RuntimeConfigBinFilePath { get; set; } - public string ProjectRuntimeConfigFilePath { get; set; } = String.Empty; - public string BoundExceptionType { get; set; } - - public string PackageNamingPolicy { get; set; } - public string Debug { get; set; } - public ITaskItem[] Environments { get; set; } - public string AndroidAotMode { get; set; } - public bool AndroidAotEnableLazyLoad { get; set; } - public bool EnableLLVM { get; set; } - public string HttpClientHandlerType { get; set; } - public string TlsProvider { get; set; } - public string AndroidSequencePointsMode { get; set; } - public bool EnableSGenConcurrent { get; set; } - public string? CustomBundleConfigFile { get; set; } - public bool EnableNativeRuntimeLinking { get; set; } - - bool _Debug { - get { - return string.Equals (Debug, "true", StringComparison.OrdinalIgnoreCase); - } - } - - public override bool RunTask () - { - var doc = AndroidAppManifest.Load (Manifest, MonoAndroidHelper.SupportedVersions); - int minApiVersion = doc.MinSdkVersion == null ? 4 : (int) doc.MinSdkVersion; - // We need to include any special assemblies in the Assemblies list - var mainFileName = Path.GetFileName (MainAssembly); - - using (var pkgmgr = MemoryStreamPool.Shared.CreateStreamWriter ()) { - pkgmgr.WriteLine ("package mono;"); - - // Write all the user assemblies - pkgmgr.WriteLine ("public class MonoPackageManager_Resources {"); - pkgmgr.WriteLine ("\tpublic static String[] Assemblies = new String[]{"); - - pkgmgr.WriteLine ("\t\t/* We need to ensure that \"{0}\" comes first in this list. */", mainFileName); - pkgmgr.WriteLine ("\t\t\"" + mainFileName + "\","); - foreach (var assembly in ResolvedUserAssemblies) { - if (string.Compare (Path.GetFileName (assembly.ItemSpec), mainFileName, StringComparison.OrdinalIgnoreCase) == 0) - continue; - pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly.ItemSpec) + "\","); - } - - // Write the assembly dependencies - pkgmgr.WriteLine ("\t};"); - pkgmgr.WriteLine ("\tpublic static String[] Dependencies = new String[]{"); - - //foreach (var assembly in assemblies.Except (args.Assemblies)) { - // if (args.SharedRuntime && !Toolbox.IsInSharedRuntime (assembly)) - // pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly) + "\","); - //} - - pkgmgr.WriteLine ("\t};"); - - pkgmgr.WriteLine ("}"); - pkgmgr.Flush (); - - // Only copy to the real location if the contents actually changed - var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); - - Files.CopyIfStreamChanged (pkgmgr.BaseStream, dest); - } - - AddEnvironment (); - - return !Log.HasLoggedErrors; - } - - static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) => MonoAndroidHelper.AbiToTargetArch (abi); - - static readonly string[] defaultLogLevel = {"MONO_LOG_LEVEL", "info"}; - static readonly string[] defaultMonoDebug = {"MONO_DEBUG", "gen-compact-seq-points"}; - static readonly string[] defaultHttpMessageHandler = {"XA_HTTP_CLIENT_HANDLER_TYPE", "System.Net.Http.HttpClientHandler, System.Net.Http"}; - static readonly string[] defaultTlsProvider = {"XA_TLS_PROVIDER", "btls"}; - - void AddEnvironment () - { - bool usesMonoAOT = false; - var environmentVariables = new Dictionary (StringComparer.Ordinal); - var systemProperties = new Dictionary (StringComparer.Ordinal); - - if (!Enum.TryParse (PackageNamingPolicy, out PackageNamingPolicy pnp)) { - pnp = PackageNamingPolicyEnum.LowercaseCrc64; - } - - AotMode aotMode = AotMode.None; - if (!string.IsNullOrEmpty (AndroidAotMode) && Aot.GetAndroidAotMode (AndroidAotMode, out aotMode) && aotMode != AotMode.None) { - usesMonoAOT = true; - } - - SequencePointsMode sequencePointsMode; - if (!Aot.TryGetSequencePointsMode (AndroidSequencePointsMode, out sequencePointsMode)) - sequencePointsMode = SequencePointsMode.None; - - // Even though environment files were potentially parsed in GenerateJavaStubs, we need to do it here again because we might have additional environment - // files (generated by us) which weren't present by the time GeneratJavaStubs ran. - var environmentParser = new EnvironmentFilesParser { - BrokenExceptionTransitions = false, - UsesAssemblyPreload = EnablePreloadAssembliesDefault, - }; - environmentParser.Parse (Environments, sequencePointsMode, Log); - - foreach (string line in environmentParser.EnvironmentVariableLines) { - AddEnvironmentVariableLine (line); - } - - if (_Debug && !environmentParser.HaveLogLevel) { - AddEnvironmentVariable (defaultLogLevel[0], defaultLogLevel[1]); - } - - if (sequencePointsMode != SequencePointsMode.None && !environmentParser.HaveMonoDebug) { - AddEnvironmentVariable (defaultMonoDebug[0], defaultMonoDebug[1]); - } - - if (!environmentParser.HaveHttpMessageHandler) { - if (HttpClientHandlerType == null) - AddEnvironmentVariable (defaultHttpMessageHandler[0], defaultHttpMessageHandler[1]); - else - AddEnvironmentVariable ("XA_HTTP_CLIENT_HANDLER_TYPE", HttpClientHandlerType.Trim ()); - } - - if (!environmentParser.HaveMonoGCParams) { - if (EnableSGenConcurrent) - AddEnvironmentVariable ("MONO_GC_PARAMS", "major=marksweep-conc"); - else - AddEnvironmentVariable ("MONO_GC_PARAMS", "major=marksweep"); - } - - global::Android.Runtime.BoundExceptionType boundExceptionType; - if (String.IsNullOrEmpty (BoundExceptionType) || String.Compare (BoundExceptionType, "System", StringComparison.OrdinalIgnoreCase) == 0) { - boundExceptionType = global::Android.Runtime.BoundExceptionType.System; - } else if (String.Compare (BoundExceptionType, "Java", StringComparison.OrdinalIgnoreCase) == 0) { - boundExceptionType = global::Android.Runtime.BoundExceptionType.Java; - } else { - throw new InvalidOperationException ($"Unsupported BoundExceptionType value '{BoundExceptionType}'"); - } - - int assemblyNameWidth = 0; - Encoding assemblyNameEncoding = Encoding.UTF8; - - Action updateNameWidth = (ITaskItem assembly) => { - if (UseAssemblyStore) { - return; - } - - string assemblyName = Path.GetFileName (assembly.ItemSpec); - int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; - if (nameBytes > assemblyNameWidth) { - assemblyNameWidth = nameBytes; - } - }; - - int assemblyCount = 0; - bool enableMarshalMethods = EnableMarshalMethods; - HashSet archAssemblyNames = null; - HashSet uniqueAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); - Action updateAssemblyCount = (ITaskItem assembly) => { - string? culture = MonoAndroidHelper.GetAssemblyCulture (assembly); - string fileName = Path.GetFileName (assembly.ItemSpec); - string assemblyName; - - if (String.IsNullOrEmpty (culture)) { - assemblyName = fileName; - } else { - assemblyName = $"{culture}/{fileName}"; - } - - if (!uniqueAssemblyNames.Contains (assemblyName)) { - uniqueAssemblyNames.Add (assemblyName); - } - - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); - archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); - - if (!archAssemblyNames.Contains (assemblyName)) { - assemblyCount++; - archAssemblyNames.Add (assemblyName); - } - }; - - if (SatelliteAssemblies != null) { - foreach (ITaskItem assembly in SatelliteAssemblies) { - updateNameWidth (assembly); - updateAssemblyCount (assembly); - } - } - - int android_runtime_jnienv_class_token = -1; - int jnienv_initialize_method_token = -1; - int jnienv_registerjninatives_method_token = -1; - foreach (var assembly in ResolvedAssemblies) { - updateNameWidth (assembly); - updateAssemblyCount (assembly); - - if (android_runtime_jnienv_class_token != -1) { + pkgmgr.WriteLine ("\t\t/* We need to ensure that \"{0}\" comes first in this list. */", mainFileName); + pkgmgr.WriteLine ("\t\t\"" + mainFileName + "\","); + foreach (var assembly in ResolvedUserAssemblies) { + if (string.Compare (Path.GetFileName (assembly.ItemSpec), mainFileName, StringComparison.OrdinalIgnoreCase) == 0) continue; pkgmgr.WriteLine ("\t\t\"" + Path.GetFileName (assembly.ItemSpec) + "\","); } @@ -285,163 +51,9 @@ void AddEnvironment () pkgmgr.WriteLine ("}"); pkgmgr.Flush (); - NativeCodeGenStateCollection? nativeCodeGenStates = null; - - if (enableMarshalMethods || EnableNativeRuntimeLinking) { - nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal ( - MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), - RegisteredTaskObjectLifetime.Build - ); - } - - bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); - var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); - LLVMIR.LlvmIrComposer appConfigAsmGen; - - if (TargetsCLR) { - Dictionary? runtimeProperties = RuntimePropertiesParser.ParseConfig (ProjectRuntimeConfigFilePath); - appConfigAsmGen = new ApplicationConfigNativeAssemblyGeneratorCLR (environmentVariables, systemProperties, runtimeProperties, Log) { - UsesAssemblyPreload = environmentParser.UsesAssemblyPreload, - AndroidPackageName = AndroidPackageName, - PackageNamingPolicy = pnp, - JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent, - NumberOfAssembliesInApk = assemblyCount, - BundledAssemblyNameWidth = assemblyNameWidth, - NativeLibraries = uniqueNativeLibraries, - AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, - JNIEnvInitializeToken = jnienv_initialize_method_token, - JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, - JniRemappingReplacementTypeCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementTypeCount, - JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, - MarshalMethodsEnabled = EnableMarshalMethods, - ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, - IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), - }; - } else { - appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { - UsesMonoAOT = usesMonoAOT, - UsesMonoLLVM = EnableLLVM, - UsesAssemblyPreload = environmentParser.UsesAssemblyPreload, - MonoAOTMode = aotMode.ToString ().ToLowerInvariant (), - AotEnableLazyLoad = AndroidAotEnableLazyLoad, - AndroidPackageName = AndroidPackageName, - BrokenExceptionTransitions = environmentParser.BrokenExceptionTransitions, - PackageNamingPolicy = pnp, - BoundExceptionType = boundExceptionType, - JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent, - HaveRuntimeConfigBlob = haveRuntimeConfigBlob, - NumberOfAssembliesInApk = assemblyCount, - BundledAssemblyNameWidth = assemblyNameWidth, - MonoComponents = (MonoComponent)monoComponents, - NativeLibraries = uniqueNativeLibraries, - HaveAssemblyStore = UseAssemblyStore, - AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, - JNIEnvInitializeToken = jnienv_initialize_method_token, - JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, - JniRemappingReplacementTypeCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementTypeCount, - JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, - MarshalMethodsEnabled = EnableMarshalMethods, - ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, - IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), - }; - } - LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); - - foreach (string abi in SupportedAbis) { - string targetAbi = abi.ToLowerInvariant (); - string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}"); - string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); - string? pinvokePreserveBaseAsmFilePath = EnableNativeRuntimeLinking ? Path.Combine (EnvironmentOutputDirectory, $"pinvoke_preserve.{targetAbi}") : null; - string environmentLlFilePath = $"{environmentBaseAsmFilePath}.ll"; - string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; - string? pinvokePreserveLlFilePath = pinvokePreserveBaseAsmFilePath != null ? $"{pinvokePreserveBaseAsmFilePath}.ll" : null; - AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); - - using var appConfigWriter = MemoryStreamPool.Shared.CreateStreamWriter (); - try { - appConfigAsmGen.Generate (appConfigModule, targetArch, appConfigWriter, environmentLlFilePath); - } catch { - throw; - } finally { - appConfigWriter.Flush (); - Files.CopyIfStreamChanged (appConfigWriter.BaseStream, environmentLlFilePath); - } - - MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; - if (enableMarshalMethods) { - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( - Log, - assemblyCount, - uniqueAssemblyNames, - EnsureCodeGenState (targetArch), - EnableManagedMarshalMethodsLookup - ); - } else { - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( - Log, - targetArch, - assemblyCount, - uniqueAssemblyNames - ); - } - - if (EnableNativeRuntimeLinking) { - var pinvokePreserveGen = new PreservePinvokesNativeAssemblyGenerator (Log, EnsureCodeGenState (targetArch), MonoComponents); - LLVMIR.LlvmIrModule pinvokePreserveModule = pinvokePreserveGen.Construct (); - using var pinvokePreserveWriter = MemoryStreamPool.Shared.CreateStreamWriter (); - try { - pinvokePreserveGen.Generate (pinvokePreserveModule, targetArch, pinvokePreserveWriter, pinvokePreserveLlFilePath); - } catch { - throw; - } finally { - pinvokePreserveWriter.Flush (); - Files.CopyIfStreamChanged (pinvokePreserveWriter.BaseStream, pinvokePreserveLlFilePath); - } - } - - LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); - using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); - try { - marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, marshalMethodsWriter, marshalMethodsLlFilePath); - } catch { - throw; - } finally { - marshalMethodsWriter.Flush (); - Files.CopyIfStreamChanged (marshalMethodsWriter.BaseStream, marshalMethodsLlFilePath); - } - } - - NativeCodeGenStateObject EnsureCodeGenState (AndroidTargetArch targetArch) - { - if (nativeCodeGenStates == null || !nativeCodeGenStates.States.TryGetValue (targetArch, out NativeCodeGenStateObject? state)) { - throw new InvalidOperationException ($"Internal error: missing native code generation state for architecture '{targetArch}'"); - } - - return state; - } - - void AddEnvironmentVariable (string name, string value) - { - if (Char.IsUpper(name [0]) || !Char.IsLetter(name [0])) - environmentVariables [ValidAssemblerString (name)] = ValidAssemblerString (value); - else - systemProperties [ValidAssemblerString (name)] = ValidAssemblerString (value); - } - - void AddEnvironmentVariableLine (string l) - { - string line = l?.Trim (); - if (String.IsNullOrEmpty (line) || line [0] == '#') - return; - - string[] nv = line.Split (new char[]{'='}, 2); - AddEnvironmentVariable (nv[0].Trim (), nv.Length < 2 ? String.Empty : nv[1].Trim ()); - } - - string ValidAssemblerString (string s) - { - return s.Replace ("\"", "\\\""); - } + // Only copy to the real location if the contents actually changed + var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); + Files.CopyIfStreamChanged (pkgmgr.BaseStream, dest); } return !Log.HasLoggedErrors; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 3944e07d0ff..2fae64887c0 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1893,40 +1893,9 @@ because xbuild doesn't support framework reference assemblies. + MainAssembly="$(TargetPath)" + OutputDirectory="$(_AndroidIntermediateJavaSourceDirectory)mono" + ResolvedUserAssemblies="@(_ResolvedUserAssemblies)"> + ProjectRuntimeConfigFilePath="$(ProjectRuntimeConfigFilePath)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)" > Date: Fri, 25 Apr 2025 12:17:43 +0200 Subject: [PATCH 35/41] Fixes after 'main' branch changes --- .../Tasks/GenerateMainAndroidManifest.cs | 9 +++++--- .../GenerateNativeMarshalMethodSources.cs | 22 ++++++++++++++++++- .../Tasks/GeneratePackageManagerJava.cs | 2 ++ .../Xamarin.Android.Common.targets | 8 +++++-- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs index 547af3de6a3..b07834f4835 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs @@ -25,6 +25,8 @@ public class GenerateMainAndroidManifest : AndroidTask public string? CheckedBuild { get; set; } public bool Debug { get; set; } public bool EmbedAssemblies { get; set; } + public bool EnableMarshalMethods { get; set; } + public bool EnableNativeRuntimeLinking { get; set; } [Required] public string IntermediateOutputDirectory { get; set; } = ""; public string []? ManifestPlaceholders { get; set; } @@ -64,9 +66,10 @@ public override bool RunTask () AdditionalProviderSources = additionalProviders.ToArray (); - // We still need the NativeCodeGenState for later tasks, but we're going to transfer - // it to a new object that doesn't require holding open Cecil AssemblyDefinitions. - var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates); + // If we still need the NativeCodeGenState in the task because we're using marshal methods, + // we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions. + if (UseMarshalMethods || EnableNativeRuntimeLinking) { + var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates); Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}"); BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateObject, RegisteredTaskObjectLifetime.Build); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeMarshalMethodSources.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeMarshalMethodSources.cs index 32868b4579a..b9749280bf0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeMarshalMethodSources.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeMarshalMethodSources.cs @@ -20,6 +20,10 @@ public class GenerateNativeMarshalMethodSources : AndroidTask public bool EnableMarshalMethods { get; set; } + public bool EnableNativeRuntimeLinking { get; set; } + + public ITaskItem[] MonoComponents { get; set; } = []; + [Required] public string EnvironmentOutputDirectory { get; set; } = ""; @@ -38,7 +42,7 @@ public override bool RunTask () { NativeCodeGenStateCollection? nativeCodeGenStates = null; - if (EnableMarshalMethods) { + if (EnableMarshalMethods || EnableNativeRuntimeLinking) { // Retrieve the stored NativeCodeGenStateCollection (and remove it from the cache) nativeCodeGenStates = BuildEngine4.UnregisterTaskObjectAssemblyLocal ( MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), @@ -57,7 +61,9 @@ void Generate (NativeCodeGenStateCollection? nativeCodeGenStates, string abi) var targetAbi = abi.ToLowerInvariant (); var targetArch = MonoAndroidHelper.AbiToTargetArch (abi); var marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); + var pinvokePreserveBaseAsmFilePath = EnableNativeRuntimeLinking ? Path.Combine (EnvironmentOutputDirectory, $"pinvoke_preserve.{targetAbi}") : null; var marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; + var pinvokePreserveLlFilePath = pinvokePreserveBaseAsmFilePath != null ? $"{pinvokePreserveBaseAsmFilePath}.ll" : null; var (assemblyCount, uniqueAssemblyNames) = GetAssemblyCountAndUniqueNames (); MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; @@ -79,6 +85,20 @@ void Generate (NativeCodeGenStateCollection? nativeCodeGenStates, string abi) ); } + if (EnableNativeRuntimeLinking) { + var pinvokePreserveGen = new PreservePinvokesNativeAssemblyGenerator (Log, EnsureCodeGenState (nativeCodeGenStates, targetArch), MonoComponents); + LLVMIR.LlvmIrModule pinvokePreserveModule = pinvokePreserveGen.Construct (); + using var pinvokePreserveWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + pinvokePreserveGen.Generate (pinvokePreserveModule, targetArch, pinvokePreserveWriter, pinvokePreserveLlFilePath!); + } catch { + throw; + } finally { + pinvokePreserveWriter.Flush (); + Files.CopyIfStreamChanged (pinvokePreserveWriter.BaseStream, pinvokePreserveLlFilePath!); + } + } + var marshalMethodsModule = marshalMethodsAsmGen.Construct (); using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 9d721dcfbda..a82c4c0408f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -29,6 +29,7 @@ public override bool RunTask () // Write all the user assemblies pkgmgr.WriteLine ("public class MonoPackageManager_Resources {"); pkgmgr.WriteLine ("\tpublic static String[] Assemblies = new String[]{"); + pkgmgr.WriteLine ("\t\t/* We need to ensure that \"{0}\" comes first in this list. */", mainFileName); pkgmgr.WriteLine ("\t\t\"" + mainFileName + "\","); foreach (var assembly in ResolvedUserAssemblies) { @@ -53,6 +54,7 @@ public override bool RunTask () // Only copy to the real location if the contents actually changed var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); + Files.CopyIfStreamChanged (pkgmgr.BaseStream, dest); } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 2fae64887c0..de43cabc1e3 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1660,6 +1660,8 @@ because xbuild doesn't support framework reference assemblies. CheckedBuild="$(_AndroidCheckedBuild)" Debug="$(AndroidIncludeDebugSymbols)" EmbedAssemblies="$(EmbedAssembliesIntoApk)" + EnableMarshalMethods="$(_AndroidUseMarshalMethods)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)" IntermediateOutputDirectory="$(IntermediateOutputPath)" ManifestPlaceholders="$(AndroidManifestPlaceholders)" ManifestTemplate="$(_AndroidManifestAbs)" @@ -1925,12 +1927,14 @@ because xbuild doesn't support framework reference assemblies. CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" TargetsCLR="$(_AndroidUseCLR)" ProjectRuntimeConfigFilePath="$(ProjectRuntimeConfigFilePath)" - EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)" > + > _CompileJava; _CreateApplicationSharedLibraries; - _LinkNativeRuntime; + _LinkNativeRuntime; _GetMonoPlatformJarPath; _GetLibraryImports; _SetProguardMappingFileProperty; From 6bf2b72bf2d8fb16edbf504df7bc7f3bfe93379d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 30 Apr 2025 13:48:06 +0200 Subject: [PATCH 36/41] Fix after rebase --- Configuration.props | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Configuration.props b/Configuration.props index b8c74df5bc6..deb5974bf17 100644 --- a/Configuration.props +++ b/Configuration.props @@ -236,6 +236,9 @@ <_RuntimeRedistDirName>redist + + <_RuntimeRedistDirName>redist +