From a04302dece9ce9bfaf2f7c5d582ea59194aec717 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 30 Jun 2021 15:48:04 +0200 Subject: [PATCH] [runtime/tools] Implement finding native support libraries when linking statically. Fixes #10950, #11145 and #12100. * Add support for Mono Components. * Modify how we look up symbols from native libraries shipped with Mono: we keep track of which native libraries we linked with, and depending on how we linked to those assemblies, we look the symbols up at runtime in either the current executable (if linking statically), or the actual library (where the P/Invoke says they're supposed to be). * This means that we have to propagate how libmono is linked from the MSBuild code to the Application class so that our existing logic is able to correctly determine which native mono lib to use. * Modify how we list the P/Invokes we need to preserve by taking into account the list of native libraries from Mono we have to link with (for .NET). For legacy Xamarin, I've reverted the logic to how it was before we started adding .NET support. Fixes https://github.com/xamarin/xamarin-macios/issues/10950. Fixes https://github.com/xamarin/xamarin-macios/issues/11145. Fixes https://github.com/xamarin/xamarin-macios/issues/12100. --- dotnet/targets/Xamarin.Shared.Sdk.targets | 50 ++++++++++++++++-- runtime/runtime.m | 52 ++++++++++++------- runtime/xamarin/main.h | 10 +++- tools/common/Application.cs | 1 + tools/common/Target.cs | 47 +++++++++-------- tools/dotnet-linker/LinkerConfiguration.cs | 8 +++ .../MonoTouch.Tuner/ListExportedSymbols.cs | 52 ++++++++----------- 7 files changed, 145 insertions(+), 75 deletions(-) diff --git a/dotnet/targets/Xamarin.Shared.Sdk.targets b/dotnet/targets/Xamarin.Shared.Sdk.targets index fa71441177f8..30fca568ee65 100644 --- a/dotnet/targets/Xamarin.Shared.Sdk.targets +++ b/dotnet/targets/Xamarin.Shared.Sdk.targets @@ -331,7 +331,40 @@ - + + + + <_MonoComponent Include="hot_reload" Condition="'$(MtouchInterpreter)' == 'true'" /> + <_MonoComponent Include="debugger" Condition="'$(_BundlerDebug)' == 'true'" /> + <_MonoComponent Include="diagnostics_tracing" Condition="'$(_BundlerDebug)' == 'true'" /> + + + + + + + + <_MonoLibrary Remove="@(_MonoRuntimeComponentDontLink -> '$(_MonoRuntimePackPath)/native/%(Identity)')" /> + + + <_MonoLibrary Include="@(_MonoRuntimeComponentLink -> '$(_MonoRuntimePackPath)/native/%(Identity)')" /> + + + + + <_ComputeLinkerArgumentsDependsOn> + _ComputeLinkMode; + _ComputeFrameworkVariables; + _ComputeFrameworkAssemblies; + ComputeResolvedFilesToPublishList; + _ParseBundlerArguments; + _ComputeVariables; + _CreateRuntimeConfiguration; + _ComputeMonoLibraries; + + + + @@ -376,6 +409,7 @@ LinkMode=$(_LinkMode) MarshalManagedExceptionMode=$(_MarshalManagedExceptionMode) MarshalObjectiveCExceptionMode=$(_MarshalObjectiveCExceptionMode) + @(_MonoLibrary -> 'MonoLibrary=%(Identity)') Optimize=$(_BundlerOptimize) PartialStaticRegistrarLibrary=$(_LibPartialStaticRegistrar) Platform=$(_PlatformName) @@ -836,9 +870,7 @@ <_XamarinMainLibraries Include="$(_XamarinNativeLibraryDirectory)/$(_LibXamarinName)" /> - <_XamarinMainLibraries Include="@(_MonoLibrary)" Condition="'$(_PlatformName)' != 'macOS'" /> - - <_XamarinMainLibraries Include="@(_MonoLibrary)" Condition="'$(_PlatformName)' == 'macOS' And ('%(_MonoLibrary.Extension)' == '.a' Or '%(_MonoLibrary.Filename)' == 'libcoreclr')" /> + <_XamarinMainLibraries Include="@(_MonoLibrary)" /> <_XamarinMainLibraries Include="@(_FileNativeReference)" /> @@ -906,7 +938,7 @@ - + <_AssemblyPublishDir Condition="'$(_PlatformName)' == 'iOS' Or '$(_PlatformName)' == 'tvOS' Or '$(_PlatformName)' == 'watchOS'">$(MSBuildProjectDirectory)\$(_AppBundlePath)\ <_AssemblyPublishDir Condition="'$(_PlatformName)' == 'macOS' Or '$(_PlatformName)' == 'MacCatalyst'">$(MSBuildProjectDirectory)\$(_AppBundlePath)\Contents\$(_CustomBundleName)\ @@ -973,6 +1005,14 @@ " /> + + diff --git a/runtime/runtime.m b/runtime/runtime.m index e44d11a0b114..39827bbdcc5f 100644 --- a/runtime/runtime.m +++ b/runtime/runtime.m @@ -83,7 +83,11 @@ enum XamarinLaunchMode xamarin_launch_mode = XamarinLaunchModeApp; bool xamarin_supports_dynamic_registration = true; const char *xamarin_runtime_configuration_name = NULL; -const char *xamarin_mono_native_lib_name = "__Internal"; + +#if DOTNET +enum XamarinNativeLinkMode xamarin_libmono_native_link_mode = XamarinNativeLinkModeStaticObject; +const char **xamarin_runtime_libraries = NULL; +#endif /* Callbacks */ @@ -2528,31 +2532,28 @@ -(void) xamarinSetFlags: (enum XamarinGCHandleFlags) flags; xamarin_assertion_message ("Failed to initialize the VM"); } +static bool +xamarin_is_native_library (const char *libraryName) +{ + if (xamarin_runtime_libraries == NULL) + return false; + + for (int i = 0; xamarin_runtime_libraries [i] != NULL; i++) { + if (!strcmp (xamarin_runtime_libraries [i], libraryName)) + return true; + } + + return false; +} + void* xamarin_pinvoke_override (const char *libraryName, const char *entrypointName) { void* symbol = NULL; -#if TARGET_OS_MACCATALYST - static void *monoNativeLibrary = NULL; -#endif - if (!strcmp (libraryName, "__Internal")) { symbol = dlsym (RTLD_DEFAULT, entrypointName); -#if TARGET_OS_MACCATALYST - } else if (!strcmp (libraryName, "libSystem.Native") || - !strcmp (libraryName, "libSystem.Security.Cryptography.Native.Apple") || - !strcmp (libraryName, "libSystem.Net.Security.Native")) { - if (monoNativeLibrary == NULL) { - if (xamarin_mono_native_lib_name == NULL || !strcmp (xamarin_mono_native_lib_name, "__Internal")) { - monoNativeLibrary = RTLD_DEFAULT; - } else { - monoNativeLibrary = dlopen (xamarin_mono_native_lib_name, RTLD_LAZY); - } - } - symbol = dlsym (monoNativeLibrary, entrypointName); -#endif // TARGET_OS_MACCATALYST #if !defined (CORECLR_RUNTIME) // we're intercepting objc_msgSend calls using the managed System.Runtime.InteropServices.ObjectiveC.Bridge.SetMessageSendCallback instead. #if defined (__i386__) || defined (__x86_64__) || defined (__arm64__) } else if (!strcmp (libraryName, "/usr/lib/libobjc.dylib")) { @@ -2573,6 +2574,21 @@ -(void) xamarinSetFlags: (enum XamarinGCHandleFlags) flags; } #endif // defined (__i386__) || defined (__x86_64__) || defined (__arm64__) #endif // !defined (CORECLR_RUNTIME) + } else if (xamarin_is_native_library (libraryName)) { + switch (xamarin_libmono_native_link_mode) { + case XamarinNativeLinkModeStaticObject: + // lookup the symbol in loaded memory, like __Internal does. + symbol = dlsym (RTLD_DEFAULT, entrypointName); + break; + case XamarinNativeLinkModeDynamicLibrary: + // if we're not linking statically, then don't do anything at all, let mono handle whatever needs to be done + return NULL; + case XamarinNativeLinkModeFramework: + default: + // handle this as "DynamicLibrary" for now - do nothing. + LOG (PRODUCT ": Unhandled libmono link mode: %i when looking up %s in %s", xamarin_libmono_native_link_mode, entrypointName, libraryName); + return NULL; + } } else { return NULL; } diff --git a/runtime/xamarin/main.h b/runtime/xamarin/main.h index bf07699f6596..646fbc54ad52 100644 --- a/runtime/xamarin/main.h +++ b/runtime/xamarin/main.h @@ -83,6 +83,13 @@ enum XamarinExceptionTypes : int { XamarinExceptionTypes_System_OutOfMemoryException, }; +// Keep in sync with AssemblyBuildTarget in AssemblyBuildTarget.cs +enum XamarinNativeLinkMode : int { + XamarinNativeLinkModeStaticObject, + XamarinNativeLinkModeDynamicLibrary, + XamarinNativeLinkModeFramework, +}; + extern bool mono_use_llvm; // this is defined inside mono #if DEBUG @@ -109,7 +116,8 @@ extern enum MarshalManagedExceptionMode xamarin_marshal_managed_exception_mode; extern enum XamarinLaunchMode xamarin_launch_mode; extern bool xamarin_supports_dynamic_registration; extern const char *xamarin_runtime_configuration_name; -extern const char *xamarin_mono_native_lib_name; +extern enum XamarinNativeLinkMode xamarin_libmono_native_link_mode; +extern const char** xamarin_runtime_libraries; typedef void (*xamarin_setup_callback) (); typedef int (*xamarin_extension_main_callback) (int argc, char** argv); diff --git a/tools/common/Application.cs b/tools/common/Application.cs index d7e6e967e7ec..2e07299fb202 100644 --- a/tools/common/Application.cs +++ b/tools/common/Application.cs @@ -89,6 +89,7 @@ public partial class Application public bool IsExtension; public ApplePlatform Platform { get { return Driver.TargetFramework.Platform; } } + public List MonoLibraries = new List (); public List InterpretedAssemblies = new List (); // EnableMSym: only implemented for Xamarin.iOS diff --git a/tools/common/Target.cs b/tools/common/Target.cs index abc38ab55fcb..855faef15b18 100644 --- a/tools/common/Target.cs +++ b/tools/common/Target.cs @@ -743,6 +743,16 @@ void GenerateIOSMain (StringWriter sw, Abi abi) sw.WriteLine ("extern \"C\" { void mono_sgen_mono_ilgen_init (void); }"); } +#if NET + if (app.MonoNativeMode != MonoNativeMode.None) { + sw.WriteLine ("static const char *xamarin_runtime_libraries_array[] = {"); + foreach (var lib in app.MonoLibraries) + sw.WriteLine ($"\t\"{Path.GetFileName (lib)}\","); + sw.WriteLine ($"\tNULL"); + sw.WriteLine ("};"); + } +#endif + sw.WriteLine ("void xamarin_setup_impl ()"); sw.WriteLine ("{"); @@ -773,33 +783,26 @@ void GenerateIOSMain (StringWriter sw, Abi abi) if (app.MonoNativeMode != MonoNativeMode.None) { #if NET - // Mono doesn't support dllmaps for Mac Catalyst / macOS in .NET for now: - // macOS: https://github.com/dotnet/runtime/issues/43204 - // Mac Catalyst: https://github.com/dotnet/runtime/issues/48110 - var addDllMap = App.Platform != ApplePlatform.MacCatalyst && app.Platform != ApplePlatform.MacOSX; + // Mono doesn't support dllmaps for Mac Catalyst / macOS in .NET, so we're using an alternative: + // the PINVOKE_OVERRIDE runtime option. Since we have to use it for Mac Catalyst + macOS, let's + // just use it everywhere to simplify code. This means that at runtime we need to know how we + // linked to mono, so store that in the xamarin_libmono_native_link_mode variable. + // Ref: https://github.com/dotnet/runtime/issues/43204 (macOS) https://github.com/dotnet/runtime/issues/48110 (Mac Catalyst) + sw.WriteLine ($"\txamarin_libmono_native_link_mode = XamarinNativeLinkMode{app.LibMonoNativeLinkMode};"); + sw.WriteLine ($"\txamarin_runtime_libraries = xamarin_runtime_libraries_array;"); #else - var addDllMap = true; -#endif string mono_native_lib; - if (app.LibMonoNativeLinkMode == AssemblyBuildTarget.StaticObject || Driver.IsDotNet) + if (app.LibMonoNativeLinkMode == AssemblyBuildTarget.StaticObject) { mono_native_lib = "__Internal"; - else - mono_native_lib = app.GetLibNativeName () + ".dylib"; - if (addDllMap) { - sw.WriteLine (); -#if NET - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Native\", NULL, \"{mono_native_lib}\", NULL);"); - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);"); - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);"); -#else - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Native\", NULL, \"{mono_native_lib}\", NULL);"); - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);"); - sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);"); -#endif - sw.WriteLine (); } else { - sw.WriteLine ($"\txamarin_mono_native_lib_name = \"{mono_native_lib}\";"); + mono_native_lib = app.GetLibNativeName () + ".dylib"; } + sw.WriteLine (); + sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Native\", NULL, \"{mono_native_lib}\", NULL);"); + sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);"); + sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);"); + sw.WriteLine (); +#endif } if (app.EnableDebug) diff --git a/tools/dotnet-linker/LinkerConfiguration.cs b/tools/dotnet-linker/LinkerConfiguration.cs index 49b8389ad43a..8e3186fc42f4 100644 --- a/tools/dotnet-linker/LinkerConfiguration.cs +++ b/tools/dotnet-linker/LinkerConfiguration.cs @@ -180,6 +180,9 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI Application.MarshalObjectiveCExceptions = mode; } break; + case "MonoLibrary": + Application.MonoLibraries.Add (value); + break; case "Optimize": user_optimize_flags = value; break; @@ -320,6 +323,8 @@ AssemblyBuildTarget ParseLinkMode (string value, string variableName) return AssemblyBuildTarget.DynamicLibrary; } else if (string.Equals (value, "static", StringComparison.OrdinalIgnoreCase)) { return AssemblyBuildTarget.StaticObject; + } else if (string.Equals (value, "framework", StringComparison.OrdinalIgnoreCase)) { + return AssemblyBuildTarget.Framework; } throw new InvalidOperationException ($"Invalid {variableName} '{value}' in {LinkerFile}"); @@ -347,6 +352,9 @@ public void Write () Console.WriteLine ($" LinkMode: {LinkMode}"); Console.WriteLine ($" MarshalManagedExceptions: {Application.MarshalManagedExceptions} (IsDefault: {Application.IsDefaultMarshalManagedExceptionMode})"); Console.WriteLine ($" MarshalObjectiveCExceptions: {Application.MarshalObjectiveCExceptions}"); + Console.WriteLine ($" {Application.MonoLibraries.Count} mono libraries:"); + foreach (var lib in Application.MonoLibraries.OrderBy (v => v)) + Console.WriteLine ($" {lib}"); Console.WriteLine ($" Optimize: {user_optimize_flags} => {Application.Optimizations}"); Console.WriteLine ($" PartialStaticRegistrarLibrary: {PartialStaticRegistrarLibrary}"); Console.WriteLine ($" Platform: {Platform}"); diff --git a/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs b/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs index 373b7fea0b7c..c0d4274f25a1 100644 --- a/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs +++ b/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using Mono.Cecil; using Mono.Linker; @@ -124,48 +126,40 @@ void ProcessMethod (MethodDefinition method) } } +#if NET + // Create a list of all the dynamic libraries from Mono that we'll link with + // We add 4 different variations for each library: + // * with and without a "lib" prefix + // * with and without the ".dylib" extension + var app = LinkerConfiguration.GetInstance (Context).Application; + var dynamicMonoLibraries = app.MonoLibraries. + Where (v => v.EndsWith (".dylib", StringComparison.OrdinalIgnoreCase)). + Select (v => Path.GetFileNameWithoutExtension (v)). + Select (v => v.StartsWith ("lib", StringComparison.OrdinalIgnoreCase) ? v.Substring (3) : v).ToHashSet (); + dynamicMonoLibraries.UnionWith (dynamicMonoLibraries.Select (v => "lib" + v).ToArray ()); + dynamicMonoLibraries.UnionWith (dynamicMonoLibraries.Select (v => v + ".dylib").ToArray ()); + // If the P/Invoke points to any of those libraries, then we add it as a P/Invoke symbol. + if (dynamicMonoLibraries.Contains (pinfo.Module.Name)) + addPInvokeSymbol = true; +#endif + switch (pinfo.Module.Name) { case "__Internal": Driver.Log (4, "Adding native reference to {0} in {1} because it's referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name); DerivedLinkContext.RequiredSymbols.AddFunction (pinfo.EntryPoint).AddMember (method); break; - case "libSystem.Net.Security.Native": +#if !NET case "System.Net.Security.Native": - addPInvokeSymbol = true; - break; - - case "libSystem.Security.Cryptography.Native.Apple": case "System.Security.Cryptography.Native.Apple": -#if NET - // https://github.com/dotnet/runtime/issues/47533 - if (DerivedLinkContext.App.Platform != ApplePlatform.MacOSX && DerivedLinkContext.App.Platform != ApplePlatform.MacCatalyst) { - Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name); - break; - } -#endif - addPInvokeSymbol = true; - break; - - case "libSystem.Native": case "System.Native": -#if NET - // https://github.com/dotnet/runtime/issues/47533 - if (DerivedLinkContext.App.Platform != ApplePlatform.MacOSX && pinfo.EntryPoint == "SystemNative_ConfigureTerminalForChildProcess") { - Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name); - break; - } -#endif - addPInvokeSymbol = true; - break; - - case "libSystem.Globalization.Native": - case "System.Globalization.Native": addPInvokeSymbol = true; break; +#endif default: - Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name); + if (!addPInvokeSymbol) + Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name); break; }