Merge pull request #3945 from devlead/feature/gh-3931

GH3931: Improve posix native library loading

.github/workflows/build.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-18.04, macos-latest]
+        os: [windows-latest, ubuntu-18.04, macos-latest, ubuntu-22.04]
     steps:
       - name: Get the sources
         uses: actions/checkout@v2

src/Cake.Core/Polyfill/AssemblyHelper.cs

@@ -5,8 +5,8 @@
 using System;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using Cake.Core.Diagnostics;
 using Cake.Core.IO;
-using Cake.Core.Reflection;
 
 namespace Cake.Core.Polyfill
 {
@@ -26,7 +26,7 @@ public static Assembly LoadAssembly(AssemblyName assemblyName)
             return Assembly.Load(assemblyName);
         }
 
-        public static Assembly LoadAssembly(ICakeEnvironment environment, IFileSystem fileSystem, FilePath path)
+        public static Assembly LoadAssembly(ICakeEnvironment environment, IFileSystem fileSystem, ICakeLog log, FilePath path)
         {
             if (path == null)
             {
@@ -49,9 +49,37 @@ public static Assembly LoadAssembly(ICakeEnvironment environment, IFileSystem fi
                     return Assembly.LoadFrom(path.FullPath);
                 }
 
-                if (environment.Platform.IsUnix())
+                if (_useLoadMacOsLibrary)
                 {
-                    LoadUnixLibrary(path.FullPath, RTLD_NOW);
+                    LoadUnixLibrary(LoadMacOSLibrary, path);
+                    LogNativeLoad(log, nameof(LoadMacOSLibrary), path);
+                }
+                else if (_useLoadUnixLibrary1)
+                {
+                    LoadUnixLibrary(LoadUnixLibrary1, path);
+                    LogNativeLoad(log, nameof(LoadUnixLibrary1), path);
+                }
+                else if (_useLoadUnixLibrary2)
+                {
+                    LoadUnixLibrary(LoadUnixLibrary2, path);
+                    LogNativeLoad(log, nameof(LoadUnixLibrary2), path);
+                }
+                else if (environment.Platform.IsUnix())
+                {
+                    if (environment.Platform.IsOSX() && TryLoadUnixLibrary(LoadMacOSLibrary, path, out _useLoadMacOsLibrary))
+                    {
+                        LogNativeLoad(log, nameof(LoadMacOSLibrary), path);
+                        return null;
+                    }
+
+                    if (TryLoadUnixLibrary(LoadUnixLibrary2, path, out _useLoadUnixLibrary2))
+                    {
+                        LogNativeLoad(log, nameof(LoadUnixLibrary2), path);
+                        return null;
+                    }
+
+                    TryLoadUnixLibrary(LoadUnixLibrary1, path, out _useLoadUnixLibrary1);
+                    LogNativeLoad(log, nameof(LoadUnixLibrary1), path);
                 }
                 else
                 {
@@ -60,21 +88,60 @@ public static Assembly LoadAssembly(ICakeEnvironment environment, IFileSystem fi
 
                 return null;
             }
-            catch (System.IO.FileLoadException)
+            catch (System.IO.FileLoadException ex)
             {
-                // TODO: LOG
+                log.Debug(Verbosity.Diagnostic,
+                    logAction => logAction("Caught error while loading {0}\r\n{1}",
+                        path.FullPath,
+                        ex));
                 return null;
             }
         }
 
+        private static void LogNativeLoad(ICakeLog log, string loadUnixLibrary, FilePath path)
+        {
+            log.Debug(Verbosity.Diagnostic,
+                logAction => logAction("Native {0}: {1}",
+                    loadUnixLibrary,
+                    path.FullPath));
+        }
+
 #pragma warning disable SA1310 // Field names should not contain underscore
         private const int RTLD_NOW = 0x002;
 #pragma warning restore SA1310 // Field names should not contain underscore
+#pragma warning disable SA1303 // Field names should start with uppercase
+        private const string dlopen = nameof(dlopen);
+#pragma warning restore SA1303 // Field names should start with uppercase
+
+        private static bool _useLoadUnixLibrary1;
+        private static bool _useLoadUnixLibrary2;
+        private static bool _useLoadMacOsLibrary;
 
-        [DllImport("libdl", EntryPoint = "dlopen")]
-        private static extern IntPtr LoadUnixLibrary(string path, int flags);
+        [DllImport("libdl.so", EntryPoint = dlopen)]
+        private static extern IntPtr LoadUnixLibrary1(string path, int flags);
+
+        [DllImport("libdl.so.2", EntryPoint = dlopen)]
+        private static extern IntPtr LoadUnixLibrary2(string path, int flags);
+
+        [DllImport("/usr/lib/libSystem.dylib", EntryPoint = dlopen)]
+        private static extern IntPtr LoadMacOSLibrary(string path, int flags);
 
         [DllImport("kernel32", EntryPoint = "LoadLibrary")]
         private static extern IntPtr LoadWindowsLibrary(string path);
+
+        private static void LoadUnixLibrary(Func<string, int, IntPtr> dlOpen, FilePath path) => dlOpen(path.FullPath, RTLD_NOW);
+
+        private static bool TryLoadUnixLibrary(Func<string, int, IntPtr> dlOpen, FilePath path, out bool result)
+        {
+            try
+            {
+                LoadUnixLibrary(dlOpen, path);
+                return result = true;
+            }
+            catch (DllNotFoundException)
+            {
+                return result = false;
+            }
+        }
     }
 }

src/Cake.Core/Reflection/AssemblyLoader.cs

@@ -4,6 +4,7 @@
 
 using System.Reflection;
 using Cake.Core.Configuration;
+using Cake.Core.Diagnostics;
 using Cake.Core.IO;
 using Cake.Core.Polyfill;
 
@@ -17,18 +18,21 @@ public sealed class AssemblyLoader : IAssemblyLoader
         private readonly ICakeEnvironment _environment;
         private readonly IFileSystem _fileSystem;
         private readonly IAssemblyVerifier _verifier;
+        private readonly ICakeLog _log;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="AssemblyLoader"/> class.
         /// </summary>
         /// <param name="environment">The environment.</param>
         /// <param name="fileSystem">The file system.</param>
         /// <param name="verifier">The assembly verifier.</param>
-        public AssemblyLoader(ICakeEnvironment environment, IFileSystem fileSystem, IAssemblyVerifier verifier)
+        /// <param name="log">The cake log.</param>
+        public AssemblyLoader(ICakeEnvironment environment, IFileSystem fileSystem, IAssemblyVerifier verifier, ICakeLog log)
         {
             _environment = environment;
             _fileSystem = fileSystem;
             _verifier = verifier;
+            _log = log;
         }
 
         /// <inheritdoc/>
@@ -40,7 +44,7 @@ public Assembly Load(AssemblyName assemblyName)
         /// <inheritdoc/>
         public Assembly Load(FilePath path, bool verify)
         {
-            var assembly = AssemblyHelper.LoadAssembly(_environment, _fileSystem, path);
+            var assembly = AssemblyHelper.LoadAssembly(_environment, _fileSystem, _log, path);
             if (verify && assembly != null)
             {
                 _verifier.Verify(assembly);

src/Cake.Core/Scripting/ScriptConventions.cs

@@ -6,7 +6,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
-using Cake.Core.Configuration;
 using Cake.Core.IO;
 using Cake.Core.Reflection;
 

src/Cake.NuGet/Cake.NuGet.csproj

@@ -25,7 +25,7 @@
     <PackageReference Include="NuGet.Resolver" Version="6.3.0" />
     <PackageReference Include="NuGet.Common" Version="6.3.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" Version="3.1.6" />
+    <PackageReference Condition="'$(TargetFramework)' == 'netcoreapp3.1'" Include="Microsoft.DotNet.PlatformAbstractions" Version="3.1.6" />
   </ItemGroup>
 
   <ItemGroup>

src/Cake.NuGet/NuGetContentResolver.cs

@@ -75,7 +75,13 @@ private IReadOnlyCollection<IFile> GetAddinAssemblies(DirectoryPath path, Packag
             var tfm = NuGetFramework.Parse(_environment.Runtime.BuiltFramework.FullName, DefaultFrameworkNameProvider.Instance);
 
             // Get current runtime identifier.
-            string rid = _environment.Runtime.IsCoreClr ? Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() : null;
+            var rid = _environment.Runtime.IsCoreClr
+#if NETCOREAPP3_1
+                ? Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier()
+#else
+                ? System.Runtime.InteropServices.RuntimeInformation.RuntimeIdentifier
+#endif
+                    : null;
 
             // Get all candidate files.
             var pathComparer = PathComparer.Default;

src/Cake/Infrastructure/Composition/ModuleSearcher.cs

@@ -75,7 +75,7 @@ private Type LoadModule(FilePath path, ICakeConfiguration configuration)
                     return null;
                 }
 
-                var loader = new AssemblyLoader(_environment, _fileSystem, new AssemblyVerifier(configuration, _log));
+                var loader = new AssemblyLoader(_environment, _fileSystem, new AssemblyVerifier(configuration, _log), _log);
                 var assembly = loader.Load(path, true);
 
                 var attribute = assembly.GetCustomAttributes<CakeModuleAttribute>().FirstOrDefault();

tests/integration/Cake.Core/Scripting/AddinDirective.cake

@@ -55,8 +55,35 @@ Task("Cake.Core.Scripting.AddinDirective.CallDuplicatedMethod")
     var result = context.EnvironmentVariable("CAKE_DOES_ROCK", true);
 });
 
+Task("Cake.Core.Scripting.AddinDirective.LoadNativeAssemblies")
+    .Does(() =>
+{
+    FilePath cakeCore = typeof(ICakeContext).GetTypeInfo().Assembly.Location;
+    FilePath cake = cakeCore.GetDirectory().CombineWithFilePath("Cake.dll");
+    var script = @"#addin nuget:?package=Cake.Git&version=2.0.0
+
+var repoRoot = GitFindRootFromPath(Context.EnvironmentVariable(""CAKE_TEST_DIR""));
+
+var hasUncommittedChanges = GitHasUncommitedChanges(repoRoot);";
+
+    CakeExecuteExpression(script,
+        new CakeSettings {
+            EnvironmentVariables = new Dictionary<string, string>{
+                {"CAKE_PATHS_ADDINS", $"{Paths.Temp}/native/tools/Addins"},
+                {"CAKE_PATHS_TOOLS", $"{Paths.Temp}/native/tools"},
+                {"CAKE_PATHS_MODULES", $"{Paths.Temp}/native/tools/Modules"},
+                {"NUGET_PACKAGES", $"{Paths.Temp}/nuget/Packages"},
+                {"NUGET_HTTP_CACHE_PATH ", $"{Paths.Temp}/nuget/Cache"},
+                {"CAKE_TEST_DIR", Context.Environment.WorkingDirectory.FullPath}
+            },
+            ToolPath = cake,
+            Verbosity = Context.Log.Verbosity
+        });
+});
+
 //////////////////////////////////////////////////////////////////////////////
 
 Task("Cake.Core.Scripting.AddinDirective")
     .IsDependentOn("Cake.Core.Scripting.AddinDirective.LoadTargetedAddin")
-    .IsDependentOn("Cake.Core.Scripting.AddinDirective.CallDuplicatedMethod");
+    .IsDependentOn("Cake.Core.Scripting.AddinDirective.CallDuplicatedMethod")
+    .IsDependentOn("Cake.Core.Scripting.AddinDirective.LoadNativeAssemblies");