Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] Use a Response file for AOT
Browse files Browse the repository at this point in the history 
Fixes https://devdiv.visualstudio.com/DevDiv/_workitems/edit/609244

Windows has a limit on the length of command line arguments.
As a result AOT can fail on windows if the project paths
are too long.

mono 2018-06 introduced a `--response=FILE` which allows
us to provide all the options in a file rather than on the
command line directly.
  • Loading branch information
@dellis1972
dellis1972 committed Aug 12, 2019
1 parent 9efb5f9 commit bcab81d
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 70 deletions.
135 changes: 70 additions & 65 deletions src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
Expand Down Expand Up @@ -173,19 +174,11 @@ static string GetNdkToolchainLibraryDir (string binDir, AndroidTargetArch arch)
return GetNdkToolchainLibraryDir (binDir, NdkUtil.GetArchDirName (arch));
}

static string GetShortPath (string path)
{
if (Environment.OSVersion.Platform != PlatformID.Win32NT)
return QuoteFileName (path);
var shortPath = KernelEx.GetShortPathName (Path.GetDirectoryName (path));
return Path.Combine (shortPath, Path.GetFileName (path));
}

static string QuoteFileName(string fileName)
{
var builder = new CommandLineBuilder();
builder.AppendFileNameIfNotNull(fileName);
return builder.ToString();
return builder.ToString().Replace (@"\", @"\\").Normalize (NormalizationForm.FormC);
}

static bool ValidateAotConfiguration (TaskLoggingHelper log, AndroidTargetArch arch, bool enableLLVM)
Expand Down Expand Up @@ -379,7 +372,6 @@ IEnumerable<Config> GetAotConfigs ()
} catch (InvalidOperationException ex) {
Diagnostic.Error (5101, ex.Message);
}

string toolchainLibDir;
if (NdkUtil.UsingClangNDK)
toolchainLibDir = GetNdkToolchainLibraryDir (toolchainPath, arch);
Expand All @@ -388,19 +380,19 @@ IEnumerable<Config> GetAotConfigs ()

var libs = new List<string>();
if (NdkUtil.UsingClangNDK) {
libs.Add ($"-L{GetShortPath (toolchainLibDir)}");
libs.Add ($"-L{GetShortPath (androidLibPath)}");
libs.Add ($"-L{QuoteFileName (toolchainLibDir)}");
libs.Add ($"-L{QuoteFileName (androidLibPath)}");

if (arch == AndroidTargetArch.Arm) {
// Needed for -lunwind to work
string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", NdkUtil.GetArchDirName (arch));
libs.Add ($"-L{GetShortPath (compilerLibDir)}");
libs.Add ($"-L{QuoteFileName (compilerLibDir)}");
}
}

libs.Add (GetShortPath (Path.Combine (toolchainLibDir, "libgcc.a")));
libs.Add (GetShortPath (Path.Combine (androidLibPath, "libc.so")));
libs.Add (GetShortPath (Path.Combine (androidLibPath, "libm.so")));
libs.Add (QuoteFileName (Path.Combine (toolchainLibDir, "libgcc.a")));
libs.Add (QuoteFileName (Path.Combine (androidLibPath, "libc.so")));
libs.Add (QuoteFileName (Path.Combine (androidLibPath, "libm.so")));

ldFlags = string.Join(";", libs);
}
Expand All @@ -422,23 +414,23 @@ IEnumerable<Config> GetAotConfigs ()
aotOptions.Add ("profile-only");
foreach (var p in Profiles) {
var fp = Path.GetFullPath (p.ItemSpec);
aotOptions.Add ($"profile={GetShortPath (fp)}");
aotOptions.Add ($"profile={QuoteFileName (fp)}");
}
}
if (!string.IsNullOrEmpty (AotAdditionalArguments))
aotOptions.Add (AotAdditionalArguments);
if (sequencePointsMode == SequencePointsMode.Offline)
aotOptions.Add ("msym-dir=" + GetShortPath (outdir));
aotOptions.Add ("msym-dir=" + QuoteFileName (outdir));
if (AotMode != AotMode.Normal)
aotOptions.Add (AotMode.ToString ().ToLowerInvariant ());

aotOptions.Add ("outfile=" + GetShortPath (outputFile));
aotOptions.Add ("outfile=" + QuoteFileName (outputFile));
aotOptions.Add ("asmwriter");
aotOptions.Add ("mtriple=" + mtriple);
aotOptions.Add ("tool-prefix=" + GetShortPath (toolPrefix));
aotOptions.Add ("tool-prefix=" + QuoteFileName (toolPrefix));
aotOptions.Add ("ld-flags=" + ldFlags);
aotOptions.Add ("llvm-path=" + GetShortPath (sdkBinDirectory));
aotOptions.Add ("temp-path=" + GetShortPath (tempDir));
aotOptions.Add ("llvm-path=" + QuoteFileName (sdkBinDirectory));
aotOptions.Add ("temp-path=" + QuoteFileName (tempDir));

string aotOptionsStr = (EnableLLVM ? "--llvm " : "") + "--aot=" + string.Join (",", aotOptions);

Expand Down Expand Up @@ -471,50 +463,63 @@ bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOption
{
var stdout_completed = new ManualResetEvent (false);
var stderr_completed = new ManualResetEvent (false);
var psi = new ProcessStartInfo () {
FileName = aotCompiler,
Arguments = aotOptions + " " + assembly,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow=true,
WindowStyle=ProcessWindowStyle.Hidden,
WorkingDirectory = WorkingDirectory,
};

// we do not want options to be provided out of band to the cross compilers
psi.EnvironmentVariables ["MONO_ENV_OPTIONS"] = String.Empty;
// the C code cannot parse all the license details, including the activation code that tell us which license level is allowed
// so we provide this out-of-band to the cross-compilers - this can be extended to communicate a few others bits as well
psi.EnvironmentVariables ["MONO_PATH"] = assembliesPath;

LogDebugMessage ("[AOT] MONO_PATH=\"{0}\" MONO_ENV_OPTIONS=\"{1}\" {2} {3}",
psi.EnvironmentVariables ["MONO_PATH"], psi.EnvironmentVariables ["MONO_ENV_OPTIONS"], psi.FileName, psi.Arguments);

using (var proc = new Process ()) {
proc.OutputDataReceived += (s, e) => {
if (e.Data != null)
OnAotOutputData (s, e);
else
stdout_completed.Set ();
};
proc.ErrorDataReceived += (s, e) => {
if (e.Data != null)
OnAotErrorData (s, e);
else
stderr_completed.Set ();
var responseFile = Path.GetTempFileName ();
try {
using (var sw = new StreamWriter (path: responseFile, append: false,
encoding: new UTF8Encoding (encoderShouldEmitUTF8Identifier: false))) {
sw.WriteLine (aotOptions + " " + assembly);
}

var psi = new ProcessStartInfo () {
FileName = aotCompiler,
Arguments = $"--response=\"{responseFile}\"",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
WorkingDirectory = WorkingDirectory,
};
proc.StartInfo = psi;
proc.Start ();
proc.BeginOutputReadLine ();
proc.BeginErrorReadLine ();
CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } });
proc.WaitForExit ();
if (psi.RedirectStandardError)
stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
if (psi.RedirectStandardOutput)
stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
return proc.ExitCode == 0;

psi.EnvironmentVariables ["MONO_LOG_LEVEL"] = "debug";
// we do not want options to be provided out of band to the cross compilers
psi.EnvironmentVariables ["MONO_ENV_OPTIONS"] = String.Empty;
// the C code cannot parse all the license details, including the activation code that tell us which license level is allowed
// so we provide this out-of-band to the cross-compilers - this can be extended to communicate a few others bits as well
psi.EnvironmentVariables ["MONO_PATH"] = assembliesPath.Normalize (NormalizationForm.FormC);

LogDebugMessage ($"[AOT] {aotOptions} {assembly}");
LogDebugMessage ("[AOT] MONO_PATH=\"{0}\" MONO_ENV_OPTIONS=\"{1}\" {2} {3}",
psi.EnvironmentVariables ["MONO_PATH"], psi.EnvironmentVariables ["MONO_ENV_OPTIONS"], psi.FileName, psi.Arguments);

using (var proc = new Process ()) {
proc.OutputDataReceived += (s, e) => {
if (e.Data != null)
OnAotOutputData (s, e);
else
stdout_completed.Set ();
};
proc.ErrorDataReceived += (s, e) => {
if (e.Data != null)
OnAotErrorData (s, e);
else
stderr_completed.Set ();
};
proc.StartInfo = psi;
proc.Start ();
proc.BeginOutputReadLine ();
proc.BeginErrorReadLine ();
CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } });
proc.WaitForExit ();
if (psi.RedirectStandardError)
stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
if (psi.RedirectStandardOutput)
stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
return proc.ExitCode == 0;
}
} finally {
if (File.Exists (responseFile))
File.Delete (responseFile);
}
}

Expand Down
10 changes: 5 additions & 5 deletions src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,9 +732,9 @@ public void BuildMkBundleApplicationReleaseAllAbi ()
[Test]
[TestCaseSource (nameof (AotChecks))]
[Category ("Minor")]
public void BuildAotApplication (string supportedAbis, bool enableLLVM, bool expectedResult)
public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult)
{
var path = Path.Combine ("temp", string.Format ("BuildAotApplication_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult));
var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult));
var proj = new XamarinAndroidApplicationProject () {
IsRelease = true,
BundleAssemblies = false,
Expand Down Expand Up @@ -768,7 +768,7 @@ public void BuildAotApplication (string supportedAbis, bool enableLLVM, bool exp
// LLVM passes a direct path to libc.so, and we need to use the libc.so
// which corresponds to the *minimum* SDK version specified in AndroidManifest.xml
// Since we overrode minSdkVersion=16, that means we should use libc.so from android-16.
var rightLibc = new Regex (@"^\s*\[AOT\].*cross-.*--llvm.*,ld-flags=.*android-16.arch-.*.usr.lib.libc\.so", RegexOptions.Multiline);
var rightLibc = new Regex (@"\s*\[aot-compiler stdout].*android-16.arch-.*.usr.lib.libc\.so", RegexOptions.Multiline);
var m = rightLibc.Match (string.Join ("\n",b.LastBuildOutput));
Assert.IsTrue (m.Success, "AOT+LLVM should use libc.so from minSdkVersion!");
}
Expand Down Expand Up @@ -803,9 +803,9 @@ public void BuildAotApplication (string supportedAbis, bool enableLLVM, bool exp
[Test]
[TestCaseSource (nameof (AotChecks))]
[Category ("Minor")]
public void BuildAotApplicationAndBundle (string supportedAbis, bool enableLLVM, bool expectedResult)
public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult)
{
var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult));
var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult));
var proj = new XamarinAndroidApplicationProject () {
IsRelease = true,
BundleAssemblies = true,
Expand Down

0 comments on commit bcab81d

Please sign in to comment.