Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[release/6.0-rc2] MonoAOTCompiler: detect when nothing has changed, and skip any precompiling #58979

Merged
merged 3 commits into from
Sep 24, 2021
Merged

[release/6.0-rc2] MonoAOTCompiler: detect when nothing has changed, and skip any precompiling #58979

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bbc570b
Refactor to allow fast-path
radical Sep 11, 2021
eecb51f
implement fast-path for MonoAOTCompiler when nothing has changed
radical Sep 11, 2021
c9c3e57
re-enable some tests that got disabled by mistake
radical Sep 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 102 additions & 44 deletions src/tasks/AotCompilerTask/MonoAOTCompiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,47 +357,77 @@ private bool ExecuteInternal()

_cache = new FileCache(CacheFilePath, Log);

//FIXME: check the nothing changed at all case
List<PrecompileArguments> argsList = new();
foreach (var assemblyItem in _assembliesToCompile)
argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths));

_totalNumAssemblies = _assembliesToCompile.Count;
int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
allowedParallelism = be9.RequestCores(allowedParallelism);

if (DisableParallelAot || allowedParallelism == 1)
if (CheckAllUpToDate(argsList))
{
foreach (var assemblyItem in _assembliesToCompile)
{
if (!PrecompileLibrarySerial(assemblyItem, monoPaths))
return !Log.HasLoggedErrors;
}
Log.LogMessage(MessageImportance.High, "Everything is up-to-date, nothing to precompile");

_fileWrites.AddRange(argsList.SelectMany(args => args.ProxyFiles).Select(pf => pf.TargetFile));
foreach (var args in argsList)
compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly);
}
else
{
ParallelLoopResult result = Parallel.ForEach(
_assembliesToCompile,
new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
(assemblyItem, state) => PrecompileLibraryParallel(assemblyItem, monoPaths, state));
int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
allowedParallelism = be9.RequestCores(allowedParallelism);

if (!result.IsCompleted)
if (DisableParallelAot || allowedParallelism == 1)
{
return false;
foreach (var args in argsList)
{
if (!PrecompileLibrarySerial(args))
return !Log.HasLoggedErrors;
}
}
else
{
ParallelLoopResult result = Parallel.ForEach(
argsList,
new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
(args, state) => PrecompileLibraryParallel(args, state));
lewing marked this conversation as resolved.
Show resolved Hide resolved

if (!result.IsCompleted)
{
return false;
}
}

int numUnchanged = _totalNumAssemblies - _numCompiled;
if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
}

int numUnchanged = _totalNumAssemblies - _numCompiled;
if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray();

if (_cache.Save(CacheFilePath!))
_fileWrites.Add(CacheFilePath!);

CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray();
FileWrites = _fileWrites.ToArray();

return !Log.HasLoggedErrors;
}

private bool CheckAllUpToDate(IList<PrecompileArguments> argsList)
{
foreach (var args in argsList)
{
// compare original assembly vs it's outputs.. all it's outputs!
string assemblyPath = args.AOTAssembly.GetMetadata("FullPath");
if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile)))
return false;
}

return true;

static bool IsNewerThanOutput(string inFile, string outFile)
=> !File.Exists(inFile) || !File.Exists(outFile) ||
(File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile));
}

private IList<ITaskItem> EnsureAndGetAssembliesInTheSameDir(ITaskItem[] originalAssemblies)
{
List<ITaskItem> filteredAssemblies = new();
Expand Down Expand Up @@ -454,7 +484,7 @@ static bool ShouldSkip(ITaskItem asmItem)
=> bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip;
}

private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths)
{
string assembly = assemblyItem.GetMetadata("FullPath");
string assemblyDir = Path.GetDirectoryName(assembly)!;
Expand All @@ -463,7 +493,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
var processArgs = new List<string>();
bool isDedup = assembly == DedupAssembly;
List<ProxyFile> proxyFiles = new(capacity: 5);
string msgPrefix = $"[{Path.GetFileName(assembly)}] ";

var a = assemblyItem.GetMetadata("AotArguments");
if (a != null)
Expand Down Expand Up @@ -674,16 +703,26 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
sw.WriteLine(responseFileContent);
}

string workingDir = assemblyDir;
return new PrecompileArguments(ResponseFilePath: responseFilePath,
EnvironmentVariables: envVariables,
WorkingDir: assemblyDir,
AOTAssembly: aotAssembly,
ProxyFiles: proxyFiles);
}

private bool PrecompileLibrary(PrecompileArguments args)
{
string assembly = args.AOTAssembly.GetMetadata("FullPath");
try
{
string msgPrefix = $"[{Path.GetFileName(assembly)}] ";

// run the AOT compiler
(int exitCode, string output) = Utils.TryRunProcess(Log,
CompilerBinaryPath,
$"--response=\"{responseFilePath}\"",
envVariables,
workingDir,
$"--response=\"{args.ResponseFilePath}\"",
args.EnvironmentVariables,
args.WorkingDir,
silent: true,
debugMessageImportance: MessageImportance.Low,
label: Path.GetFileName(assembly));
Expand All @@ -692,9 +731,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
// Log the command in a compact format which can be copy pasted
{
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (KeyValuePair<string, string> kvp in envVariables)
foreach (KeyValuePair<string, string> kvp in args.EnvironmentVariables)
envStr.Append($"{kvp.Key}={kvp.Value} ");
Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}");
Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {args.WorkingDir}: {envStr}{CompilerBinaryPath} {File.ReadAllText(args.ResponseFilePath)}");
}

Log.LogMessage(importance, output);
Expand All @@ -713,66 +752,66 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
}
finally
{
File.Delete(responseFilePath);
File.Delete(args.ResponseFilePath);
}

bool copied = false;
foreach (var proxyFile in proxyFiles)
foreach (var proxyFile in args.ProxyFiles)
{
copied |= proxyFile.CopyOutputFileIfChanged();
_fileWrites.Add(proxyFile.TargetFile);
}

if (copied)
{
string copiedFiles = string.Join(", ", proxyFiles.Select(tf => Path.GetFileName(tf.TargetFile)));
string copiedFiles = string.Join(", ", args.ProxyFiles.Select(tf => Path.GetFileName(tf.TargetFile)));
int count = Interlocked.Increment(ref _numCompiled);
Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}");
}

compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly);
compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly);
return true;
}

private bool PrecompileLibrarySerial(ITaskItem assemblyItem, string? monoPaths)
private bool PrecompileLibrarySerial(PrecompileArguments args)
{
try
{
if (PrecompileLibrary(assemblyItem, monoPaths))
if (PrecompileLibrary(args))
return true;
}
catch (LogAsErrorException laee)
{
Log.LogError($"Precompile failed for {assemblyItem}: {laee.Message}");
Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
}
catch (Exception ex)
{
if (Log.HasLoggedErrors)
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}");
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
else
Log.LogError($"Precompile failed for {assemblyItem}: {ex}");
Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
}

return false;
}

private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths, ParallelLoopState state)
private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state)
{
try
{
if (PrecompileLibrary(assemblyItem, monoPaths))
if (PrecompileLibrary(args))
return;
}
catch (LogAsErrorException laee)
{
Log.LogError($"Precompile failed for {assemblyItem}: {laee.Message}");
Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
}
catch (Exception ex)
{
if (Log.HasLoggedErrors)
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}");
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
else
Log.LogError($"Precompile failed for {assemblyItem}: {ex}");
Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
}

state.Break();
Expand Down Expand Up @@ -908,6 +947,24 @@ private static IList<ITaskItem> ConvertAssembliesDictToOrderedList(ConcurrentDic
}
return outItems;
}

internal class PrecompileArguments
{
public PrecompileArguments(string ResponseFilePath, IDictionary<string, string> EnvironmentVariables, string WorkingDir, ITaskItem AOTAssembly, IList<ProxyFile> ProxyFiles)
{
this.ResponseFilePath = ResponseFilePath;
this.EnvironmentVariables = EnvironmentVariables;
this.WorkingDir = WorkingDir;
this.AOTAssembly = AOTAssembly;
this.ProxyFiles = ProxyFiles;
}

public string ResponseFilePath { get; private set; }
public IDictionary<string, string> EnvironmentVariables { get; private set; }
public string WorkingDir { get; private set; }
public ITaskItem AOTAssembly { get; private set; }
public IList<ProxyFile> ProxyFiles { get; private set; }
}
}

internal class FileCache
Expand Down Expand Up @@ -1017,6 +1074,7 @@ public bool CopyOutputFileIfChanged()
File.Delete(TempFile);
}
}

}

public enum MonoAotMode
Expand Down