From cce1bfa3d98d478b89a599701de9dbd33c26b287 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Tue, 23 Feb 2021 15:01:17 -0800 Subject: [PATCH 1/9] Rebuild as many projects as possible --- eng/test-rebuild.ps1 | 39 +++++++++++++++++-- src/Tools/BuildValidator/BuildConstructor.cs | 12 ++++-- src/Tools/BuildValidator/CompilationDiff.cs | 9 +++-- .../CompilationOptionsReader.cs | 20 +++++++++- src/Tools/BuildValidator/Program.cs | 30 +++++++++++--- 5 files changed, 92 insertions(+), 18 deletions(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 3bb1aac9eb35e..86ef63485da2c 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -36,13 +36,44 @@ try { $dotnetInstallDir = (InitializeDotNetCli -install:$true) $rebuildArgs = ("--verbose" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/$configuration`"" + - " --assembliesPath $ArtifactsDir/obj/csc/$configuration/netcoreapp3.1" + + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/`"" + + " --assembliesPath `"$ArtifactsDir/obj/BuildValidator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/CodeStyleConfigFileGenerator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/csc/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/CSharpResultProvider.NetFX20/`"" + + " --assembliesPath `"$ArtifactsDir/obj/CSharpSyntaxGenerator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/csi/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle.Fixes/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Compiler.Test.Resources/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle.Fixes/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Features/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Workspaces/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.EditorFeatures.Text/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExpressionCompiler/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Debugger/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Razor/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Razor.ServiceHub/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Workspaces/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ResultProvider/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting.TestUtilities/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.TestSourceGenerator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Workspaces/`"" + + " --assembliesPath `"$ArtifactsDir/obj/PrepareTests/`"" + + " --assembliesPath `"$ArtifactsDir/obj/RunTests/`"" + + " --assembliesPath `"$ArtifactsDir/obj/vbc/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/VBCSCompiler/$configuration/netcoreapp3.1`"" + + " --debugPath `"$ArtifactsDir/BuildValidator`"" + " --sourcePath `"$RepoRoot`"" + " --referencesPath `"$ArtifactsDir/bin`"" + - " --referencesPath `"$dotnetInstallDir/packs/Microsoft.AspNetCore.App.Ref`"" + - " --referencesPath `"$dotnetInstallDir/packs/Microsoft.NETCore.App.Ref`"") + " --referencesPath `"$dotnetInstallDir/packs`"") Exec-Console "$ArtifactsDir/bin/BuildValidator/$configuration/net472/BuildValidator.exe" $rebuildArgs exit 0 diff --git a/src/Tools/BuildValidator/BuildConstructor.cs b/src/Tools/BuildValidator/BuildConstructor.cs index bfac2b9d93830..da5726be8ae27 100644 --- a/src/Tools/BuildValidator/BuildConstructor.cs +++ b/src/Tools/BuildValidator/BuildConstructor.cs @@ -38,12 +38,16 @@ public BuildConstructor(LocalReferenceResolver referenceResolver, LocalSourceRes _logger = logger; } - public Compilation CreateCompilation(CompilationOptionsReader compilationOptionsReader, string fileName) + public Compilation? CreateCompilation(CompilationOptionsReader compilationOptionsReader, string fileName) { - var pdbCompilationOptions = compilationOptionsReader.GetMetadataCompilationOptions(); - if (pdbCompilationOptions.Length == 0) + // We try to handle assemblies missing compilation options gracefully by skipping them. + // However, if an assembly has some bad combination of data, for example if it contains + // compilation options but not metadata references, then we throw an exception. + if (!compilationOptionsReader.TryGetMetadataCompilationOptions(out var pdbCompilationOptions) + || pdbCompilationOptions.Length == 0) { - throw new InvalidDataException("Did not find compilation options in pdb"); + _logger.LogInformation($"{fileName} did not contain compilation options in its PDB"); + return null; } var metadataReferenceInfos = compilationOptionsReader.GetMetadataReferences(); diff --git a/src/Tools/BuildValidator/CompilationDiff.cs b/src/Tools/BuildValidator/CompilationDiff.cs index f0729cb069dc8..87c80739e286f 100644 --- a/src/Tools/BuildValidator/CompilationDiff.cs +++ b/src/Tools/BuildValidator/CompilationDiff.cs @@ -30,9 +30,7 @@ internal class CompilationDiff public string OriginalPath { get; } public ImmutableArray Diagnostics { get; } - private CompilationDiff( - string originalPath, - bool? areEqual) + private CompilationDiff(string originalPath, bool? areEqual) { AreEqual = areEqual; OriginalPath = originalPath; @@ -44,6 +42,11 @@ private CompilationDiff(ImmutableArray diagnostics, string originalP OriginalPath = originalPath; } + public static CompilationDiff CreatePlaceholder(FileInfo originalBinaryPath) + { + return new CompilationDiff(originalBinaryPath.FullName, areEqual: null); + } + public static unsafe CompilationDiff Create( FileInfo originalBinaryPath, CompilationOptionsReader optionsReader, diff --git a/src/Tools/BuildValidator/CompilationOptionsReader.cs b/src/Tools/BuildValidator/CompilationOptionsReader.cs index 0cdf44a87f7f1..61d9ec0535406 100644 --- a/src/Tools/BuildValidator/CompilationOptionsReader.cs +++ b/src/Tools/BuildValidator/CompilationOptionsReader.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Compression; using System.Linq; @@ -77,14 +78,29 @@ public CompilationOptionsReader(ILogger logger, MetadataReader pdbReader, PERead PeReader = peReader; } + public bool TryGetMetadataCompilationOptionsBlobReader(out BlobReader reader) + { + return TryGetCustomDebugInformationBlobReader(CompilationOptionsGuid, out reader); + } + public BlobReader GetMetadataCompilationOptionsBlobReader() { - if (!TryGetCustomDebugInformationBlobReader(CompilationOptionsGuid, out var optionsBlob)) + if (!TryGetMetadataCompilationOptionsBlobReader(out var reader)) { throw new InvalidOperationException(); } + return reader; + } + + public bool TryGetMetadataCompilationOptions([NotNullWhen(true)] out MetadataCompilationOptions? options) + { + if (_metadataCompilationOptions is null && TryGetMetadataCompilationOptionsBlobReader(out var optionsBlob)) + { + _metadataCompilationOptions = new MetadataCompilationOptions(ParseCompilationOptions(optionsBlob)); + } - return optionsBlob; + options = _metadataCompilationOptions; + return options != null; } public MetadataCompilationOptions GetMetadataCompilationOptions() diff --git a/src/Tools/BuildValidator/Program.cs b/src/Tools/BuildValidator/Program.cs index 734921b08fc0a..33ead1ddf3790 100644 --- a/src/Tools/BuildValidator/Program.cs +++ b/src/Tools/BuildValidator/Program.cs @@ -104,13 +104,20 @@ static int HandleCommand(string[] assembliesPath, string sourcePath, string[]? r try { + var artifactsDirs = options.AssembliesPaths.Select(path => new DirectoryInfo(path)); + using (logger.BeginScope("Rebuild Search Paths")) + { + foreach (var artifactsDir in artifactsDirs) + { + logger.LogInformation($@"""{artifactsDir.FullName}"""); + } + } + var sourceResolver = new LocalSourceResolver(options, loggerFactory); var referenceResolver = new LocalReferenceResolver(options, loggerFactory); var buildConstructor = new BuildConstructor(referenceResolver, sourceResolver, logger); - var artifactsDirs = options.AssembliesPaths.Select(path => new DirectoryInfo(path)); - var filesToValidate = artifactsDirs.SelectMany(dir => dir.EnumerateFiles("*.exe", SearchOption.AllDirectories) .Concat(dir.EnumerateFiles("*.dll", SearchOption.AllDirectories))) @@ -155,6 +162,16 @@ private static bool ValidateFiles(IEnumerable originalBinaries, BuildC } } + using (logger.BeginScope("Rebuilds with configuration issues")) + { + foreach (var diff in assembliesCompiled.Where(a => a.AreEqual is null && a.Diagnostics.IsDefaultOrEmpty)) + { + logger.LogError($"{diff.OriginalPath} was missing required metadata for rebuilding. Was it built with a recent enough compiler with the required settings?"); + // dependencies which don't have the required metadata have a way of sneaking into the obj folder. + // for now, let's not let presence of these assemblies cause the rebuild to fail. + } + } + using (logger.BeginScope("Rebuilds with output differences")) { foreach (var diff in assembliesCompiled.Where(a => a.AreEqual == false)) @@ -164,10 +181,9 @@ private static bool ValidateFiles(IEnumerable originalBinaries, BuildC success = false; } } - using (logger.BeginScope("Rebuilds with compilation errors")) { - foreach (var diff in assembliesCompiled.Where(a => a.AreEqual == null)) + foreach (var diff in assembliesCompiled.Where(a => !a.Diagnostics.IsDefaultOrEmpty)) { logger.LogError($"{diff.OriginalPath} had {diff.Diagnostics.Length} diagnostics."); success = false; @@ -202,7 +218,7 @@ private static bool ValidateFiles(IEnumerable originalBinaries, BuildC if (!pdbOpened || pdbReaderProvider is null) { logger.LogError($"Could not find pdb for {originalBinary.FullName}"); - return null; + return CompilationDiff.CreatePlaceholder(originalBinary); } using var _ = logger.BeginScope($"Verifying {originalBinary.FullName} with pdb {pdbPath ?? "[embedded]"}"); @@ -213,6 +229,10 @@ private static bool ValidateFiles(IEnumerable originalBinaries, BuildC var compilation = buildConstructor.CreateCompilation( optionsReader, originalBinary.Name); + if (compilation is null) + { + return CompilationDiff.CreatePlaceholder(originalBinary); + } var compilationDiff = CompilationDiff.Create(originalBinary, optionsReader, compilation, getDebugEntryPoint(), logger, options); return compilationDiff; From 99c967c6cd82e41a00e4ad7af9a9b171182be374 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Tue, 23 Feb 2021 15:05:57 -0800 Subject: [PATCH 2/9] Rebuild with release bootstrap compiler --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 13b78bf4cffba..3d1e6a024468e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -227,7 +227,7 @@ jobs: filePath: eng/build.ps1 arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog - - powershell: .\eng\test-rebuild.ps1 -ci + - powershell: .\eng\test-rebuild.ps1 -ci -configuration Release displayName: Run BuildValidator - task: PublishBuildArtifacts@1 From 32be0402855d8e7485e4938ceacb5617c96e7ade Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Tue, 23 Feb 2021 15:38:44 -0800 Subject: [PATCH 3/9] don't rebuild net472 yet --- eng/test-rebuild.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 86ef63485da2c..effd14197fe74 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -36,14 +36,16 @@ try { $dotnetInstallDir = (InitializeDotNetCli -install:$true) $rebuildArgs = ("--verbose" + - " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/`"" + - " --assembliesPath `"$ArtifactsDir/obj/BuildValidator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/net5.0`"" + + " --assembliesPath `"$ArtifactsDir/obj/BuildValidator/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/CodeStyleConfigFileGenerator/`"" + " --assembliesPath `"$ArtifactsDir/obj/csc/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/CSharpResultProvider.NetFX20/`"" + " --assembliesPath `"$ArtifactsDir/obj/CSharpSyntaxGenerator/`"" + " --assembliesPath `"$ArtifactsDir/obj/csi/$configuration/netcoreapp3.1`"" + - " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/`"" + + " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/$configuration/net5.0`"" + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/`"" + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle/`"" + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle.Fixes/`"" + From da70de15881382d06006db488f642cd443e21100 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Tue, 23 Feb 2021 15:39:52 -0800 Subject: [PATCH 4/9] cleanup --- eng/test-rebuild.ps1 | 52 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index effd14197fe74..8eaf09a4f16b2 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -39,36 +39,36 @@ try { " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/net5.0`"" + " --assembliesPath `"$ArtifactsDir/obj/BuildValidator/$configuration/netcoreapp3.1`"" + - " --assembliesPath `"$ArtifactsDir/obj/CodeStyleConfigFileGenerator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/CodeStyleConfigFileGenerator`"" + " --assembliesPath `"$ArtifactsDir/obj/csc/$configuration/netcoreapp3.1`"" + - " --assembliesPath `"$ArtifactsDir/obj/CSharpResultProvider.NetFX20/`"" + - " --assembliesPath `"$ArtifactsDir/obj/CSharpSyntaxGenerator/`"" + + " --assembliesPath `"$ArtifactsDir/obj/CSharpResultProvider.NetFX20`"" + + " --assembliesPath `"$ArtifactsDir/obj/CSharpSyntaxGenerator`"" + " --assembliesPath `"$ArtifactsDir/obj/csi/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/IdeCoreBenchmarks/$configuration/net5.0`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle.Fixes/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Compiler.Test.Resources/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle.Fixes/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Features/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Workspaces/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.EditorFeatures.Text/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExpressionCompiler/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Debugger/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Razor/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Razor.ServiceHub/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Workspaces/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ResultProvider/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting.TestUtilities/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.TestSourceGenerator/`"" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Workspaces/`"" + - " --assembliesPath `"$ArtifactsDir/obj/PrepareTests/`"" + - " --assembliesPath `"$ArtifactsDir/obj/RunTests/`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CodeStyle.Fixes`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Compiler.Test.Resources`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.CodeStyle.Fixes`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Features`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.CSharp.Workspaces`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.EditorFeatures.Text`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExpressionCompiler`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Debugger`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ExternalAccess.Razor`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Razor.ServiceHub`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Remote.Workspaces`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.ResultProvider`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Scripting.TestUtilities`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.TestSourceGenerator`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis.Workspaces`"" + + " --assembliesPath `"$ArtifactsDir/obj/PrepareTests`"" + + " --assembliesPath `"$ArtifactsDir/obj/RunTests`"" + " --assembliesPath `"$ArtifactsDir/obj/vbc/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/VBCSCompiler/$configuration/netcoreapp3.1`"" + From cab3f6f27ea3f5deb2b8a98c988826bdc121128f Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Wed, 24 Feb 2021 11:20:30 -0800 Subject: [PATCH 5/9] Temporarily move back to vmImage --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d1e6a024468e..cac2be8fa8dc4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,8 +215,7 @@ jobs: - job: Correctness_Rebuild pool: - name: NetCorePublic-Pool - queue: BuildPool.Windows.10.Amd64.Open + vmImage: 'windows-2019' timeoutInMinutes: 90 steps: - template: eng/pipelines/checkout-windows-task.yml From e5fd46e00675fd1ea42aaa2d68a342014748d5fa Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Wed, 24 Feb 2021 12:15:06 -0800 Subject: [PATCH 6/9] remove failing --- eng/test-rebuild.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 8eaf09a4f16b2..e7faac8c8ebc2 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -38,7 +38,6 @@ try { $rebuildArgs = ("--verbose" + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/AnalyzerRunner/$configuration/net5.0`"" + - " --assembliesPath `"$ArtifactsDir/obj/BuildValidator/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/CodeStyleConfigFileGenerator`"" + " --assembliesPath `"$ArtifactsDir/obj/csc/$configuration/netcoreapp3.1`"" + " --assembliesPath `"$ArtifactsDir/obj/CSharpResultProvider.NetFX20`"" + From 81c20b70aa35da70d8529382acf1ca5a8fb7f6ed Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Wed, 24 Feb 2021 12:25:16 -0800 Subject: [PATCH 7/9] embed ildasm --- eng/Versions.props | 15 +++++-- .../{ILTools.targets => ILAsm.targets} | 0 eng/targets/ILDAsm.targets | 31 ++++++++++++++ ....CodeAnalysis.CSharp.Emit.UnitTests.csproj | 2 +- ...nalysis.CSharp.IOperation.UnitTests.csproj | 2 +- ...eAnalysis.CSharp.Semantic.UnitTests.csproj | 2 +- ...odeAnalysis.CSharp.Symbol.UnitTests.csproj | 2 +- ...CodeAnalysis.CSharp.WinRT.UnitTests.csproj | 2 +- ...Analysis.VisualBasic.Emit.UnitTests.vbproj | 2 +- ...ysis.VisualBasic.Semantic.UnitTests.vbproj | 2 +- ...alysis.VisualBasic.Symbol.UnitTests.vbproj | 2 +- ...sis.CSharp.EditorFeatures.UnitTests.csproj | 2 +- ...CSharp.ExpressionCompiler.UnitTests.csproj | 2 +- ...sis.CSharp.ResultProvider.UnitTests.csproj | 2 +- ...Analysis.FunctionResolver.UnitTests.csproj | 2 +- ...lBasic.ExpressionCompiler.UnitTests.vbproj | 2 +- ...isualBasic.ResultProvider.UnitTests.vbproj | 2 +- .../BuildValidator/BuildValidator.csproj | 1 + src/Tools/BuildValidator/CompilationDiff.cs | 5 +-- src/Tools/BuildValidator/IldasmUtilities.cs | 42 +++++++++++++++++++ 20 files changed, 101 insertions(+), 21 deletions(-) rename eng/targets/{ILTools.targets => ILAsm.targets} (100%) create mode 100644 eng/targets/ILDAsm.targets create mode 100644 src/Tools/BuildValidator/IldasmUtilities.cs diff --git a/eng/Versions.props b/eng/Versions.props index 4f0a00a774765..faafc40fe5f80 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -32,7 +32,8 @@ 1.0.1-beta1.20623.3 3.8.0 16.8.181 - 5.0.0-alpha1.19409.1 + 5.0.0-alpha1.19409.1 + 5.0.0-preview.1.20112.8 16.9.150 16.8.30406.65-pre + + + + + PreserveNewest + runtimes + false + false + + + PreserveNewest + runtimes + false + false + + + PreserveNewest + runtimes + false + false + + + + + + + \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/Emit/Microsoft.CodeAnalysis.CSharp.Emit.UnitTests.csproj b/src/Compilers/CSharp/Test/Emit/Microsoft.CodeAnalysis.CSharp.Emit.UnitTests.csproj index 85d2492702f27..263273fa60cbc 100644 --- a/src/Compilers/CSharp/Test/Emit/Microsoft.CodeAnalysis.CSharp.Emit.UnitTests.csproj +++ b/src/Compilers/CSharp/Test/Emit/Microsoft.CodeAnalysis.CSharp.Emit.UnitTests.csproj @@ -30,5 +30,5 @@ - + \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/IOperation/Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests.csproj b/src/Compilers/CSharp/Test/IOperation/Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests.csproj index fa56463514b76..125343cfda957 100644 --- a/src/Compilers/CSharp/Test/IOperation/Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests.csproj +++ b/src/Compilers/CSharp/Test/IOperation/Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests.csproj @@ -21,5 +21,5 @@ - + \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/Semantic/Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests.csproj b/src/Compilers/CSharp/Test/Semantic/Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests.csproj index e6be84fb87db4..0a1c989caf04b 100644 --- a/src/Compilers/CSharp/Test/Semantic/Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests.csproj +++ b/src/Compilers/CSharp/Test/Semantic/Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests.csproj @@ -22,5 +22,5 @@ - + \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/Symbol/Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests.csproj b/src/Compilers/CSharp/Test/Symbol/Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests.csproj index 222edb6d31c18..b457b865a4d00 100644 --- a/src/Compilers/CSharp/Test/Symbol/Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests.csproj +++ b/src/Compilers/CSharp/Test/Symbol/Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests.csproj @@ -20,5 +20,5 @@ - + diff --git a/src/Compilers/CSharp/Test/WinRT/Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests.csproj b/src/Compilers/CSharp/Test/WinRT/Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests.csproj index d356fd48dad1f..72a86bd676d9a 100644 --- a/src/Compilers/CSharp/Test/WinRT/Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests.csproj +++ b/src/Compilers/CSharp/Test/WinRT/Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests.csproj @@ -18,5 +18,5 @@ - + diff --git a/src/Compilers/VisualBasic/Test/Emit/Microsoft.CodeAnalysis.VisualBasic.Emit.UnitTests.vbproj b/src/Compilers/VisualBasic/Test/Emit/Microsoft.CodeAnalysis.VisualBasic.Emit.UnitTests.vbproj index f31c1eeb62e5a..991366659f83d 100644 --- a/src/Compilers/VisualBasic/Test/Emit/Microsoft.CodeAnalysis.VisualBasic.Emit.UnitTests.vbproj +++ b/src/Compilers/VisualBasic/Test/Emit/Microsoft.CodeAnalysis.VisualBasic.Emit.UnitTests.vbproj @@ -168,5 +168,5 @@ - + diff --git a/src/Compilers/VisualBasic/Test/Semantic/Microsoft.CodeAnalysis.VisualBasic.Semantic.UnitTests.vbproj b/src/Compilers/VisualBasic/Test/Semantic/Microsoft.CodeAnalysis.VisualBasic.Semantic.UnitTests.vbproj index 0fed500079196..57ff244223cac 100644 --- a/src/Compilers/VisualBasic/Test/Semantic/Microsoft.CodeAnalysis.VisualBasic.Semantic.UnitTests.vbproj +++ b/src/Compilers/VisualBasic/Test/Semantic/Microsoft.CodeAnalysis.VisualBasic.Semantic.UnitTests.vbproj @@ -83,5 +83,5 @@ - + \ No newline at end of file diff --git a/src/Compilers/VisualBasic/Test/Symbol/Microsoft.CodeAnalysis.VisualBasic.Symbol.UnitTests.vbproj b/src/Compilers/VisualBasic/Test/Symbol/Microsoft.CodeAnalysis.VisualBasic.Symbol.UnitTests.vbproj index 84e8efbcdfad3..e08abf620eb6f 100644 --- a/src/Compilers/VisualBasic/Test/Symbol/Microsoft.CodeAnalysis.VisualBasic.Symbol.UnitTests.vbproj +++ b/src/Compilers/VisualBasic/Test/Symbol/Microsoft.CodeAnalysis.VisualBasic.Symbol.UnitTests.vbproj @@ -32,5 +32,5 @@ - + \ No newline at end of file diff --git a/src/EditorFeatures/CSharpTest/Microsoft.CodeAnalysis.CSharp.EditorFeatures.UnitTests.csproj b/src/EditorFeatures/CSharpTest/Microsoft.CodeAnalysis.CSharp.EditorFeatures.UnitTests.csproj index d4b4827ca2ec6..9d45d519b36c2 100644 --- a/src/EditorFeatures/CSharpTest/Microsoft.CodeAnalysis.CSharp.EditorFeatures.UnitTests.csproj +++ b/src/EditorFeatures/CSharpTest/Microsoft.CodeAnalysis.CSharp.EditorFeatures.UnitTests.csproj @@ -75,5 +75,5 @@ - + diff --git a/src/ExpressionEvaluator/CSharp/Test/ExpressionCompiler/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler.UnitTests.csproj b/src/ExpressionEvaluator/CSharp/Test/ExpressionCompiler/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler.UnitTests.csproj index 44580297f2bcc..185a531feed87 100644 --- a/src/ExpressionEvaluator/CSharp/Test/ExpressionCompiler/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler.UnitTests.csproj +++ b/src/ExpressionEvaluator/CSharp/Test/ExpressionCompiler/Microsoft.CodeAnalysis.CSharp.ExpressionCompiler.UnitTests.csproj @@ -30,5 +30,5 @@ - + \ No newline at end of file diff --git a/src/ExpressionEvaluator/CSharp/Test/ResultProvider/Microsoft.CodeAnalysis.CSharp.ResultProvider.UnitTests.csproj b/src/ExpressionEvaluator/CSharp/Test/ResultProvider/Microsoft.CodeAnalysis.CSharp.ResultProvider.UnitTests.csproj index 077b60fd24523..2a2309cb7b90e 100644 --- a/src/ExpressionEvaluator/CSharp/Test/ResultProvider/Microsoft.CodeAnalysis.CSharp.ResultProvider.UnitTests.csproj +++ b/src/ExpressionEvaluator/CSharp/Test/ResultProvider/Microsoft.CodeAnalysis.CSharp.ResultProvider.UnitTests.csproj @@ -32,5 +32,5 @@ - + \ No newline at end of file diff --git a/src/ExpressionEvaluator/Core/Test/FunctionResolver/Microsoft.CodeAnalysis.FunctionResolver.UnitTests.csproj b/src/ExpressionEvaluator/Core/Test/FunctionResolver/Microsoft.CodeAnalysis.FunctionResolver.UnitTests.csproj index 16ddfbcb98932..89d587c4f25c1 100644 --- a/src/ExpressionEvaluator/Core/Test/FunctionResolver/Microsoft.CodeAnalysis.FunctionResolver.UnitTests.csproj +++ b/src/ExpressionEvaluator/Core/Test/FunctionResolver/Microsoft.CodeAnalysis.FunctionResolver.UnitTests.csproj @@ -26,5 +26,5 @@ - + \ No newline at end of file diff --git a/src/ExpressionEvaluator/VisualBasic/Test/ExpressionCompiler/Microsoft.CodeAnalysis.VisualBasic.ExpressionCompiler.UnitTests.vbproj b/src/ExpressionEvaluator/VisualBasic/Test/ExpressionCompiler/Microsoft.CodeAnalysis.VisualBasic.ExpressionCompiler.UnitTests.vbproj index 1d56e4d4c94da..24990afab6887 100644 --- a/src/ExpressionEvaluator/VisualBasic/Test/ExpressionCompiler/Microsoft.CodeAnalysis.VisualBasic.ExpressionCompiler.UnitTests.vbproj +++ b/src/ExpressionEvaluator/VisualBasic/Test/ExpressionCompiler/Microsoft.CodeAnalysis.VisualBasic.ExpressionCompiler.UnitTests.vbproj @@ -34,5 +34,5 @@ - + \ No newline at end of file diff --git a/src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/Microsoft.CodeAnalysis.VisualBasic.ResultProvider.UnitTests.vbproj b/src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/Microsoft.CodeAnalysis.VisualBasic.ResultProvider.UnitTests.vbproj index 6bde6cecbbd52..d7d9ee3090407 100644 --- a/src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/Microsoft.CodeAnalysis.VisualBasic.ResultProvider.UnitTests.vbproj +++ b/src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/Microsoft.CodeAnalysis.VisualBasic.ResultProvider.UnitTests.vbproj @@ -35,5 +35,5 @@ - + \ No newline at end of file diff --git a/src/Tools/BuildValidator/BuildValidator.csproj b/src/Tools/BuildValidator/BuildValidator.csproj index 1f87b9e58f7e8..a3a2cb6306373 100644 --- a/src/Tools/BuildValidator/BuildValidator.csproj +++ b/src/Tools/BuildValidator/BuildValidator.csproj @@ -26,4 +26,5 @@ + diff --git a/src/Tools/BuildValidator/CompilationDiff.cs b/src/Tools/BuildValidator/CompilationDiff.cs index 87c80739e286f..b233112653301 100644 --- a/src/Tools/BuildValidator/CompilationDiff.cs +++ b/src/Tools/BuildValidator/CompilationDiff.cs @@ -212,9 +212,8 @@ public static unsafe CompilationDiff Create( var ildasmOriginalOutputPath = Path.Combine(originalPath, assemblyName + ".il"); var ildasmRebuildOutputPath = Path.Combine(rebuildPath, assemblyName + ".il"); - // TODO: can we bundle ildasm in with the utility? - Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{originalBinaryPath.FullName} /all /out={ildasmOriginalOutputPath}").WaitForExit(); - Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{rebuildAssemblyPath} /all /out={ildasmRebuildOutputPath}").WaitForExit(); + Process.Start(IldasmUtilities.IldasmPath, $@"{originalBinaryPath.FullName} /all /out={ildasmOriginalOutputPath}").WaitForExit(); + Process.Start(IldasmUtilities.IldasmPath, $@"{rebuildAssemblyPath} /all /out={ildasmRebuildOutputPath}").WaitForExit(); File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-pe.mdv.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{originalPeMdvPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{rebuildPeMdvPath.Substring(assemblyDebugPath.Length)}"")"); File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-pdb.mdv.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{originalPdbMdvPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{rebuildPdbMdvPath.Substring(assemblyDebugPath.Length)}"")"); diff --git a/src/Tools/BuildValidator/IldasmUtilities.cs b/src/Tools/BuildValidator/IldasmUtilities.cs new file mode 100644 index 0000000000000..52adab7da7600 --- /dev/null +++ b/src/Tools/BuildValidator/IldasmUtilities.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using Roslyn.Utilities; + +namespace BuildValidator +{ + public static class IldasmUtilities + { + private static string GetIldasmPath() + { + var ildasmExeName = PlatformInformation.IsWindows ? "ildasm.exe" : "ildasm"; + var directory = Path.GetDirectoryName(typeof(IldasmUtilities).GetTypeInfo().Assembly.Location) ?? throw new DirectoryNotFoundException(); + string ridName; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + ridName = "win-x64"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + ridName = "osx-x64"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + ridName = "linux-x64"; + } + else + { + throw new PlatformNotSupportedException("Runtime platform not supported for testing"); + } + + return Path.Combine(directory, "runtimes", ridName, "native", ildasmExeName); + } + + internal static readonly string IldasmPath = GetIldasmPath(); + } +} From 884a9c10bdec31dd3a658b8008525c924586097f Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Wed, 24 Feb 2021 12:26:28 -0800 Subject: [PATCH 8/9] use helix queue --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cac2be8fa8dc4..3d1e6a024468e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,7 +215,8 @@ jobs: - job: Correctness_Rebuild pool: - vmImage: 'windows-2019' + name: NetCorePublic-Pool + queue: BuildPool.Windows.10.Amd64.Open timeoutInMinutes: 90 steps: - template: eng/pipelines/checkout-windows-task.yml From b948154fb6fdb4b7391311a03c72d01e153616da Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Thu, 25 Feb 2021 15:12:12 -0800 Subject: [PATCH 9/9] public -> internal IldasmUtilities --- src/Tools/BuildValidator/IldasmUtilities.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tools/BuildValidator/IldasmUtilities.cs b/src/Tools/BuildValidator/IldasmUtilities.cs index 52adab7da7600..8204473a95087 100644 --- a/src/Tools/BuildValidator/IldasmUtilities.cs +++ b/src/Tools/BuildValidator/IldasmUtilities.cs @@ -10,7 +10,7 @@ namespace BuildValidator { - public static class IldasmUtilities + internal static class IldasmUtilities { private static string GetIldasmPath() {