Skip to content

Commit

Permalink
[Opt-in] Parallelize Targets when building a solution (dotnet#7512)
Browse files Browse the repository at this point in the history 
Fixes
dotnet#5072 (comment)

Context
When building a SLN, a metaproj is used to represent the build behavior. When there are multiple targets (ex clean;build), the current behavior is to run all of first Target in the projects, then run second Target. To improve the parallelism, the solution can pass both target to the project. Each project can start the second target without waiting for all of the first Target to finish.
When the feature is enabled via environment variable, MSBuildSolutionBatchTargets, Solution Generator will create a "SlnProjectResolveProjectReference" target to build all the project/targets. All targets will depend on this new target.

Add support for "SkipNonexistentProjects" as a metadata in MSBuild task. This allow the removal of it as a parameter during solution generation.

Testing
Added unit tests.
  • Loading branch information
@yuehuang010
yuehuang010 authored Jul 18, 2022
1 parent a1d9d69 commit c849248
Show file tree
Hide file tree
Showing 7 changed files with 427 additions and 51 deletions.
50 changes: 50 additions & 0 deletions src/Build.UnitTests/BackEnd/MSBuild_Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,56 @@ public void SkipNonexistentProjectsBuildingInParallel()
Assert.DoesNotContain(error, logger.FullLog);
}

/// <summary>
/// Verifies that nonexistent projects are skipped when requested when building in parallel.
/// DDB # 125831
/// </summary>
[Fact]
public void SkipNonexistentProjectsAsMetadataBuildingInParallel()
{
ObjectModelHelpers.DeleteTempProjectDirectory();
ObjectModelHelpers.CreateFileInTempProjectDirectory(
"SkipNonexistentProjectsMain.csproj",
@"<Project ToolsVersion=`msbuilddefaulttoolsversion` xmlns=`msbuildnamespace`>
<Target Name=`t` >
<ItemGroup>
<ProjectReference Include=`this_project_does_not_exist_warn.csproj` >
<SkipNonexistentProjects>true</SkipNonexistentProjects>
</ProjectReference>
<ProjectReference Include=`this_project_does_not_exist_error.csproj` >
</ProjectReference>
<ProjectReference Include=`foo.csproj` >
<SkipNonexistentProjects>false</SkipNonexistentProjects>
</ProjectReference>
</ItemGroup>
<MSBuild Projects=`@(ProjectReference)` BuildInParallel=`true` />
</Target>
</Project>
");

ObjectModelHelpers.CreateFileInTempProjectDirectory(
"foo.csproj",
@"<Project ToolsVersion=`msbuilddefaulttoolsversion` xmlns=`msbuildnamespace`>
<Target Name=`t` >
<Message Text=`Hello from foo.csproj`/>
</Target>
</Project>
");

MockLogger logger = new MockLogger(_testOutput);
ObjectModelHelpers.BuildTempProjectFileExpectFailure(@"SkipNonexistentProjectsMain.csproj", logger);

logger.AssertLogContains("Hello from foo.csproj");
string message = String.Format(AssemblyResources.GetString("MSBuild.ProjectFileNotFoundMessage"), "this_project_does_not_exist_warn.csproj");
string error = String.Format(AssemblyResources.GetString("MSBuild.ProjectFileNotFound"), "this_project_does_not_exist_warn.csproj");
string error2 = String.Format(AssemblyResources.GetString("MSBuild.ProjectFileNotFound"), "this_project_does_not_exist_error.csproj");
Assert.Equal(0, logger.WarningCount);
Assert.Equal(1, logger.ErrorCount);
Assert.Contains(message, logger.FullLog); // for the missing project
Assert.Contains(error2, logger.FullLog);
Assert.DoesNotContain(error, logger.FullLog);
}

[Fact]
public void LogErrorWhenBuildingVCProj()
{
Expand Down
155 changes: 155 additions & 0 deletions src/Build.UnitTests/Construction/SolutionProjectGenerator_Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,161 @@ public void BuildProjectAsTarget()
}
}

/// <summary>
/// Build Solution with Multiple Targets (ex. Clean;Build;Custom).
/// </summary>
[Fact]
public void BuildProjectWithMultipleTargets()
{
using (TestEnvironment testEnvironment = TestEnvironment.Create())
{
TransientTestFolder folder = testEnvironment.CreateFolder(createFolder: true);
TransientTestFolder classLibFolder = testEnvironment.CreateFolder(Path.Combine(folder.Path, "classlib"), createFolder: true);
TransientTestFile classLibrary = testEnvironment.CreateFile(classLibFolder, "classlib.csproj",
@"<Project>
<Target Name=""Build"">
<Message Text=""classlib.Build""/>
</Target>
<Target Name=""Clean"">
<Message Text=""classlib.Clean""/>
</Target>
<Target Name=""Custom"">
<Message Text=""classlib.Custom""/>
</Target>
</Project>
");

TransientTestFolder simpleProjectFolder = testEnvironment.CreateFolder(Path.Combine(folder.Path, "simpleProject"), createFolder: true);
TransientTestFile simpleProject = testEnvironment.CreateFile(simpleProjectFolder, "simpleProject.csproj",
@"<Project>
<Target Name=""Build"">
<Message Text=""simpleProject.Build""/>
</Target>
<Target Name=""Clean"">
<Message Text=""simpleProject.Clean""/>
</Target>
<Target Name=""Custom"">
<Message Text=""simpleProject.Custom""/>
</Target>
</Project>
");

TransientTestFile solutionFile = testEnvironment.CreateFile(folder, "testFolder.sln",
@"
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.6.30114.105
MinimumVisualStudioVersion = 10.0.40219.1
Project(""{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"") = ""simpleProject"", ""simpleProject\simpleProject.csproj"", ""{AA52A05F-A9C0-4C89-9933-BF976A304C91}""
EndProject
Project(""{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"") = ""classlib"", ""classlib\classlib.csproj"", ""{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}""
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x86 = Debug|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{AA52A05F-A9C0-4C89-9933-BF976A304C91}.Debug|x86.ActiveCfg = Debug|x86
{AA52A05F-A9C0-4C89-9933-BF976A304C91}.Debug|x86.Build.0 = Debug|x86
{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}.Debug|x86.ActiveCfg = Debug|x86
{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}.Debug|x86.Build.0 = Debug|x86
EndGlobalSection
EndGlobal
");

string output = RunnerUtilities.ExecMSBuild(solutionFile.Path + " /t:Clean;Build;Custom", out bool success);
success.ShouldBeTrue();
output.ShouldContain("classlib.Build");
output.ShouldContain("classlib.Clean");
output.ShouldContain("classlib.Custom");
output.ShouldContain("simpleProject.Build");
output.ShouldContain("simpleProject.Clean");
output.ShouldContain("simpleProject.Custom");
}
}


/// <summary>
/// Build Solution with Multiple Targets (ex. Clean;Build;Custom).
/// </summary>
[Fact]
public void BuildProjectWithMultipleTargetsInParallel()
{
using (TestEnvironment testEnvironment = TestEnvironment.Create())
{
TransientTestFolder folder = testEnvironment.CreateFolder(createFolder: true);
TransientTestFolder classLibFolder = testEnvironment.CreateFolder(Path.Combine(folder.Path, "classlib"), createFolder: true);
TransientTestFile classLibrary = testEnvironment.CreateFile(classLibFolder, "classlib.csproj",
@"<Project>
<Target Name=""Build"">
<Message Text=""classlib.Build""/>
</Target>
<Target Name=""Clean"">
<Message Text=""classlib.Clean""/>
</Target>
<Target Name=""Custom"">
<Message Text=""classlib.Custom""/>
</Target>
</Project>
");

TransientTestFolder simpleProjectFolder = testEnvironment.CreateFolder(Path.Combine(folder.Path, "simpleProject"), createFolder: true);
TransientTestFile simpleProject = testEnvironment.CreateFile(simpleProjectFolder, "simpleProject.csproj",
@"<Project>
<Target Name=""Build"">
<Message Text=""simpleProject.Build""/>
</Target>
<Target Name=""Clean"">
<Message Text=""simpleProject.Clean""/>
</Target>
<Target Name=""Custom"">
<Message Text=""simpleProject.Custom""/>
</Target>
</Project>
");

TransientTestFile solutionFile = testEnvironment.CreateFile(folder, "testFolder.sln",
@"
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.6.30114.105
MinimumVisualStudioVersion = 10.0.40219.1
Project(""{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"") = ""simpleProject"", ""simpleProject\simpleProject.csproj"", ""{AA52A05F-A9C0-4C89-9933-BF976A304C91}""
EndProject
Project(""{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"") = ""classlib"", ""classlib\classlib.csproj"", ""{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}""
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x86 = Debug|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{AA52A05F-A9C0-4C89-9933-BF976A304C91}.Debug|x86.ActiveCfg = Debug|x86
{AA52A05F-A9C0-4C89-9933-BF976A304C91}.Debug|x86.Build.0 = Debug|x86
{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}.Debug|x86.ActiveCfg = Debug|x86
{80B8E6B8-E46D-4456-91B1-848FD35C4AB9}.Debug|x86.Build.0 = Debug|x86
EndGlobalSection
EndGlobal
");

try
{
Environment.SetEnvironmentVariable("MSBuildSolutionBatchTargets", "1");
var output = RunnerUtilities.ExecMSBuild(solutionFile.Path + " /m /t:Clean;Build;Custom", out bool success);
success.ShouldBeTrue();
output.ShouldContain("classlib.Build");
output.ShouldContain("classlib.Clean");
output.ShouldContain("classlib.Custom");
output.ShouldContain("simpleProject.Build");
output.ShouldContain("simpleProject.Clean");
output.ShouldContain("simpleProject.Custom");
}
finally
{
Environment.SetEnvironmentVariable("MSBuildSolutionBatchTargets", "");
}
}
}

/// <summary>
/// Verify the AddNewErrorWarningMessageElement method
/// </summary>
Expand Down
73 changes: 55 additions & 18 deletions src/Build/BackEnd/Components/RequestBuilder/IntrinsicTasks/MSBuild.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,13 @@ internal class MSBuild : ITask
/// <summary>
/// Enum describing the behavior when a project doesn't exist on disk.
/// </summary>
private enum SkipNonexistentProjectsBehavior
private enum SkipNonExistentProjectsBehavior
{
/// <summary>
/// Default when unset by user.
/// </summary>
Undefined,

/// <summary>
/// Skip the project if there is no file on disk.
/// </summary>
Expand All @@ -49,7 +54,7 @@ private enum SkipNonexistentProjectsBehavior
private readonly List<ITaskItem> _targetOutputs = new List<ITaskItem>();

// Whether to skip project files that don't exist on disk. By default we error for such projects.
private SkipNonexistentProjectsBehavior _skipNonexistentProjects = SkipNonexistentProjectsBehavior.Error;
private SkipNonExistentProjectsBehavior _skipNonExistentProjects = SkipNonExistentProjectsBehavior.Undefined;

private TaskLoggingHelper _logHelper;

Expand Down Expand Up @@ -162,19 +167,22 @@ public string SkipNonexistentProjects
{
get
{
switch (_skipNonexistentProjects)
switch (_skipNonExistentProjects)
{
case SkipNonexistentProjectsBehavior.Build:
case SkipNonExistentProjectsBehavior.Undefined:
return "Undefined";

case SkipNonExistentProjectsBehavior.Build:
return "Build";

case SkipNonexistentProjectsBehavior.Error:
case SkipNonExistentProjectsBehavior.Error:
return "False";

case SkipNonexistentProjectsBehavior.Skip:
case SkipNonExistentProjectsBehavior.Skip:
return "True";

default:
ErrorUtilities.ThrowInternalError("Unexpected case {0}", _skipNonexistentProjects);
ErrorUtilities.ThrowInternalError("Unexpected case {0}", _skipNonExistentProjects);
break;
}

Expand All @@ -184,15 +192,9 @@ public string SkipNonexistentProjects

set
{
if (String.Equals("Build", value, StringComparison.OrdinalIgnoreCase))
{
_skipNonexistentProjects = SkipNonexistentProjectsBehavior.Build;
}
else
if (TryParseSkipNonExistentProjects(value, out SkipNonExistentProjectsBehavior behavior))
{
ErrorUtilities.VerifyThrowArgument(ConversionUtilities.CanConvertStringToBool(value), "MSBuild.InvalidSkipNonexistentProjectValue");
bool originalSkipValue = ConversionUtilities.ConvertStringToBool(value);
_skipNonexistentProjects = originalSkipValue ? SkipNonexistentProjectsBehavior.Skip : SkipNonexistentProjectsBehavior.Error;
_skipNonExistentProjects = behavior;
}
}
}
Expand Down Expand Up @@ -324,7 +326,21 @@ public async Task<bool> ExecuteInternal()
break;
}

if (FileSystems.Default.FileExists(projectPath) || (_skipNonexistentProjects == SkipNonexistentProjectsBehavior.Build))
// Try to get the behavior from metadata if it is undefined.
var skipNonExistProjects = _skipNonExistentProjects;
if (_skipNonExistentProjects == SkipNonExistentProjectsBehavior.Undefined)
{
if (TryParseSkipNonExistentProjects(project.GetMetadata("SkipNonexistentProjects"), out SkipNonExistentProjectsBehavior behavior))
{
skipNonExistProjects = behavior;
}
else
{
skipNonExistProjects = SkipNonExistentProjectsBehavior.Error;
}
}

if (FileSystems.Default.FileExists(projectPath) || (skipNonExistProjects == SkipNonExistentProjectsBehavior.Build))
{
if (FileUtilities.IsVCProjFilename(projectPath))
{
Expand Down Expand Up @@ -365,13 +381,13 @@ public async Task<bool> ExecuteInternal()
}
else
{
if (_skipNonexistentProjects == SkipNonexistentProjectsBehavior.Skip)
if (skipNonExistProjects == SkipNonExistentProjectsBehavior.Skip)
{
Log.LogMessageFromResources(MessageImportance.High, "MSBuild.ProjectFileNotFoundMessage", project.ItemSpec);
}
else
{
ErrorUtilities.VerifyThrow(_skipNonexistentProjects == SkipNonexistentProjectsBehavior.Error, "skipNonexistentProjects has unexpected value {0}", _skipNonexistentProjects);
ErrorUtilities.VerifyThrow(skipNonExistProjects == SkipNonExistentProjectsBehavior.Error, "skipNonexistentProjects has unexpected value {0}", skipNonExistProjects);
Log.LogErrorWithCodeFromResources("MSBuild.ProjectFileNotFound", project.ItemSpec);
success = false;
}
Expand Down Expand Up @@ -714,6 +730,27 @@ internal static async Task<bool> ExecuteTargets(
return success;
}

private bool TryParseSkipNonExistentProjects(string value, out SkipNonExistentProjectsBehavior behavior)
{
if (string.IsNullOrEmpty(value))
{
behavior = SkipNonExistentProjectsBehavior.Error;
return false;
}
else if (String.Equals("Build", value, StringComparison.OrdinalIgnoreCase))
{
behavior = SkipNonExistentProjectsBehavior.Build;
}
else
{
ErrorUtilities.VerifyThrowArgument(ConversionUtilities.CanConvertStringToBool(value), "MSBuild.InvalidSkipNonexistentProjectValue");
bool originalSkipValue = ConversionUtilities.ConvertStringToBool(value);
behavior = originalSkipValue ? SkipNonExistentProjectsBehavior.Skip : SkipNonExistentProjectsBehavior.Error;
}

return true;
}

#endregion
}
}
Loading

0 comments on commit c849248

Please sign in to comment.