From b207c25ec5c1ff2c81d8e90200e096a6d393f154 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 23 Jun 2021 12:32:14 +0000 Subject: [PATCH 01/17] Update dependencies from https://github.com/dotnet/arcade build 20210621.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21321.2 --- eng/Version.Details.xml | 8 +- eng/common/build.ps1 | 2 + eng/common/cross/arm64/tizen-fetch.sh | 2 +- eng/common/cross/build-rootfs.sh | 4 + eng/common/dotnet-install.sh | 2 +- eng/common/generate-locproject.ps1 | 25 +- eng/common/internal-feed-operations.ps1 | 6 +- eng/common/internal-feed-operations.sh | 2 +- eng/common/msbuild.ps1 | 1 + eng/common/performance/blazor_perf.proj | 30 -- eng/common/performance/crossgen_perf.proj | 110 ------- eng/common/performance/microbenchmarks.proj | 144 --------- eng/common/performance/performance-setup.ps1 | 139 -------- eng/common/performance/performance-setup.sh | 297 ------------------ .../post-build/sourcelink-validation.ps1 | 80 +++-- eng/common/post-build/symbols-validation.ps1 | 32 +- eng/common/sdk-task.ps1 | 8 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 2 +- eng/common/sdl/init-sdl.ps1 | 2 +- eng/common/sdl/run-sdl.ps1 | 2 +- eng/common/templates/job/onelocbuild.yml | 14 +- eng/common/templates/job/performance.yml | 95 ------ eng/common/templates/job/source-build.yml | 3 + .../templates/job/source-index-stage1.yml | 6 +- .../channels/generic-internal-channel.yml | 6 + .../channels/generic-public-channel.yml | 6 + .../templates/steps/perf-send-to-helix.yml | 50 --- eng/common/templates/steps/source-build.yml | 7 +- eng/common/tools.ps1 | 116 +++---- eng/common/tools.sh | 9 +- global.json | 8 +- 31 files changed, 223 insertions(+), 995 deletions(-) delete mode 100644 eng/common/performance/blazor_perf.proj delete mode 100644 eng/common/performance/crossgen_perf.proj delete mode 100644 eng/common/performance/microbenchmarks.proj delete mode 100644 eng/common/performance/performance-setup.ps1 delete mode 100755 eng/common/performance/performance-setup.sh delete mode 100644 eng/common/templates/job/performance.yml delete mode 100644 eng/common/templates/steps/perf-send-to-helix.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 096efad97e425..eb273f386839d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 46ad27d3cc557f2b84ce30b2c4e27438526dc91d + 28a6403ee97077256fcdc60f599f0ad9e38e3cfa https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 46ad27d3cc557f2b84ce30b2c4e27438526dc91d + 28a6403ee97077256fcdc60f599f0ad9e38e3cfa diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c0817e8b..8943da242f6e9 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh index a48a6f51c49d0..16d1301f21e4c 100644 --- a/eng/common/cross/arm64/tizen-fetch.sh +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -157,7 +157,7 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index b26622444f5ba..591d8666a84ef 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -74,6 +74,10 @@ __IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" zlib-1.2.11" +# ML.NET dependencies +__UbuntuPackages+=" libomp5" +__UbuntuPackages+=" libomp-dev" + __UseMirror=0 __UnprocessedBuildArgs= diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index d6efeb44340ba..fdfeea66e7d43 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -70,7 +70,7 @@ case $cpuname in ;; esac -dotnetRoot="$repo_root/.dotnet" +dotnetRoot="${repo_root}.dotnet" if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 24c00b5be98b0..25e97ac00772d 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop" Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') -$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } if (Test-Path -Path $exclusionsFilePath) { @@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to # Template files $jsonFiles = @() -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $xlfFiles = @() @@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object { $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } -$locFiles = $jsonFiles + $xlfFiles +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles $locJson = @{ Projects = @( @@ -92,14 +99,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n" Pop-Location if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json } else { - New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." exit 1 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 418c09930cf16..92b77347d9904 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -45,11 +45,11 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - $nugetConfigPath = "$RepoRoot\NuGet.config" + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" if (-Not (Test-Path -Path $nugetConfigPath)) { Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 + ExitWithExitCode 1 } $endpoints = New-Object System.Collections.ArrayList @@ -85,7 +85,7 @@ function SetupCredProvider { #Workaround for https://github.com/microsoft/msbuild/issues/4430 function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetTempDir = Join-Path $RepoRoot "dotnet" $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) $dotnet = "$dotnetTempDir\dotnet.exe" $restoreProjPath = "$PSScriptRoot\restore.proj" diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index e2233e781220f..9378223ba0955 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -39,7 +39,7 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - local nugetConfigPath="$repo_root/NuGet.config" + local nugetConfigPath="{$repo_root}NuGet.config" if [ ! "$nugetConfigPath" ]; then Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index c6401230002fe..eea19cd8452fd 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -5,6 +5,7 @@ Param( [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, + [switch] $excludePrereleaseVS, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/performance/blazor_perf.proj b/eng/common/performance/blazor_perf.proj deleted file mode 100644 index 3b25359c43807..0000000000000 --- a/eng/common/performance/blazor_perf.proj +++ /dev/null @@ -1,30 +0,0 @@ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - - - - - %(Identity) - - - - - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)blazor\ - - - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)blazor/ - - - - - $(WorkItemDirectory) - cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27 - $(Python) test.py sod --scenario-name "%(Identity)" - $(Python) post.py - - - \ No newline at end of file diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj deleted file mode 100644 index eb8bdd9c440cd..0000000000000 --- a/eng/common/performance/crossgen_perf.proj +++ /dev/null @@ -1,110 +0,0 @@ - - - - - %(Identity) - - - - - - py -3 - $(HelixPreCommands) - %HELIX_CORRELATION_PAYLOAD%\Core_Root - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)crossgen\ - $(ScenarioDirectory)crossgen2\ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update;chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - $HELIX_CORRELATION_PAYLOAD/Core_Root - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)crossgen/ - $(ScenarioDirectory)crossgen2/ - - - - - - - - - - - - - - - - - - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) --singlethreaded True - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)pre.py crossgen --core-root $(CoreRoot) --single %(Identity) - $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(CrossgenDirectory)post.py - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(Crossgen2Directory)post.py - - - - - - - 4:00 - - - - 4:00 - - - 4:00 - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp - 1:00 - - - 4:00 - - - 4:00 - - - \ No newline at end of file diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj deleted file mode 100644 index 318ca5f1b8d4c..0000000000000 --- a/eng/common/performance/microbenchmarks.proj +++ /dev/null @@ -1,144 +0,0 @@ - - - - %HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj) - --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP% - py -3 - %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe - %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline - %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj - %HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe - %25%25 - %HELIX_WORKITEM_ROOT%\testResults.xml - - - - $HELIX_CORRELATION_PAYLOAD - $(BaseDirectory)/performance - - - - $HELIX_WORKITEM_PAYLOAD - $(BaseDirectory) - - - - $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) - --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP - python3 - $(BaseDirectory)/Core_Root/corerun - $(BaseDirectory)/Baseline_Core_Root/corerun - $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj - $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet - %25 - $HELIX_WORKITEM_ROOT/testResults.xml - - - - $(CliArguments) --wasm - - - - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe - - - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun - - - - --corerun $(CoreRun) - - - - --corerun $(BaselineCoreRun) - - - - $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments) - - - - $(WorkItemCommand) $(CliArguments) - - - - 2:30 - 0:15 - - - - - %(Identity) - - - - - 30 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - $(WorkItemTimeout) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) - 4:00 - - - diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 deleted file mode 100644 index 9a64b07e692f6..0000000000000 --- a/eng/common/performance/performance-setup.ps1 +++ /dev/null @@ -1,139 +0,0 @@ -Param( - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, - [string] $CoreRootDirectory, - [string] $BaselineCoreRootDirectory, - [string] $Architecture="x64", - [string] $Framework="net5.0", - [string] $CompilationMode="Tiered", - [string] $Repository=$env:BUILD_REPOSITORY_NAME, - [string] $Branch=$env:BUILD_SOURCEBRANCH, - [string] $CommitSha=$env:BUILD_SOURCEVERSION, - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="Libraries Runtime", - [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", - [string] $Kind="micro", - [switch] $LLVM, - [switch] $MonoInterpreter, - [switch] $MonoAOT, - [switch] $Internal, - [switch] $Compare, - [string] $MonoDotnet="", - [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind", - [string] $LogicalMachine="" -) - -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") -$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) -$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) - -$PayloadDirectory = (Join-Path $SourceDirectory "Payload") -$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") -$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") -$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -$Creator = $env:BUILD_DEFINITIONNAME -$PerfLabArguments = "" -$HelixSourcePrefix = "pr" - -$Queue = "" - -if ($Internal) { - switch ($LogicalMachine) { - "perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - "perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf" } - "perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf" } - Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - } - $PerfLabArguments = "--upload-to-perflab-container" - $ExtraBenchmarkDotNetArguments = "" - $Creator = "" - $HelixSourcePrefix = "official" -} -else { - $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -} - -if($MonoInterpreter) -{ - $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" -} - -if($MonoDotnet -ne "") -{ - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" - if($ExtraBenchmarkDotNetArguments -eq "") - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } - else - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } -} - -# FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" -$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" - - -if ($RunFromPerformanceRepo) { - $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" - - robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git -} -else { - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory -} - -if($MonoDotnet -ne "") -{ - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath -} - -if ($UseCoreRun) { - $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") - Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot -} -if ($UseBaselineCoreRun) { - $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") - Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot -} - -$DocsDir = (Join-Path $PerformanceDirectory "docs") -robocopy $DocsDir $WorkItemDirectory - -# Set variables that we will need to have in future steps -$ci = $true - -. "$PSScriptRoot\..\pipeline-logging-functions.ps1" - -# Directories -Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false - -# Script Arguments -Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false - -# Helix Arguments -Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false - -exit 0 \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh deleted file mode 100755 index 33b60b5033742..0000000000000 --- a/eng/common/performance/performance-setup.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash - -source_directory=$BUILD_SOURCESDIRECTORY -core_root_directory= -baseline_core_root_directory= -architecture=x64 -framework=net5.0 -compilation_mode=tiered -repository=$BUILD_REPOSITORY_NAME -branch=$BUILD_SOURCEBRANCH -commit_sha=$BUILD_SOURCEVERSION -build_number=$BUILD_BUILDNUMBER -internal=false -compare=false -mono_dotnet= -kind="micro" -llvm=false -monointerpreter=false -monoaot=false -run_categories="Libraries Runtime" -csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations="CompliationMode=$compilation_mode RunKind=$kind" -run_from_perf_repo=false -use_core_run=true -use_baseline_core_run=true -using_mono=false -wasm_runtime_loc= -using_wasm=false -use_latest_dotnet=false -logical_machine= - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --sourcedirectory) - source_directory=$2 - shift 2 - ;; - --corerootdirectory) - core_root_directory=$2 - shift 2 - ;; - --baselinecorerootdirectory) - baseline_core_root_directory=$2 - shift 2 - ;; - --architecture) - architecture=$2 - shift 2 - ;; - --framework) - framework=$2 - shift 2 - ;; - --compilationmode) - compilation_mode=$2 - shift 2 - ;; - --logicalmachine) - logical_machine=$2 - shift 2 - ;; - --repository) - repository=$2 - shift 2 - ;; - --branch) - branch=$2 - shift 2 - ;; - --commitsha) - commit_sha=$2 - shift 2 - ;; - --buildnumber) - build_number=$2 - shift 2 - ;; - --kind) - kind=$2 - configurations="CompilationMode=$compilation_mode RunKind=$kind" - shift 2 - ;; - --runcategories) - run_categories=$2 - shift 2 - ;; - --csproj) - csproj=$2 - shift 2 - ;; - --internal) - internal=true - shift 1 - ;; - --alpine) - alpine=true - shift 1 - ;; - --llvm) - llvm=true - shift 1 - ;; - --monointerpreter) - monointerpreter=true - shift 1 - ;; - --monoaot) - monoaot=true - shift 1 - ;; - --monodotnet) - mono_dotnet=$2 - shift 2 - ;; - --wasm) - wasm_runtime_loc=$2 - shift 2 - ;; - --compare) - compare=true - shift 1 - ;; - --configurations) - configurations=$2 - shift 2 - ;; - --latestdotnet) - use_latest_dotnet=true - shift 1 - ;; - *) - echo "Common settings:" - echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" - echo " --architecture Architecture of the testing being run" - echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." - echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --framework The framework to run, if not running in master" - echo " --compliationmode The compilation mode if not passing --configurations" - echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" - echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" - echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" - echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" - echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" - echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" - echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" - echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" - echo " --internal If the benchmarks are running as an official job." - echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." - echo " --wasm Path to the unpacked wasm runtime pack." - echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " - echo " --alpine Set for runs on Alpine" - echo "" - exit 0 - ;; - esac -done - -if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then - run_from_perf_repo=true -fi - -if [ -z "$configurations" ]; then - configurations="CompilationMode=$compilation_mode" -fi - -if [ -z "$core_root_directory" ]; then - use_core_run=false -fi - -if [ -z "$baseline_core_root_directory" ]; then - use_baseline_core_run=false -fi - -payload_directory=$source_directory/Payload -performance_directory=$payload_directory/performance -workitem_directory=$source_directory/workitem -extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -perflab_arguments= -queue=Ubuntu.1804.Amd64.Open -creator=$BUILD_DEFINITIONNAME -helix_source_prefix="pr" - -if [[ "$internal" == true ]]; then - perflab_arguments="--upload-to-perflab-container" - helix_source_prefix="official" - creator= - extra_benchmark_dotnet_arguments= - - if [[ "$architecture" = "arm64" ]]; then - queue=Ubuntu.1804.Arm64.Perf - else - if [[ "$logical_machine" = "perfowl" ]]; then - queue=Ubuntu.1804.Amd64.Owl.Perf - else - queue=Ubuntu.1804.Amd64.Tiger.Perf - fi - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -else - if [[ "$architecture" = "arm64" ]]; then - queue=ubuntu.1804.armarch.open - else - queue=Ubuntu.1804.Amd64.Open - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - configurations="CompilationMode=wasm RunKind=$kind" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono" -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" -fi - -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" -setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -if [[ "$run_from_perf_repo" = true ]]; then - payload_directory= - workitem_directory=$source_directory - performance_directory=$workitem_directory - setup_arguments="--perf-hash $commit_sha $common_setup_arguments" -else - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory - - docs_directory=$performance_directory/docs - mv $docs_directory $workitem_directory -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - using_wasm=true - wasm_dotnet_path=$payload_directory/dotnet-wasm - mv $wasm_runtime_loc $wasm_dotnet_path - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" -fi - -if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path -fi - -if [[ "$use_core_run" = true ]]; then - new_core_root=$payload_directory/Core_Root - mv $core_root_directory $new_core_root -fi - -if [[ "$use_baseline_core_run" = true ]]; then - new_baseline_core_root=$payload_directory/Baseline_Core_Root - mv $baseline_core_root_directory $new_baseline_core_root -fi - -ci=true - -_script_dir=$(pwd)/eng/common -. "$_script_dir/pipeline-logging-functions.sh" - -# Make sure all of our variables are available for future steps -Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false -Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false -Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false -Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false -Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false -Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false -Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false -Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false -Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false -Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 1c46f7b634148..85c89861719ad 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -14,7 +14,9 @@ param( $global:RepoFiles = @{} # Maximum number of jobs to run in parallel -$MaxParallelJobs = 6 +$MaxParallelJobs = 16 + +$MaxRetries = 5 # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -29,7 +31,10 @@ $ValidatePackage = { # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-Host "Input file does not exist: $PackagePath" - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } } # Extensions for which we'll look for SourceLink information @@ -59,7 +64,10 @@ $ValidatePackage = { # We ignore resource DLLs if ($FileName.EndsWith('.resources.dll')) { - return + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) @@ -91,36 +99,49 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + $totalRetries = 0 + + while ($totalRetries -lt $using:MaxRetries) { + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + # If it's not a github link, we want to break out of the loop and not retry. + $Status = 0 + $totalRetries = $using:MaxRetries + } } - else { + catch { + Write-Host $_ $Status = 0 } } - catch { - write-host $_ - $Status = 0 - } - } - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host + if ($Status -ne 200) { + $totalRetries++ + + if ($totalRetries -ge $using:MaxRetries) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ } - - Write-Host "`tFile $RealPath has broken links:" } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ + else { + break + } } } } @@ -136,7 +157,7 @@ $ValidatePackage = { } } catch { - + Write-Host $_ } finally { $zip.Dispose() @@ -220,6 +241,7 @@ function ValidateSourceLinkLinks { # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { + Write-Host "Starting $($_.FullName)" Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -267,6 +289,10 @@ function InstallSourcelinkCli { try { InstallSourcelinkCli + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + ValidateSourceLinkLinks } catch { diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index e6d5d2fd05455..a5af041ba775c 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -8,7 +8,7 @@ param( ) # Maximum number of jobs to run in parallel -$MaxParallelJobs = 6 +$MaxParallelJobs = 16 # Max number of retries $MaxRetry = 5 @@ -114,6 +114,7 @@ $CountMissingSymbols = { $totalRetries = 0 while ($totalRetries -lt $using:MaxRetry) { + # Save the output and get diagnostic output $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String @@ -132,20 +133,28 @@ $CountMissingSymbols = { elseif (Test-Path $SymbolPath) { return 'Module' } - elseif ($output.Contains("503 Service Unavailable")) { - # If we got a 503 error, we should retry. + else + { $totalRetries++ } - else { - return $null - } } return $null } - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--microsoft-symbol-server' ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--internal-server' ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam Write-Host -NoNewLine "`t Checking file " $FileName "... " @@ -208,6 +217,7 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalPackages = 0 $TotalFailures = 0 $DupedSymbols = 0 @@ -230,6 +240,8 @@ function CheckSymbolsAvailable { return } + $TotalPackages++ + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -255,11 +267,11 @@ function CheckSymbolsAvailable { if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" } if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" } ExitWithExitCode 1 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index f55c43c6f4786..b1bca63ab1d82 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -34,7 +34,7 @@ function Print-Usage() { function Build([string]$target) { $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } $log = Join-Path $LogDir "$task$logSuffix.binlog" - $outputPath = Join-Path $ToolsetDir "$task\\" + $outputPath = Join-Path $ToolsetDir "$task\" MSBuild $taskProject ` /bl:$log ` @@ -53,7 +53,7 @@ try { } if ($task -eq "") { - Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" -ForegroundColor Red + Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" Print-Usage ExitWithExitCode 1 } @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true @@ -78,7 +78,7 @@ try { $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red + Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" ExitWithExitCode 1 } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 81b729f74a4d4..2881a56083cf5 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -32,7 +32,7 @@ try { $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true - $LASTEXITCODE = 0 + $global:LASTEXITCODE = 0 # `tools.ps1` checks $ci to perform some actions. Since the SDL # scripts don't necessarily execute in the same agent that run the diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 1fe9271193cc6..3ac1d92b37006 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -10,7 +10,7 @@ Param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 # `tools.ps1` checks $ci to perform some actions. Since the SDL # scripts don't necessarily execute in the same agent that run the diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index fe95ab35aa5d1..3d9c87aba6acf 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -13,7 +13,7 @@ Param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 try { # `tools.ps1` checks $ci to perform some actions. Since the SDL diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index b27d6faf3035d..e8bc77d2ebbe3 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -12,11 +12,15 @@ parameters: SourcesDirectory: $(Build.SourcesDirectory) CreatePr: true AutoCompletePr: false + UseLfLineEndings: true UseCheckedInLocProjectJson: false LanguageSet: VS_Main_Languages LclSource: lclFilesInRepo LclPackageId: '' RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main condition: '' jobs: @@ -52,18 +56,24 @@ jobs: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: - locProj: Localize/LocProject.json + locProj: eng/Localize/LocProject.json outDir: $(Build.ArtifactStagingDirectory) lclSource: ${{ parameters.LclSource }} lclPackageId: ${{ parameters.LclPackageId }} isCreatePrSelected: ${{ parameters.CreatePr }} ${{ if eq(parameters.CreatePr, true) }}: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} condition: ${{ parameters.condition }} - task: PublishBuildArtifacts@1 @@ -77,7 +87,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish LocProject.json inputs: - PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' PublishLocation: Container ArtifactName: Loc condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml deleted file mode 100644 index f877fd7a89800..0000000000000 --- a/eng/common/templates/job/performance.yml +++ /dev/null @@ -1,95 +0,0 @@ -parameters: - steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) - variables: [] # optional -- list of additional variables to send to the template - jobName: '' # required -- job name - displayName: '' # optional -- display name for the job. Will use jobName if not passed - pool: '' # required -- name of the Build pool - container: '' # required -- name of the container - osGroup: '' # required -- operating system for the job - extraSetupParameters: '' # optional -- extra arguments to pass to the setup script - frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against - continueOnError: 'false' # optional -- determines whether to continue the build if the step errors - dependsOn: '' # optional -- dependencies of the job - timeoutInMinutes: 320 # optional -- timeout for the job - enableTelemetry: false # optional -- enable for telemetry - -jobs: -- template: ../jobs/jobs.yml - parameters: - dependsOn: ${{ parameters.dependsOn }} - enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true - continueOnError: ${{ parameters.continueOnError }} - - jobs: - - job: '${{ parameters.jobName }}' - - ${{ if ne(parameters.displayName, '') }}: - displayName: '${{ parameters.displayName }}' - ${{ if eq(parameters.displayName, '') }}: - displayName: '${{ parameters.jobName }}' - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - variables: - - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - - IsInternal: '' - - HelixApiAccessToken: '' - - HelixPreCommand: '' - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - - IsInternal: -Internal - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - IsInternal: --internal - - - group: DotNet-HelixApi-Access - - group: dotnet-benchview - - workspace: - clean: all - pool: - ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ each framework in parameters.frameworks }}: - ${{ framework }}: - _Framework: ${{ framework }} - steps: - - checkout: self - clean: true - # Run all of the steps to setup repo - - ${{ each step in parameters.steps }}: - - ${{ step }} - - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Unix) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) - displayName: Run ci setup script - # Run perf testing in helix - - template: /eng/common/templates/steps/perf-send-to-helix.yml - parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - Creator: $(Creator) - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy - CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index aad41464926c1..5023d36dcb3c5 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -15,6 +15,9 @@ parameters: # nonPortable: false # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. # container: '' # A container to use. Runs in docker. # pool: {} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index a649d2b5990c0..b58d42364b9e7 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,15 +1,19 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20210421.1 + sourceIndexPackageVersion: 1.0.1-20210614.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog pool: vmImage: vs2017-win2016 + condition: '' + dependsOn: '' jobs: - job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} variables: - name: SourceIndexPackageVersion value: ${{ parameters.sourceIndexPackageVersion }} diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 58fa9a35b8ca9..8990dfc8c87cc 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -40,6 +40,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + # This is necessary whenever we want to publish/restore to an AzDO private feed - task: NuGetAuthenticate@0 displayName: 'Authenticate to AzDO Feeds' @@ -110,6 +113,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index b50c0b3bdb8e6..3220c6a4f92ff 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -42,6 +42,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -109,6 +112,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml deleted file mode 100644 index 0fb786fabf7e3..0000000000000 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - ProjectFile: '' # required -- project file that specifies the helix workitems - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - osGroup: '' # required -- operating system for the job - - -steps: -- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml - parameters: - osGroup: ${{ parameters.osGroup }} - sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} - condition: ${{ parameters.condition }} - shouldContinueOnError: ${{ parameters.continueOnError }} - environment: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 8e336b7d16b34..e20637ed6a177 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -34,9 +34,14 @@ steps: targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack --publish \ + --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d52467eea11d2..5619c7aaee1d7 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -48,6 +48,9 @@ # True to use global NuGet cache instead of restoring packages to repository-local directory. [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } +# True to exclude prerelease versions Visual Studio during build +[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false } + # An array of names of processes to stop on script exit if prepareMachine is true. $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') } @@ -190,38 +193,42 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir = $dotnetRoot } +function Retry($downloadBlock, $maxRetries = 5) { + $retries = 1 + + while($true) { + try { + & $downloadBlock + break + } + catch { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." + break + } + + } +} + function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - - $maxRetries = 5 - $retries = 1 - $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - while($true) { - try { - Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript - break - } - catch { - Write-Host "Failed to download '$uri'" - Write-Error $_.Exception.Message -ErrorAction Continue - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - throw "Unable to download file in $maxRetries attempts." - } - - } + Retry({ + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + }) } return $installScript @@ -305,8 +312,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview - $defaultXCopyMSBuildVersion = '16.8.0-preview3' + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview + $defaultXCopyMSBuildVersion = '16.10.0-preview2' if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } @@ -400,9 +407,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { } Create-Directory $packageDir + Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + Retry({ + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + }) + Unzip $packagePath $packageDir } @@ -439,31 +450,17 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir Write-Host 'Downloading vswhere' - $maxRetries = 5 - $retries = 1 - - while($true) { - try { - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe - break - } - catch{ - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." - } - } + Retry({ + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + }) } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + + if (!$excludePrereleaseVS) { + $args += '-prerelease' + } if (Get-Member -InputObject $vsRequirements -Name 'version') { $args += '-version' @@ -489,7 +486,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){ function InitializeBuildTool() { if (Test-Path variable:global:_BuildTool) { - return $global:_BuildTool + # If the requested msbuild parameters do not match, clear the cached variables. + if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) { + Remove-Item variable:global:_BuildTool + Remove-Item variable:global:_MSBuildExe + } else { + return $global:_BuildTool + } } if (-not $msbuildEngine) { @@ -517,7 +520,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } } else { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 @@ -542,7 +545,7 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build. + # Use local cache on CI to ensure deterministic build. # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 @@ -699,7 +702,10 @@ function MSBuild-Core() { } foreach ($arg in $args) { - if ($arg -ne $null -and $arg.Trim() -ne "") { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } $cmdArgs += " `"$arg`"" } } @@ -771,7 +777,7 @@ function Get-Darc($version) { . $PSScriptRoot\pipeline-logging-functions.ps1 -$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\') $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..') $ArtifactsDir = Join-Path $RepoRoot 'artifacts' $ToolsetDir = Join-Path $ArtifactsDir 'toolset' diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 5fad1846e5a53..05ca99c6b2813 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -485,13 +485,14 @@ _script_dir=`dirname "$_ResolvePath"` eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` -artifacts_dir="$repo_root/artifacts" +repo_root="${repo_root}/" +artifacts_dir="${repo_root}artifacts" toolset_dir="$artifacts_dir/toolset" -tools_dir="$repo_root/.tools" +tools_dir="${repo_root}.tools" log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" -global_json_file="$repo_root/global.json" +global_json_file="${repo_root}global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false if command -v jq &> /dev/null; then @@ -504,7 +505,7 @@ fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then - export HOME="$repo_root/artifacts/.home/" + export HOME="${repo_root}artifacts/.home/" mkdir -p "$HOME" fi diff --git a/global.json b/global.json index 812f9823e3a73..a95d3189afce6 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,18 @@ { "sdk": { - "version": "6.0.100-preview.1.21103.13", + "version": "6.0.100-preview.4.21255.9", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "6.0.100-preview.1.21103.13", + "dotnet": "6.0.100-preview.4.21255.9", "vs": { "version": "16.8" }, "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21230.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21230.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21321.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21321.2" } } From ef69de8fd1fbd9a0233f3d7b65a48b6c568a5e3f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 24 Jun 2021 12:37:07 +0000 Subject: [PATCH 02/17] Update dependencies from https://github.com/dotnet/arcade build 20210623.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21323.1 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index eb273f386839d..dcfb439be2901 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 28a6403ee97077256fcdc60f599f0ad9e38e3cfa + 1685a12f35cfb396f9352d63736eb675673b0e20 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 28a6403ee97077256fcdc60f599f0ad9e38e3cfa + 1685a12f35cfb396f9352d63736eb675673b0e20 diff --git a/global.json b/global.json index a95d3189afce6..f2b63c0b741ef 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21321.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21321.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21323.1", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21323.1" } } From 15f1b85be05b9fc1e796c860ef34138a73a7b320 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 25 Jun 2021 12:30:40 +0000 Subject: [PATCH 03/17] Update dependencies from https://github.com/dotnet/arcade build 20210624.3 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21324.3 --- eng/Version.Details.xml | 8 ++++---- eng/common/tools.ps1 | 11 ++++++++++- global.json | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dcfb439be2901..61850e76f0111 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 1685a12f35cfb396f9352d63736eb675673b0e20 + a68ec1edf328e737b31a09cb49e1929c28e91d0c https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 1685a12f35cfb396f9352d63736eb675673b0e20 + a68ec1edf328e737b31a09cb49e1929c28e91d0c diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5619c7aaee1d7..7942ffaf4cb3c 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -378,7 +378,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + + $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" + $local:Prefer64bit = if ($vsRequirements.Prefer64bit) { $vsRequirements.Prefer64bit } else { $false } + if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { + $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" + } else { + $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" + } + + return $global:_MSBuildExe } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { diff --git a/global.json b/global.json index f2b63c0b741ef..f66016cbfb268 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21323.1", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21323.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21324.3", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21324.3" } } From fa181401dfbfd18c32175cffb7c0f2d84ca755c2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 29 Jun 2021 12:32:57 +0000 Subject: [PATCH 04/17] Update dependencies from https://github.com/dotnet/arcade build 20210628.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21328.2 --- eng/Version.Details.xml | 8 ++++---- eng/common/tools.ps1 | 2 +- global.json | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 61850e76f0111..506bc1bc881ec 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - a68ec1edf328e737b31a09cb49e1929c28e91d0c + 685c1a4fa207d81e881a402a32ffa1c0fb191b42 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - a68ec1edf328e737b31a09cb49e1929c28e91d0c + 685c1a4fa207d81e881a402a32ffa1c0fb191b42 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7942ffaf4cb3c..4b25520324931 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -380,7 +380,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" - $local:Prefer64bit = if ($vsRequirements.Prefer64bit) { $vsRequirements.Prefer64bit } else { $false } + $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false } if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" } else { diff --git a/global.json b/global.json index f66016cbfb268..3fc127e8de481 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21324.3", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21324.3" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21328.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21328.2" } } From 3383966f012be3e61111a584452b7ae95ab059d2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 30 Jun 2021 12:37:58 +0000 Subject: [PATCH 05/17] Update dependencies from https://github.com/dotnet/arcade build 20210629.8 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21329.8 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 506bc1bc881ec..1efdba95b4dde 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 685c1a4fa207d81e881a402a32ffa1c0fb191b42 + 6b9d24236d8d1906284e6cb6c28e3fe93a69b7d2 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 685c1a4fa207d81e881a402a32ffa1c0fb191b42 + 6b9d24236d8d1906284e6cb6c28e3fe93a69b7d2 diff --git a/global.json b/global.json index 3fc127e8de481..eec23915e18bc 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21328.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21328.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21329.8", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21329.8" } } From 785d873304636be084c9893fcecfba3f6cddf5d0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 1 Jul 2021 12:33:59 +0000 Subject: [PATCH 06/17] Update dependencies from https://github.com/dotnet/arcade build 20210630.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21330.2 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1efdba95b4dde..c55aaac1a3bd9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 6b9d24236d8d1906284e6cb6c28e3fe93a69b7d2 + 26345756f99087811b1fe4d02ff213eb172ec506 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 6b9d24236d8d1906284e6cb6c28e3fe93a69b7d2 + 26345756f99087811b1fe4d02ff213eb172ec506 diff --git a/global.json b/global.json index eec23915e18bc..2dfd8aab3395c 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21329.8", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21329.8" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21330.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21330.2" } } From 078613939a1b16abffe449ae8673973132f7645f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 2 Jul 2021 12:30:42 +0000 Subject: [PATCH 07/17] Update dependencies from https://github.com/dotnet/arcade build 20210701.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21351.2 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c55aaac1a3bd9..0c652e1cbb62a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 26345756f99087811b1fe4d02ff213eb172ec506 + 7f52af24016a4b749a2c1f219105c0f5a7100028 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 26345756f99087811b1fe4d02ff213eb172ec506 + 7f52af24016a4b749a2c1f219105c0f5a7100028 diff --git a/global.json b/global.json index 2dfd8aab3395c..065066869ab84 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21330.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21330.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21351.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21351.2" } } From 6602a339a8b530f14f48eaac0e1c22ff4ec4b919 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 6 Jul 2021 12:31:13 +0000 Subject: [PATCH 08/17] Update dependencies from https://github.com/dotnet/arcade build 20210705.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21355.2 --- eng/Version.Details.xml | 8 ++++---- eng/common/SetupNugetSources.ps1 | 6 ++++++ eng/common/SetupNugetSources.sh | 24 ++++++++++++++++++++++++ global.json | 4 ++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0c652e1cbb62a..855f9c40dfe9e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 7f52af24016a4b749a2c1f219105c0f5a7100028 + 615fbd8c498067deae160a696701eaf33155da56 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 7f52af24016a4b749a2c1f219105c0f5a7100028 + 615fbd8c498067deae160a696701eaf33155da56 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a0b5fc37f4388..18823840b1127 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } +$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") +if ($dotnet6Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 2734601c13c4b..ad3fb74fd2cc8 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet5-internal-transport') fi +# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet6-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal-transport') +fi + # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/global.json b/global.json index 065066869ab84..0343c8168b937 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21351.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21351.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21355.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21355.2" } } From 875981bb2ec8c7c0c031b4ebf3b2d5148642ea22 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 14 Jul 2021 12:39:26 +0000 Subject: [PATCH 09/17] Update dependencies from https://github.com/dotnet/arcade build 20210713.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21363.2 --- eng/Version.Details.xml | 8 ++++---- eng/common/cross/build-rootfs.sh | 6 +++++- eng/common/tools.ps1 | 4 ++-- eng/common/tools.sh | 4 ++-- global.json | 4 ++-- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 855f9c40dfe9e..a13cf913717fc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 615fbd8c498067deae160a696701eaf33155da56 + 1b053babb8a542e3ab20f93b3d0aedc732b7e0c7 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 615fbd8c498067deae160a696701eaf33155da56 + 1b053babb8a542e3ab20f93b3d0aedc732b7e0c7 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 591d8666a84ef..735a4c8283898 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -33,7 +33,6 @@ __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" __AlpinePackagesEdgeCommunity=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm10-libs" __AlpinePackagesEdgeMain+=" python3" __AlpinePackagesEdgeMain+=" libedit" @@ -115,6 +114,8 @@ while :; do __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') unset __LLDB_Package ;; x86) @@ -191,6 +192,8 @@ while :; do __CodeName=alpine __UbuntuRepo= __AlpineVersion=3.9 + __AlpinePackagesEdgeMain+=" llvm11-libs" + __AlpinePackagesEdgeMain+=" clang-libs" ;; alpine3.13) __CodeName=alpine @@ -201,6 +204,7 @@ while :; do __AlpinePackagesEdgeCommunity= __AlpinePackages+=$__AlpinePackagesEdgeMain __AlpinePackagesEdgeMain= + __AlpinePackages+=" llvm10-libs" ;; freebsd11) __FreeBSDBase="11.3-RELEASE" diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 4b25520324931..2df0909937d10 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -42,7 +42,7 @@ [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dot.net/v1/dotnet-install.ps1 +# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -223,7 +223,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" + $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" Retry({ Write-Host "GET $uri" diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 05ca99c6b2813..828119be411b3 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dot.net/v1/dotnet-install.sh +# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -262,7 +262,7 @@ function with_retries { function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" - local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh" + local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" if [[ ! -a "$install_script" ]]; then mkdir -p "$root" diff --git a/global.json b/global.json index 0343c8168b937..a6c76ccad656f 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21355.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21355.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21363.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21363.2" } } From 3c94d64b46d12083bccfd451779cd9ef5ec694eb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 15 Jul 2021 12:39:52 +0000 Subject: [PATCH 10/17] Update dependencies from https://github.com/dotnet/arcade build 20210714.3 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21364.3 --- eng/Version.Details.xml | 8 +- eng/common/sdl/configure-sdl-tool.ps1 | 109 +++++++++++++++++++ eng/common/sdl/execute-all-sdl-tools.ps1 | 71 ++++++++++-- eng/common/sdl/extract-artifact-archives.ps1 | 63 +++++++++++ eng/common/sdl/run-sdl.ps1 | 50 +++------ eng/common/templates/job/execute-sdl.yml | 108 ++++++++++++++++-- eng/common/tools.ps1 | 51 +++++++++ global.json | 4 +- 8 files changed, 401 insertions(+), 63 deletions(-) create mode 100644 eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 eng/common/sdl/extract-artifact-archives.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a13cf913717fc..86cd10feed232 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 1b053babb8a542e3ab20f93b3d0aedc732b7e0c7 + 6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 1b053babb8a542e3ab20f93b3d0aedc732b7e0c7 + 6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1 diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 new file mode 100644 index 0000000000000..4999c307088a0 --- /dev/null +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -0,0 +1,109 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + # The list of Guardian tools to configure. For each object in the array: + # - If the item is a [hashtable], it must contain these entries: + # - Name = The tool name as Guardian knows it. + # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique + # among all tool entries with the same Name. + # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' + # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' + [object[]] $ToolsList, + [string] $GuardianLoggerLevel='Standard', + # Optional: Additional params to add to any tool using CredScan. + [string[]] $CrScanAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using PoliCheck. + [string[]] $PoliCheckAdditionalRunConfigParams +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # Normalize tools list: all in [hashtable] form with defined values for each key. + $ToolsList = $ToolsList | + ForEach-Object { + if ($_ -is [string]) { + $_ = @{ Name = $_ } + } + + if (-not ($_['Scenario'])) { $_.Scenario = "" } + if (-not ($_['Args'])) { $_.Args = @() } + $_ + } + + Write-Host "List of tools to configure:" + $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + foreach ($tool in $ToolsList) { + # Put together the name and scenario to make a unique key. + $toolConfigName = $tool.Name + if ($tool.Scenario) { + $toolConfigName += "_" + $tool.Scenario + } + + Write-Host "=== Configuring $toolConfigName..." + + $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" + + # For some tools, add default and automatic args. + if ($tool.Name -eq 'credscan') { + if ($targetDirectory) { + $tool.Args += "TargetDirectory < $TargetDirectory" + } + $tool.Args += "OutputType < pre" + $tool.Args += $CrScanAdditionalRunConfigParams + } elseif ($tool.Name -eq 'policheck') { + if ($targetDirectory) { + $tool.Args += "Target < $TargetDirectory" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams + } + + # Create variable pointing to the args array directly so we can use splat syntax later. + $toolArgs = $tool.Args + + # Configure the tool. If args array is provided or the current tool has some default arguments + # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", + # one per parameter. Doc page for "guardian configure": + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure + Exec-BlockVerbosely { + & $GuardianCliLocation configure ` + --working-directory $WorkingDirectory ` + --tool $tool.Name ` + --output-path $gdnConfigFile ` + --logger-level $GuardianLoggerLevel ` + --noninteractive ` + --force ` + $(if ($toolArgs) { "--args" }) @toolArgs + Exit-IfNZEC "Sdl" + } + + Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 2881a56083cf5..1157151f4862a 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -7,8 +7,17 @@ Param( [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + + # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list + # format. + [object[]] $SourceToolsList, + # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools + # list format. + [object[]] $ArtifactToolsList, + # Optional: list of SDL tools to run without automatically specifying a target directory. See + # 'configure-sdl-tool.ps1' for tools list format. + [object[]] $CustomToolsList, + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. @@ -63,13 +72,16 @@ try { ExitWithExitCode 1 } - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + } $gdnFolder = Join-Path $workingDirectory '.gdn' if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -80,11 +92,41 @@ try { } } - if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. + function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { + if ($tools -and $tools.Count -gt 0) { + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -TargetDirectory $targetDirectory ` + -GdnFolder $gdnFolder ` + -ToolsList $tools ` + -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` + -GuardianLoggerLevel $GuardianLoggerLevel ` + -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` + -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + if ($BreakOnFailure) { + Exit-IfNZEC "Sdl" + } + } + } } - if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + + # Configure Artifact and Source tools with default Target directories. + Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory + Configure-ToolsList $SourceToolsList $SourceDirectory + # Configure custom tools with no default Target directory. + Configure-ToolsList $CustomToolsList $null + + # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. + # (If we used "run" multiple times, each run would overwrite data from earlier runs.) + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -UpdateBaseline $UpdateBaseline ` + -GdnFolder $gdnFolder } if ($TsaPublish) { @@ -92,8 +134,9 @@ try { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -106,7 +149,11 @@ try { if ($BreakOnFailure) { Write-Host "Failing the build in case of breaking results..." - & $guardianCliLocation break + Exec-BlockVerbosely { + & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + } else { + Write-Host "Letting the build pass even if there were breaking results..." } } catch { diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1 new file mode 100644 index 0000000000000..68da4fbf25717 --- /dev/null +++ b/eng/common/sdl/extract-artifact-archives.ps1 @@ -0,0 +1,63 @@ +# This script looks for each archive file in a directory and extracts it into the target directory. +# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". +# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. +param( + # Full path to directory where archives are stored. + [Parameter(Mandatory=$true)][string] $InputPath, + # Full path to directory to extract archives into. May be the same as $InputPath. + [Parameter(Mandatory=$true)][string] $ExtractPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + Measure-Command { + $jobs = @() + + # Find archive files for non-Windows and Windows builds. + $archiveFiles = @( + Get-ChildItem (Join-Path $InputPath "*.tar.gz") + Get-ChildItem (Join-Path $InputPath "*.zip") + ) + + foreach ($targzFile in $archiveFiles) { + $jobs += Start-Job -ScriptBlock { + $file = $using:targzFile + $fileName = [System.IO.Path]::GetFileName($file) + $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" + + New-Item $extractDir -ItemType Directory -Force | Out-Null + + Write-Host "Extracting '$file' to '$extractDir'..." + + # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. + # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the + # error. Save output so it can be stored in the exception string along with context. + $output = tar -xf $file -C $extractDir 2>&1 + # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we + # don't have access to the outer scope. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" + } + + Write-Host "Extracted to $extractDir" + } + } + + Receive-Job $jobs -Wait + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 3d9c87aba6acf..2eac8c78f103d 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -1,13 +1,9 @@ Param( [string] $GuardianCliLocation, [string] $WorkingDirectory, - [string] $TargetDirectory, [string] $GdnFolder, - [string[]] $ToolsList, [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard', - [string[]] $CrScanAdditionalRunConfigParams, - [string[]] $PoliCheckAdditionalRunConfigParams + [string] $GuardianLoggerLevel='Standard' ) $ErrorActionPreference = 'Stop' @@ -23,7 +19,6 @@ try { . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn - Write-Host $ToolsList $gdnConfigPath = Join-Path $GdnFolder 'r' $ValidPath = Test-Path $GuardianCliLocation @@ -33,37 +28,18 @@ try { ExitWithExitCode 1 } - $configParam = @('--config') - - foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq 'credscan') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - if ($tool -eq 'policheck') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - - $configParam+=$gdnConfigFile - } - - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE + $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' + Write-Host "Discovered Guardian config files:" + $gdnConfigFiles | Out-String | Write-Host + + Exec-BlockVerbosely { + & $GuardianCliLocation run ` + --working-directory $WorkingDirectory ` + --baseline mainbaseline ` + --update-baseline $UpdateBaseline ` + --logger-level $GuardianLoggerLevel ` + --config @gdnConfigFiles + Exit-IfNZEC "Sdl" } } catch { diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 4a32181fd8f93..69eb67849d741 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -2,17 +2,41 @@ parameters: enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - downloadArtifacts: true # optional: determines if the artifacts should be dowloaded + # optional: determines if build artifacts should be downloaded. + downloadArtifacts: true + # optional: determines if this job should search the directory of downloaded artifacts for + # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. + extractArchiveArtifacts: false dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: # artifactNames: # - 'BlobArtifacts' # - 'Artifacts_Windows_NT_Release' + # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, + # not pipeline artifacts, so doesn't affect the use of this parameter. + pipelineArtifactNames: [] + # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be + # downloaded from. By default, uses runtime expressions to decide based on the variables set by + # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are + # running without Maestro++/BAR involved, or to download artifacts from a specific existing build + # to iterate quickly on SDL changes. + AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] jobs: - job: Run_SDL @@ -22,16 +46,29 @@ jobs: variables: - group: DotNet-VSTS-Bot - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + value: ${{ parameters.AzDOProjectName }} - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + value: ${{ parameters.AzDOPipelineId }} - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + value: ${{ parameters.AzDOBuildId }} + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.53.3 + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: - name: Hosted VS2017 + # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019. + ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}: + name: Hosted VS2017 + ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: + vmImage: windows-2019 steps: - checkout: self clean: true + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: - ${{ each artifactName in parameters.artifactNames }}: @@ -59,16 +96,51 @@ jobs: itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true + + - ${{ each artifactName in parameters.pipelineArtifactNames }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + checkDownloadedFiles: true + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: + - powershell: eng/common/sdl/extract-artifact-archives.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts + displayName: Extract Archive Artifacts + continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - powershell: | + $content = Get-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content was:`n$content" + + $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') + $content | Set-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content updated to:`n$content" + displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} + - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -79,15 +151,35 @@ jobs: nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config externalFeedCredentials: GuardianConnect restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 2df0909937d10..5d526c74d518c 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -106,6 +106,46 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } +# Take the given block, print it, print what the block probably references from the current set of +# variables using low-effort string matching, then run the block. +# +# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes, +# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less +# maintainable and less reliable. It is easy to make a mistake and modify the command without +# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of +# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in +# existing source code can also be difficult, because the strings are not aligned to each other and +# the line may be 300+ columns long. +# +# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues. +# +# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically. +# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it +# doesn't print any info about the variables being used by the command, which is normally the +# interesting part to diagnose. +function Exec-BlockVerbosely([scriptblock] $block) { + Write-Host "--- Running script block:" + $blockString = $block.ToString().Trim() + Write-Host $blockString + + Write-Host "--- List of variables that might be used:" + # For each variable x in the environment, check the block for a reference to x via simple "$x" or + # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x", + # among others). It only catches what this function was originally written for: simple + # command-line commands. + $variableTable = Get-Variable | + Where-Object { + $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)") + } | + Format-Table -AutoSize -HideTableHeaders -Wrap | + Out-String + Write-Host $variableTable.Trim() + + Write-Host "--- Executing:" + & $block + Write-Host "--- Done running script block!" +} + # createSdkLocationFile parameter enables a file being generated under the toolset directory # which writes the sdk's location into. This is only necessary for cmd --> powershell invocations # as dot sourcing isn't possible. @@ -632,6 +672,17 @@ function ExitWithExitCode([int] $exitCode) { exit $exitCode } +# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for +# diagnostics, then exit the script with the $LASTEXITCODE. +function Exit-IfNZEC([string] $category = "General") { + Write-Host "Exit code $LASTEXITCODE" + if ($LASTEXITCODE -ne 0) { + $message = "Last command failed with exit code $LASTEXITCODE." + Write-PipelineTelemetryError -Force -Category $category -Message $message + ExitWithExitCode $LASTEXITCODE + } +} + function Stop-Processes() { Write-Host 'Killing running build processes...' foreach ($processName in $processesToStopOnExit) { diff --git a/global.json b/global.json index a6c76ccad656f..2f823fad84231 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21363.2", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21363.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21364.3", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21364.3" } } From 511d04a9072abdacfd8946b01e62a37e551a37a6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 16 Jul 2021 12:39:17 +0000 Subject: [PATCH 11/17] Update dependencies from https://github.com/dotnet/arcade build 20210715.11 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21365.11 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 86cd10feed232..942679d9f13fe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1 + 7b88b24068ecb4f6cf1da8395dbc2dee25a700bd https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1 + 7b88b24068ecb4f6cf1da8395dbc2dee25a700bd diff --git a/global.json b/global.json index 2f823fad84231..594f6232b98c3 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21364.3", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21364.3" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21365.11", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21365.11" } } From 1cffe086238a9f10e9e383eb3826746b912bb4ed Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 22 Jul 2021 12:22:03 +0000 Subject: [PATCH 12/17] Update dependencies from https://github.com/dotnet/arcade build 20210720.12 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21370.12 --- eng/Version.Details.xml | 8 ++++---- global.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 942679d9f13fe..362ec5d452cb6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 7b88b24068ecb4f6cf1da8395dbc2dee25a700bd + 89806f0b9e93ad2bbe32c654412835c0801a2032 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 7b88b24068ecb4f6cf1da8395dbc2dee25a700bd + 89806f0b9e93ad2bbe32c654412835c0801a2032 diff --git a/global.json b/global.json index 594f6232b98c3..7e432504e9ce4 100644 --- a/global.json +++ b/global.json @@ -12,7 +12,7 @@ "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21365.11", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21365.11" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21370.12", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21370.12" } } From c48e6842edb6f2ce16ab916b2501e860ab7fe9f9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 24 Jul 2021 12:30:17 +0000 Subject: [PATCH 13/17] Update dependencies from https://github.com/dotnet/arcade build 20210723.11 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21373.11 --- eng/Version.Details.xml | 8 ++++---- eng/common/templates/steps/source-build.yml | 6 ++++++ global.json | 8 ++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 362ec5d452cb6..5204a3a2f957a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 89806f0b9e93ad2bbe32c654412835c0801a2032 + c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - 89806f0b9e93ad2bbe32c654412835c0801a2032 + c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index e20637ed6a177..705b7a1c847b9 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -29,6 +29,11 @@ steps: officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' @@ -43,6 +48,7 @@ steps: --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ + $internalRuntimeDownloadArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true diff --git a/global.json b/global.json index 7e432504e9ce4..ad00aa9ead107 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,18 @@ { "sdk": { - "version": "6.0.100-preview.4.21255.9", + "version": "6.0.100-preview.6.21355.2", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "6.0.100-preview.4.21255.9", + "dotnet": "6.0.100-preview.6.21355.2", "vs": { "version": "16.8" }, "xcopy-msbuild": "16.8.0-preview2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21370.12", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21370.12" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21373.11", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21373.11" } } From 9162f37fedbf2ee844f6f37356c93693e3312a48 Mon Sep 17 00:00:00 2001 From: Joey Robichaud Date: Mon, 26 Jul 2021 12:28:53 -0700 Subject: [PATCH 14/17] Update minimum VS to 16.10; Update contributing doc --- azure-pipelines-official.yml | 3 +- azure-pipelines-pr-validation.yml | 1 + azure-pipelines.yml | 4 +-- ...ding, Debugging, and Testing on Windows.md | 6 ++-- eng/Versions.props | 4 +-- eng/test-build-correctness.ps1 | 6 ++-- eng/test-determinism.ps1 | 34 +++++++++---------- eng/test-rebuild.ps1 | 4 +-- global.json | 10 +++--- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index cfe38a4a3fa3f..77ab83500c415 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -47,7 +47,7 @@ stages: timeoutInMinutes: 360 pool: name: NetCoreInternal-Pool - queue: BuildPool.Server.Amd64.VS2017 + queue: BuildPool.Server.Amd64.VS2019 steps: - powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))" @@ -142,6 +142,7 @@ stages: /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) /p:DotnetPublishUsingPipelines=true + /p:IgnoreIbcMergeErrors=true condition: succeeded() - task: PowerShell@2 diff --git a/azure-pipelines-pr-validation.yml b/azure-pipelines-pr-validation.yml index 6ab83e9f52a29..8dc17a33bdd33 100644 --- a/azure-pipelines-pr-validation.yml +++ b/azure-pipelines-pr-validation.yml @@ -123,6 +123,7 @@ stages: /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) /p:DotnetPublishUsingPipelines=false + /p:IgnoreIbcMergeErrors=true condition: succeeded() # Publish OptProf generated JSON files as a build artifact. This allows for easy inspection from diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4aff1d181fdf4..62cca8e159565 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -153,7 +153,7 @@ jobs: - job: Correctness_Determinism pool: name: NetCorePublic-Pool - queue: BuildPool.Windows.10.Amd64.VS2017.Open + queue: BuildPool.Windows.10.Amd64.VS2019.Open timeoutInMinutes: 90 steps: - template: eng/pipelines/checkout-windows-task.yml @@ -169,7 +169,7 @@ jobs: - job: Correctness_Build pool: name: NetCorePublic-Pool - queue: BuildPool.Windows.10.Amd64.VS2017.Open + queue: BuildPool.Windows.10.Amd64.VS2019.Open timeoutInMinutes: 90 steps: - template: eng/pipelines/checkout-windows-task.yml diff --git a/docs/contributing/Building, Debugging, and Testing on Windows.md b/docs/contributing/Building, Debugging, and Testing on Windows.md index f3c08c93ae68d..5261b177b8966 100644 --- a/docs/contributing/Building, Debugging, and Testing on Windows.md +++ b/docs/contributing/Building, Debugging, and Testing on Windows.md @@ -15,12 +15,12 @@ The minimal required version of .NET Framework is 4.7.2. ## Developing with Visual Studio 2019 -1. [Visual Studio 2019 16.9](https://visualstudio.microsoft.com/downloads/) +1. [Visual Studio 2019 16.10](https://visualstudio.microsoft.com/downloads/) - Ensure C#, VB, MSBuild, .NET Core and Visual Studio Extensibility are included in the selected work loads - - Ensure Visual Studio is on Version "16.8" or greater + - Ensure Visual Studio is on Version "16.10" or greater - Ensure "Use previews of the .NET Core SDK" is checked in Tools -> Options -> Environment -> Preview Features - Restart Visual Studio -1. [.NET 6.0 Preview 1 SDK](https://dotnet.microsoft.com/download/dotnet-core/6.0) [Windows x64 installer](https://dotnet.microsoft.com/download/dotnet/thank-you/sdk-6.0.100-preview.1-windows-x64-installer) +1. [.NET 6.0 Preview 6 SDK](https://dotnet.microsoft.com/download/dotnet-core/6.0) [Windows x64 installer](https://dotnet.microsoft.com/download/dotnet/thank-you/sdk-6.0.100-preview.6-windows-x64-installer) 1. [PowerShell 5.0 or newer](https://docs.microsoft.com/en-us/powershell/scripting/setup/installing-windows-powershell). If you are on Windows 10, you are fine; you'll only need to upgrade if you're on earlier versions of Windows. The download link is under the ["Upgrading existing Windows PowerShell"](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-windows-powershell?view=powershell-6#upgrading-existing-windows-powershell) heading. 1. Run Restore.cmd 1. Open Roslyn.sln diff --git a/eng/Versions.props b/eng/Versions.props index a0bb34384e7ee..4f594ca7c290f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -91,7 +91,7 @@ 1.1.0-beta2-20115-01 16.9.0-beta1.21055.5 1.5.0 - 6.0.0-beta.21230.2 + 6.0.0-beta.21373.11 5.0.0 5.0.0 3.13.8 @@ -189,7 +189,7 @@ 16.3.43 4.3.0 5.0.0 - 2.1.133 + 2.1.500 0.1.0 6.6.0.161 4.10.1 diff --git a/eng/test-build-correctness.ps1 b/eng/test-build-correctness.ps1 index a8e9ecb67ebfe..819bce1a52d34 100644 --- a/eng/test-build-correctness.ps1 +++ b/eng/test-build-correctness.ps1 @@ -59,15 +59,15 @@ try { # Verify the state of our various build artifacts Write-Host "Running BuildBoss" $buildBossPath = GetProjectOutputBinary "BuildBoss.exe" - Exec-Console $buildBossPath "-r `"$RepoRoot`" -c $configuration" -p Roslyn.sln + Exec-Console $buildBossPath "-r `"$RepoRoot/`" -c $configuration" -p Roslyn.sln Write-Host "" # Verify the state of our generated syntax files Write-Host "Checking generated compiler files" Exec-Block { & (Join-Path $PSScriptRoot "generate-compiler-code.ps1") -test -configuration:$configuration } Exec-Console dotnet "format . --include-generated --include src/Compilers/CSharp/Portable/Generated/ src/Compilers/VisualBasic/Portable/Generated/ src/ExpressionEvaluator/VisualBasic/Source/ResultProvider/Generated/ --check -f" - Write-Host "" - + Write-Host "" + exit 0 } catch [exception] { diff --git a/eng/test-determinism.ps1 b/eng/test-determinism.ps1 index fa5d1001bd27c..e7c36de72ba8d 100644 --- a/eng/test-determinism.ps1 +++ b/eng/test-determinism.ps1 @@ -21,7 +21,7 @@ if ($help) { } # List of binary names that should be skipped because they have a known issue that -# makes them non-deterministic. +# makes them non-deterministic. $script:skipList = @( # Added to work around https://github.com/dotnet/roslyn/issues/48417 "Microsoft.CodeAnalysis.EditorFeatures2.UnitTests.dll" @@ -67,11 +67,11 @@ function Run-Build([string]$rootDir, [string]$logFileName) { Stop-Processes } -function Get-ObjDir([string]$rootDir) { +function Get-ObjDir([string]$rootDir) { return Join-Path $rootDir "artifacts\obj" } -function Get-BinDir([string]$rootDir) { +function Get-BinDir([string]$rootDir) { return Join-Path $rootDir "artifacts\bin" } @@ -80,7 +80,7 @@ function Get-BinDir([string]$rootDir) { function Get-FilesToProcess([string]$rootDir) { $objDir = Get-ObjDir $rootDir foreach ($item in Get-ChildItem -re -in *.dll,*.exe,*.pdb,*.sourcelink.json $objDir) { - $filePath = $item.FullName + $filePath = $item.FullName $fileName = Split-Path -leaf $filePath $relativeDirectory = Split-Path -parent $filePath $relativeDirectory = $relativeDirectory.Substring($objDir.Length) @@ -103,7 +103,7 @@ function Get-FilesToProcess([string]$rootDir) { $keyFilePath = $filePath + ".key" $keyFileName = Split-Path -leaf $keyFilePath - if (Test-Path $keyFilePath) { + if (Test-Path $keyFilePath) { $data.KeyFileName = $keyFileName $data.KeyFilePath = $keyFilePath $data.KeyFileContent = [IO.File]::ReadAllBytes($keyFilePath) @@ -124,7 +124,7 @@ function Record-Binaries([string]$rootDir) { Write-Host "Recording file hashes" $map = @{ } - foreach ($fileData in Get-FilesToProcess $rootDir) { + foreach ($fileData in Get-FilesToProcess $rootDir) { Write-Host "`t$($fileData.FileId) = $($fileData.Hash)" $map[$fileData.FileId] = $fileData } @@ -134,9 +134,9 @@ function Record-Binaries([string]$rootDir) { } # This is a sanity check to ensure that we're actually putting the right entries into -# the core data map. Essentially to ensure things like if we change our directory layout -# that this test fails beacuse we didn't record the binaries we intended to record. -function Test-MapContents($dataMap) { +# the core data map. Essentially to ensure things like if we change our directory layout +# that this test fails beacuse we didn't record the binaries we intended to record. +function Test-MapContents($dataMap) { # Sanity check to ensure we didn't return a false positive because we failed # to examine any binaries. @@ -151,16 +151,16 @@ function Test-MapContents($dataMap) { "Microsoft.CodeAnalysis.Workspaces.dll", "Microsoft.VisualStudio.LanguageServices.Implementation.dll") - foreach ($fileName in $list) { + foreach ($fileName in $list) { $found = $false - foreach ($value in $dataMap.Values) { - if ($value.FileName -eq $fileName) { + foreach ($value in $dataMap.Values) { + if ($value.FileName -eq $fileName) { $found = $true break; } } - if (-not $found) { + if (-not $found) { throw "Did not find the expected binary $fileName" } } @@ -187,7 +187,7 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFileName) { } $oldfileData = $datamap[$fileId] - if ($fileData.Hash -ne $oldFileData.Hash) { + if ($fileData.Hash -ne $oldFileData.Hash) { Write-Host "`tERROR! $relativeDir\$fileName contents don't match" $allGood = $false $errorList += $fileName @@ -242,13 +242,13 @@ function Run-Test() { # Run a test against the source in the same directory location Test-Build -rootDir $RepoRoot -dataMap $dataMap -logFileName "test1" - # Run another build in a different source location and verify that path mapping - # allows the build to be identical. To do this we'll copy the entire source + # Run another build in a different source location and verify that path mapping + # allows the build to be identical. To do this we'll copy the entire source # tree under the artifacts\q directory and run a build from there. Write-Host "Building in a different directory" Exec-Command "subst" "$altRootDrive $(Split-Path -parent $RepoRoot)" try { - $altRootDir = Join-Path "$($altRootDrive)\" (Split-Path -leaf $RepoRoot) + $altRootDir = Join-Path (Join-Path "$($altRootDrive)\" (Split-Path -leaf $RepoRoot)) "\" Test-Build -rootDir $altRootDir -dataMap $dataMap -logFileName "test2" } finally { diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index b2b0f3914e525..8cd2f7adc0731 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -1,5 +1,5 @@ <# - This script tests that Roslyn artifacts are rebuildable--i.e. that the source code and resources can be identified + This script tests that Roslyn artifacts are rebuildable--i.e. that the source code and resources can be identified #> [CmdletBinding(PositionalBinding=$false)] @@ -63,7 +63,7 @@ try { " --exclude net472\Zip\tools\vsixexpinstaller\VSIXExpInstaller.exe" + " --debugPath `"$ArtifactsDir/BuildValidator`"" + - " --sourcePath `"$RepoRoot`"" + + " --sourcePath `"$RepoRoot/`"" + " --referencesPath `"$ArtifactsDir/bin`"" + " --referencesPath `"$dotnetInstallDir/packs`"") Exec-Console "$ArtifactsDir/bin/BuildValidator/$configuration/net472/BuildValidator.exe" $rebuildArgs diff --git a/global.json b/global.json index ad00aa9ead107..6dc93ebae19fb 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,16 @@ { "sdk": { - "version": "6.0.100-preview.6.21355.2", - "allowPrerelease": true, - "rollForward": "major" + "version": "6.0.100-preview.6.21355.2" }, "tools": { "dotnet": "6.0.100-preview.6.21355.2", "vs": { - "version": "16.8" + "version": "16.10" }, - "xcopy-msbuild": "16.8.0-preview2.1" + "xcopy-msbuild": "16.10.0-preview2" }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21373.11", "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21373.11" } -} +} \ No newline at end of file From 6cfcfb5aba1cbb63233da031a8863880a523e1bd Mon Sep 17 00:00:00 2001 From: Joey Robichaud Date: Mon, 26 Jul 2021 16:04:58 -0700 Subject: [PATCH 15/17] Downgrade DotNetBuildTasksFeed version --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 2180792ac512c..46c3df62b3c6c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -91,7 +91,7 @@ 1.1.0-beta2-20115-01 16.9.0-beta1.21055.5 1.5.0 - 6.0.0-beta.21373.11 + 6.0.0-beta.21319.2 5.0.0 5.0.0 3.13.8 From aae7ff89509d0449da656058ebd272bb68f4f0fa Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 27 Jul 2021 12:38:55 +0000 Subject: [PATCH 16/17] Update dependencies from https://github.com/dotnet/arcade build 20210726.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 6.0.0-beta.21230.2 -> To Version 6.0.0-beta.21376.4 --- eng/Version.Details.xml | 8 ++++---- global.json | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5204a3a2f957a..aa527a62d883b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 + e6abc186425e675e0f201acc1c33e183a0e51c86 https://github.com/dotnet/roslyn 9b40ab852a3b59ce9cef33b3e97c17af13c8e694 - + https://github.com/dotnet/arcade - c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 + e6abc186425e675e0f201acc1c33e183a0e51c86 diff --git a/global.json b/global.json index 6dc93ebae19fb..818b43b532773 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ "xcopy-msbuild": "16.10.0-preview2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21373.11", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21373.11" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21376.4", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21376.4" } -} \ No newline at end of file +} From 5ecf7edc4e9460564371420f41e93603e42818bc Mon Sep 17 00:00:00 2001 From: Joey Robichaud Date: Tue, 27 Jul 2021 11:29:56 -0700 Subject: [PATCH 17/17] Restore NuGetRepack.Tasks so they are available during sourcebuild --- eng/Tools.props | 1 + eng/Versions.props | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Tools.props b/eng/Tools.props index 6be67695ad0a4..1579b5105586b 100644 --- a/eng/Tools.props +++ b/eng/Tools.props @@ -8,6 +8,7 @@ See https://github.com/dotnet/arcade/issues/7205 --> + \ No newline at end of file diff --git a/eng/Versions.props b/eng/Versions.props index 46c3df62b3c6c..636afa5f8d950 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -91,7 +91,6 @@ 1.1.0-beta2-20115-01 16.9.0-beta1.21055.5 1.5.0 - 6.0.0-beta.21319.2 5.0.0 5.0.0 3.13.8