From 73e25ad7b4ad073c47a826fafca7a246ec3d940a Mon Sep 17 00:00:00 2001 From: David Barbet Date: Thu, 26 May 2022 10:47:40 -0700 Subject: [PATCH 01/11] Don't throw in logging when the document path contains curly braces (#61524) --- .../Core/Def/LanguageClient/LogHubLspLogger.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/VisualStudio/Core/Def/LanguageClient/LogHubLspLogger.cs b/src/VisualStudio/Core/Def/LanguageClient/LogHubLspLogger.cs index 23852a3ee91d8..d35664d71111e 100644 --- a/src/VisualStudio/Core/Def/LanguageClient/LogHubLspLogger.cs +++ b/src/VisualStudio/Core/Def/LanguageClient/LogHubLspLogger.cs @@ -37,7 +37,13 @@ public void Dispose() } public void TraceInformation(string message) - => _traceSource.TraceInformation(message); + { + // Explicitly call TraceEvent here instead of TraceInformation. + // TraceInformation indirectly calls string.Format which throws if the message + // has unescaped curlies in it (can be a part of a URI for example). + // Since we have no need to call string.Format here, we don't. + _traceSource.TraceEvent(TraceEventType.Information, id: 0, message); + } public void TraceWarning(string message) => _traceSource.TraceEvent(TraceEventType.Warning, id: 0, message); From eb2fce99b60b9f097e5902e274b81de522ab19b0 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 30 Mar 2023 21:29:52 -0700 Subject: [PATCH 02/11] Add archived feeds (#67430) --- NuGet.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NuGet.config b/NuGet.config index 3cd2f6f4e6a03..75854b34c9de1 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,9 @@ + + From ee6b2ad1d144c71d6f85cd1cc16ffdedda4ebe4f Mon Sep 17 00:00:00 2001 From: David Barbet Date: Tue, 18 Apr 2023 11:36:21 -0700 Subject: [PATCH 03/11] Use newer version of ubuntu to fix build --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6da3cd93e18bb..4d4fccd9c4347 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -109,7 +109,7 @@ jobs: jobName: Build_Unix_Debug testArtifactName: Transport_Artifacts_Unix_Debug configuration: Debug - vmImageName: 'ubuntu-18.04' + vmImageName: 'ubuntu-20.04' - template: eng/pipelines/test-unix-job.yml parameters: From 0856b769476465fb6791217ff3ef4a42ae2f36c0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 19 Apr 2023 19:07:39 +0000 Subject: [PATCH 04/11] Update dependencies from https://github.com/dotnet/arcade build 20230411.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 8.0.0-beta.22524.5 -> To Version 7.0.0-beta.23211.2 --- eng/Version.Details.xml | 8 +-- eng/common/build.ps1 | 2 +- eng/common/cross/arm/tizen-fetch.sh | 6 +-- eng/common/cross/arm64/tizen-fetch.sh | 8 +-- eng/common/cross/armel/tizen-fetch.sh | 8 +-- eng/common/cross/armel/tizen/tizen-dotnet.ks | 50 ++++++++++++++++++ eng/common/cross/build-android-rootfs.sh | 4 +- eng/common/cross/build-rootfs.sh | 7 ++- eng/common/cross/toolchain.cmake | 7 +++ eng/common/cross/x86/tizen-fetch.sh | 6 +-- eng/common/generate-locproject.ps1 | 46 +++++++++++++++- eng/common/loc/P22DotNetHtmlLocalization.lss | Bin 0 -> 3810 bytes eng/common/native/init-compiler.sh | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/sdl/NuGet.config | 5 -- eng/common/sdl/sdl.ps1 | 1 - eng/common/templates/job/execute-sdl.yml | 6 +-- eng/common/templates/job/job.yml | 41 ++++++++++++-- eng/common/templates/job/onelocbuild.yml | 14 ++--- .../templates/job/publish-build-assets.yml | 1 + eng/common/templates/job/source-build.yml | 16 ++---- eng/common/templates/jobs/source-build.yml | 2 +- .../templates/post-build/post-build.yml | 2 +- eng/common/templates/steps/execute-sdl.yml | 6 +-- eng/common/templates/steps/generate-sbom.yml | 4 -- eng/common/templates/steps/source-build.yml | 8 +-- .../templates/variables/pool-providers.yml | 49 ++++++++++------- eng/common/tools.ps1 | 4 +- eng/common/tools.sh | 24 +++------ global.json | 8 +-- 30 files changed, 225 insertions(+), 122 deletions(-) create mode 100644 eng/common/cross/armel/tizen/tizen-dotnet.ks create mode 100644 eng/common/loc/P22DotNetHtmlLocalization.lss diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0cf9362e41a48..f4ab434f034a9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,18 +13,18 @@ - + https://github.com/dotnet/arcade - c5dd6a1da2e6d9b3423ab809fcda8af2927a408b + 7c5e5a782c67460b123c8e41d484ebcca8002c93 https://github.com/dotnet/roslyn c1d8c6f043bc80425c6828455eb57f8a404759c6 - + https://github.com/dotnet/arcade - c5dd6a1da2e6d9b3423ab809fcda8af2927a408b + 7c5e5a782c67460b123c8e41d484ebcca8002c93 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 33a6f2d0e2481..e0420a6427550 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -67,8 +67,8 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/arm/tizen-fetch.sh b/eng/common/cross/arm/tizen-fetch.sh index 0adb0f11ec329..eabd06c4afede 100644 --- a/eng/common/cross/arm/tizen-fetch.sh +++ b/eng/common/cross/arm/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen +TIZEN_URL=http://download.tizen.org/snapshots/tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -155,7 +155,7 @@ fetch_tizen_pkgs() } Inform "Initialize arm base" -fetch_tizen_pkgs_init standard Tizen-Base +fetch_tizen_pkgs_init standard base Inform "fetch common packages" fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard Tizen-Unified +fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh index 785bd85484e25..16d1301f21e4c 100644 --- a/eng/common/cross/arm64/tizen-fetch.sh +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen +TIZEN_URL=http://download.tizen.org/snapshots/tizen/ BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -154,8 +154,8 @@ fetch_tizen_pkgs() done } -Inform "Initialize arm64 base" -fetch_tizen_pkgs_init standard Tizen-Base +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base Inform "fetch common packages" fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard Tizen-Unified +fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index 3d8f177a34444..64f0187e5aaa2 100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen +TIZEN_URL=http://download.tizen.org/snapshots/tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -154,8 +154,8 @@ fetch_tizen_pkgs() done } -Inform "Initialize armel base" -fetch_tizen_pkgs_init standard Tizen-Base +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base Inform "fetch common packages" fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard Tizen-Unified +fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks new file mode 100644 index 0000000000000..506d455bd4ff8 --- /dev/null +++ b/eng/common/cross/armel/tizen/tizen-dotnet.ks @@ -0,0 +1,50 @@ +lang en_US.UTF-8 +keyboard us +timezone --utc Asia/Seoul + +part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime + +rootpw tizen +desktop --autologinuser=root +user --name root --groups audio,video --password 'tizen' + +repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no +repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no + +%packages +tar +gzip + +sed +grep +gawk +perl + +binutils +findutils +util-linux +lttng-ust +userspace-rcu +procps-ng +tzdata +ca-certificates + + +### Core FX +libicu +libunwind +iputils +zlib +krb5 +libcurl +libopenssl + +%end + +%post + +### Update /tmp privilege +chmod 777 /tmp +#################################### + +%end diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index f163fb9dae966..42516bbeebc3f 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\ +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do if [[ "$path" != "Filename:" ]]; then echo "Working on: $path" - wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir" + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" fi done diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index eddb4c380af01..5680980fa296e 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -391,9 +391,9 @@ elif [[ "$__CodeName" == "illumos" ]]; then --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --disable-libquadmath-support --disable-shared --enable-tls make -j "$JOBS" && make install && cd .. - BaseUrl=https://pkgsrc.smartos.org + BaseUrl=https://pkgsrc.joyent.com if [[ "$__UseMirror" == 1 ]]; then - BaseUrl=https://pkgsrc.smartos.skylime.net + BaseUrl=http://pkgsrc.smartos.skylime.net fi BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" echo "Downloading manifest" @@ -402,8 +402,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do echo "Installing '$package'" - # find last occurrence of package in listing and extract its name - package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" + package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')" echo "Resolved name '$package'" wget "$BaseUrl"/"$package".tgz ar -x "$package".tgz diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 561576be97c26..964610524760a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,5 +1,12 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) +# reset platform variables (e.g. cmake 3.25 sets LINUX=1) +unset(LINUX) +unset(FREEBSD) +unset(ILLUMOS) +unset(ANDROID) +unset(TIZEN) + set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) diff --git a/eng/common/cross/x86/tizen-fetch.sh b/eng/common/cross/x86/tizen-fetch.sh index cf8d1ce4a8e55..fa5f88b7d6c0d 100644 --- a/eng/common/cross/x86/tizen-fetch.sh +++ b/eng/common/cross/x86/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen +TIZEN_URL=http://download.tizen.org/snapshots/tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -155,7 +155,7 @@ fetch_tizen_pkgs() } Inform "Initialize i686 base" -fetch_tizen_pkgs_init standard Tizen-Base +fetch_tizen_pkgs_init standard base Inform "fetch common packages" fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard Tizen-Unified +fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index dbf2ab4ee7d17..bcb579e37a91b 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} $xlfFiles = @() @@ -99,8 +118,7 @@ $locJson = @{ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { + if ($_.FullName.Contains($exclusion)) { $continue = $false } } @@ -115,6 +133,30 @@ $locJson = @{ } } ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) } ) } diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 0000000000000000000000000000000000000000..6661fed566e49b0c206665bc21f135e06c9b89c4 GIT binary patch literal 3810 zcmd^CT~8BH5S?ce|HG9Bo&v?0;P_m6l= znU-wb=}VLT7UH{>{5H;0M4iLmRE8QeBRr|>&k=uV*L;heKY+dq>B$0^=0I}|x`)s{ zht4t9zaiEh5Fe>E-;zVT;c4G?v;9N0G=rWwo~*(Cs`OS6ZL`HL;vsL0J@I%$+0YvE zx{9ukK|FtFx1PlPD5M;6ZM>f;1BhCf?`8y6QH*?RT9T>XwF z#~m_N+i^UKE^j{e;KdNW`kH9Rbj{G8tDY}mafCgG+m3H`I@_PhrDmcIzxD&IX@s083kV|lLUE^0(h6wWRPN0QN1n^PU5eX8r6OZ*s^g)tt77#SZCB}znxye#U$Dtinr6lnVu z!LzA{A}0~no7p$thFGJAnI}oSW||9H=Bz}I7kD#2MLg7WfrlE5o9sQjePc>qmv+6iQCmdp(y}(Vr literal 0 HcmV?d00001 diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 41a26d802a93f..f13b74080edf9 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -71,7 +71,7 @@ if [[ -z "$CLR_CC" ]]; then # Set default versions if [[ -z "$majorVersion" ]]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [[ "$compiler" == "clang" ]]; then versions=( 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + if [[ "$compiler" == "clang" ]]; then versions=( 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi for version in "${versions[@]}"; do diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index c35087a06019e..e10a596879746 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.2.1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf51e..0c5451c11415f 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -7,11 +7,6 @@ - - - - - diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7d60..ac196e164a401 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -1,7 +1,6 @@ function Install-Gdn { param( - [Parameter(Mandatory=$true)] [string]$Path, # If omitted, install the latest version of Guardian, otherwise install that specific version. diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 7aabaa18017bf..7c164ac02f4da 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -34,7 +34,7 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool - condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) + condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot - name: AzDOProjectName @@ -60,9 +60,7 @@ jobs: - checkout: self clean: true - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml + - template: /eng/common/templates/post-build/setup-maestro-vars.yml - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 9f55d3f466600..e3ba9398016be 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -25,7 +25,6 @@ parameters: enablePublishTestResults: false enablePublishUsingPipelines: false disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' mergeTestResults: false testRunTitle: '' testResultsFormat: '' @@ -141,14 +140,11 @@ jobs: languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - task: ComponentGovernanceComponentDetection@0 continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -187,6 +183,24 @@ jobs: displayName: Publish logs continueOnError: true condition: always() + - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - task: PublishBuildArtifacts@1 @@ -220,11 +234,28 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - template: /eng/common/templates/steps/generate-sbom.yml parameters: PackageVersion: ${{ parameters.packageVersion}} BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index c2cabcf9e0686..60ab00c4de3ac 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -14,6 +14,7 @@ parameters: ReusePr: true UseLfLineEndings: true UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false LanguageSet: VS_Main_Languages LclSource: lclFilesInRepo LclPackageId: '' @@ -56,12 +57,13 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64 steps: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} - task: OneLocBuild@2 displayName: OneLocBuild diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 1f1b78f2d4598..c5fedd7f70ce6 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -34,6 +34,7 @@ jobs: - job: Asset_Registry_Publish dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: displayName: Publish Assets diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index e40bf35203b9e..8a3deef2b7274 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,20 +46,12 @@ jobs: # source-build builds run in Docker, including the default managed platform. # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: - # Main environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore-Public + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - # Servicing build environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore1ESPool-Svc-Internal + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51d9d..8dd2d355f22d2 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -14,7 +14,7 @@ parameters: # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-20220809204800-17a4aab' # Defines the platforms on which to run build jobs. One job is created for each platform, and the # object in this array is sent to the job template as 'platform'. If no platforms are specified, diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 91251d08973c6..c051f1b65e9cd 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -99,7 +99,7 @@ stages: jobs: - job: displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 9dd5709f66dc7..86cf578c43144 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -17,16 +17,14 @@ steps: - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 + . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1 $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian (Overridden) - ${{ if eq(parameters.overrideGuardianVersion, '') }}: - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 + . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1 $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa5d..4cea8c33187c9 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -2,14 +2,12 @@ # PackageName - The name of the package this SBOM represents. # PackageVersion - The version of the package this SBOM represents. # ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' sbomContinueOnError: true steps: @@ -36,8 +34,6 @@ steps: BuildDropPath: ${{ parameters.buildDropPath }} PackageVersion: ${{ parameters.packageVersion }} ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - task: PublishPipelineArtifact@1 displayName: Publish SBOM manifest diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index a97a185a367a9..4624885e3bfeb 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -73,11 +73,6 @@ steps: publishArgs='--publish' fi - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ @@ -87,8 +82,7 @@ steps: $targetRidArgs \ $runtimeOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName + /p:ArcadeBuildFromSource=true displayName: Build # Upload build logs for diagnosis. diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index a7b943c2fa4c1..99c80212bac19 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,22 +1,22 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. # Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS) # Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services # team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. # How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. +# If we find alternate naming conventions in broad usage these can be added to the condition below. # # First, import the template in an arcade-ified repo to pick up the variables, e.g.: # # variables: -# - template: eng/common/templates/variables/pool-providers.yml +# - template: /eng/common/templates/variables/pool-providers.yml # # ... then anywhere specifying the pool provider use the runtime variables, # $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: @@ -26,23 +26,32 @@ # demands: ImageOverride -equals windows.vs2019.amd64 variables: -# Coalesce the target and source branches so we know when a PR targets a release branch -# If these variables are somehow missing, fall back to main (tends to have more capacity) -- name: BranchNameForPoolSelection - value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) -# Any new -Svc alternative pools should have variables added here to allow for splitting work - -# Main branch pools -- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}: + # Any new -Svc alternative pools should have variables added here to allow for splitting work - name: DncEngPublicBuildPool - value: NetCore-Public - - name: DncEngInternalBuildPool - value: NetCore1ESPool-Internal + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] -# Release branch pools -- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}: - - name: DncEngPublicBuildPool - value: NetCore-Svc-Public - name: DncEngInternalBuildPool - value: NetCore1ESPool-Svc-Internal \ No newline at end of file + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index aba6308ad313c..021555cf3381a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -365,8 +365,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.2.1&view=overview - $defaultXCopyMSBuildVersion = '17.2.1' + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview + $defaultXCopyMSBuildVersion = '17.4.1' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6586eab458e55..e9a7ed9af6794 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -417,11 +417,12 @@ function MSBuild { Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" - # https://github.com/dotnet/arcade/issues/11369 - disable new MSBuild server feature on linux - # This feature is new and can result in build failures from connection timeout errors. - export DOTNET_CLI_DO_NOT_USE_MSBUILD_SERVER=1 - Write-PipelineSetVariable -name "DOTNET_CLI_DO_NOT_USE_MSBUILD_SERVER" -value "1" - + export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true + export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6 + export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000 + Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000" fi local toolset_dir="${_InitializeToolset%/*}" @@ -492,17 +493,6 @@ function MSBuild-Core { RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } -function GetDarc { - darc_path="$temp_dir/darc" - version="$1" - - if [[ -n "$version" ]]; then - version="--darcversion $version" - fi - - "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version -} - ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` @@ -521,7 +511,7 @@ global_json_file="${repo_root}global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false if command -v jq &> /dev/null; then - if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then + if jq -e '.tools | has("runtimes")' "$global_json_file" &> /dev/null; then global_json_has_runtimes=true fi elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then diff --git a/global.json b/global.json index cdcd9c978161e..4a1c2ad8b856a 100644 --- a/global.json +++ b/global.json @@ -1,11 +1,11 @@ { "sdk": { - "version": "7.0.100-rc.1.22431.12", + "version": "7.0.105", "allowPrerelease": true, "rollForward": "latestPatch" }, "tools": { - "dotnet": "7.0.100-rc.1.22431.12", + "dotnet": "7.0.105", "runtimes": { "dotnet": [ "3.1.0" @@ -17,7 +17,7 @@ "xcopy-msbuild": "17.2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22524.5", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.22524.5" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.23211.2", + "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.23211.2" } } From c2699446015b9dcd02aa61090fa5e6638efc4da2 Mon Sep 17 00:00:00 2001 From: Jared Parsons Date: Thu, 26 Jan 2023 15:08:25 -0800 Subject: [PATCH 05/11] Don't overwrite binary logs (#66559) * Don't overwrite binary logs Several of our legs were calling `build.ps1 -binaryLog` in a row and that re-uses the binary log file name which causes an overwrite. This change allows us to specify the name and adds a warning if we do accidentally overwrite in the future. * enable trace for diff issue * Work around NuGet static graph restore bug https://github.com/NuGet/Home/issues/12373 --- azure-pipelines.yml | 2 +- eng/build.ps1 | 30 ++++++++++++++++++++++++++---- eng/common/tools.ps1 | 2 ++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d1a522d501568..79b992f97e7fe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -211,7 +211,7 @@ jobs: displayName: Restore inputs: filePath: eng/build.ps1 - arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog + arguments: -configuration Debug -prepareMachine -ci -restore -binaryLogName Restore.binlog - powershell: .\eng\test-rebuild.ps1 -ci -configuration Release displayName: Run BuildValidator diff --git a/eng/build.ps1 b/eng/build.ps1 index 3837556fef550..5b8d533b5ce48 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -34,6 +34,7 @@ param ( [switch]$bootstrap, [string]$bootstrapConfiguration = "Release", [switch][Alias('bl')]$binaryLog, + [string]$binaryLogName = "", [switch]$buildServerLog, [switch]$ci, [switch]$collectDumps, @@ -78,6 +79,7 @@ function Print-Usage() { Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" Write-Host " -deployExtensions Deploy built vsixes (short: -d)" Write-Host " -binaryLog Create MSBuild binary log (short: -bl)" + Write-Host " -binaryLogName Name of the binary log (default Build.binlog)" Write-Host " -buildServerLog Create Roslyn build server log" Write-Host "" Write-Host "Actions:" @@ -165,6 +167,10 @@ function Process-Arguments() { $script:applyOptimizationData = $false } + if ($binaryLogName -ne "") { + $script:binaryLog = $true + } + if ($ci) { $script:binaryLog = $true if ($bootstrap) { @@ -172,6 +178,10 @@ function Process-Arguments() { } } + if ($binaryLog -and ($binaryLogName -eq "")) { + $script:binaryLogName = "Build.binlog" + } + $anyUnit = $testDesktop -or $testCoreClr if ($anyUnit -and $testVsi) { Write-Host "Cannot combine unit and VSI testing" @@ -212,7 +222,14 @@ function BuildSolution() { Write-Host "$($solution):" - $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $bl = "" + if ($binaryLog) { + $binaryLogPath = Join-Path $LogDir $binaryLogName + $bl = "/bl:" + $binaryLogPath + if ($ci -and (Test-Path $binaryLogPath)) { + Write-LogIssue -Type "warning" -Message "Overwriting binary log file $($binaryLogPath)" + } + } if ($buildServerLog) { ${env:ROSLYNCOMMANDLINELOGFILE} = Join-Path $LogDir "Build.Server.log" @@ -239,6 +256,11 @@ function BuildSolution() { $generateDocumentationFile = if ($skipDocumentation) { "/p:GenerateDocumentationFile=false" } else { "" } $roslynUseHardLinks = if ($ci) { "/p:ROSLYNUSEHARDLINKS=true" } else { "" } + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # in our CI builds + # https://github.com/NuGet/Home/issues/12373 + $restoreUseStaticGraphEvaluation = if ($ci) { $false } else { $true } + try { MSBuild $toolsetBuildProj ` $bl ` @@ -258,7 +280,7 @@ function BuildSolution() { /p:TreatWarningsAsErrors=$warnAsError ` /p:EnableNgenOptimization=$applyOptimizationData ` /p:IbcOptimizationDataDir=$ibcDir ` - /p:RestoreUseStaticGraphEvaluation=true ` + /p:RestoreUseStaticGraphEvaluation=$restoreUseStaticGraphEvaluation ` /p:VisualStudioIbcDrop=$ibcDropName ` /p:VisualStudioDropAccessToken=$officialVisualStudioDropAccessToken ` $suppressExtensionDeployment ` @@ -690,7 +712,7 @@ try { catch { if ($ci) { - echo "##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Build failed" + Write-LogIssue -Type "error" -Message "(NETCORE_ENGINEERING_TELEMETRY=Build) Build failed" } throw $_ } @@ -708,7 +730,7 @@ try { catch { if ($ci) { - echo "##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Test) Tests failed" + Write-LogIssue -Type "error" -Message "(NETCORE_ENGINEERING_TELEMETRY=Test) Tests failed" } throw $_ } diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 021555cf3381a..a688564404c62 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -802,6 +802,8 @@ function MSBuild-Core() { $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + Write-Host "Exec $($buildTool.Path) and $cmdArgs" + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { From d32f3130239e920b4b3fcd0f21f6510329043d4e Mon Sep 17 00:00:00 2001 From: Jared Parsons Date: Fri, 3 Feb 2023 13:34:31 -0800 Subject: [PATCH 06/11] Don't overwrite binary logs in CI (#66683) * Don't overwrite binary logs in CI This will make it an error whenever we overwrite a binary log during CI. Doing so means we've lost one of our main tools for investigating build failures. Turning this into an error so we don't make the mistake again. * actually fail * fixup --- eng/build.ps1 | 3 ++- eng/pipelines/build-windows-job.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/build.ps1 b/eng/build.ps1 index 5b8d533b5ce48..b64c3646a478d 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -227,7 +227,8 @@ function BuildSolution() { $binaryLogPath = Join-Path $LogDir $binaryLogName $bl = "/bl:" + $binaryLogPath if ($ci -and (Test-Path $binaryLogPath)) { - Write-LogIssue -Type "warning" -Message "Overwriting binary log file $($binaryLogPath)" + Write-LogIssue -Type "error" -Message "Overwriting binary log file $($binaryLogPath)" + throw "Overwriting binary log files" } } diff --git a/eng/pipelines/build-windows-job.yml b/eng/pipelines/build-windows-job.yml index eaba27ebaf7b1..4aba1ec632a75 100644 --- a/eng/pipelines/build-windows-job.yml +++ b/eng/pipelines/build-windows-job.yml @@ -37,13 +37,13 @@ jobs: displayName: Restore inputs: filePath: eng/build.ps1 - arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -restore -binaryLog + arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -restore -binaryLogName Restore.binlog - task: PowerShell@2 displayName: Build inputs: filePath: eng/build.ps1 - arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -build -publish -binaryLog -skipDocumentation ${{ parameters.buildArguments }} + arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -build -publish -binaryLogName Build.binlog -skipDocumentation ${{ parameters.buildArguments }} - task: PowerShell@2 displayName: Prepare Unit Tests From bd47e181374c754be2858d4acb9278aa3a82e1f1 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Mon, 30 Jan 2023 09:47:08 -0800 Subject: [PATCH 07/11] Disable static restore in more locations (#66567) * Disable static restore in more locations * Run compilers CI when build scripts change * Include compiler generators in compilers code change check * Remove separate restore in bootstrap build --- eng/build-utils.ps1 | 4 ++++ eng/test-determinism.ps1 | 7 ++++++- eng/test-rebuild.ps1 | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index 02a535cd42c40..42bd1a09730a5 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -301,6 +301,10 @@ function Run-MSBuild([string]$projectFilePath, [string]$buildArgs = "", [string] if ($ci) { $args += " /p:ContinuousIntegrationBuild=true" + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # in our CI builds + # https://github.com/NuGet/Home/issues/12373 + $args += " /p:RestoreUseStaticGraphEvaluation=false" } if ($bootstrapDir -ne "") { diff --git a/eng/test-determinism.ps1 b/eng/test-determinism.ps1 index e7c36de72ba8d..dc5e3e59dbfd0 100644 --- a/eng/test-determinism.ps1 +++ b/eng/test-determinism.ps1 @@ -48,6 +48,11 @@ function Run-Build([string]$rootDir, [string]$logFileName) { Stop-Processes + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # in our CI builds + # https://github.com/NuGet/Home/issues/12373 + $restoreUseStaticGraphEvaluation = if ($ci) { $false } else { $true } + Write-Host "Building $solution using $bootstrapDir" MSBuild $toolsetBuildProj ` /p:Projects=$solution ` @@ -61,7 +66,7 @@ function Run-Build([string]$rootDir, [string]$logFileName) { /p:BootstrapBuildPath=$bootstrapDir ` /p:RunAnalyzers=false ` /p:RunAnalyzersDuringBuild=false ` - /p:RestoreUseStaticGraphEvaluation=true ` + /p:RestoreUseStaticGraphEvaluation=$restoreUseStaticGraphEvaluation ` /bl:$logFilePath Stop-Processes diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 8cd2f7adc0731..1ebf48a0e3fcb 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -33,7 +33,7 @@ try { if (-not $noBuild) { Write-Host "Building Roslyn" - Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -build -bootstrap -ci:$ci -useGlobalNuGetCache:$useGlobalNuGetCache -configuration:$configuration -pack -binaryLog } + Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -bootstrap -ci:$ci -useGlobalNuGetCache:$useGlobalNuGetCache -configuration:$configuration -pack -binaryLog } } $dotnetInstallDir = (InitializeDotNetCli -install:$true) From e337be8987c445817d5f579573af05cb0e99ffcd Mon Sep 17 00:00:00 2001 From: Gen Lu Date: Tue, 21 Mar 2023 10:56:58 -0700 Subject: [PATCH 08/11] Merge pull request #67381 from genlu/Fix17.5Ci Fix 17.5 CI build --- src/Tools/BuildBoss/BuildBoss.csproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Tools/BuildBoss/BuildBoss.csproj b/src/Tools/BuildBoss/BuildBoss.csproj index b3e129fb7769b..ef3d0eb4fe7d1 100644 --- a/src/Tools/BuildBoss/BuildBoss.csproj +++ b/src/Tools/BuildBoss/BuildBoss.csproj @@ -7,10 +7,18 @@ false false false + + + 2.1.787 + From 1f28c5956fd17fbc2939948ed266c08e1f20baf9 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Thu, 20 Apr 2023 17:47:26 -0700 Subject: [PATCH 09/11] Disable tests on servicing branch to get building --- src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs | 10 +++++----- .../VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb | 2 +- src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs index 4ec1a41d83fbc..106852faf2128 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs @@ -16,7 +16,7 @@ namespace Microsoft.CodeAnalysis.CSharp.UnitTests.PDB { public class PDBConstantTests : CSharpTestBase { - [Fact] + [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] public void StringsWithSurrogateChar() { var source = @" @@ -84,7 +84,7 @@ public static void Main() } [WorkItem(546862, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/546862")] - [Fact] + [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] public void InvalidUnicodeString() { var source = @" @@ -145,7 +145,7 @@ public static void Main() ", format: DebugInformationFormat.PortablePdb); } - [Fact] + [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] public void AllTypes() { var source = @" @@ -466,7 +466,7 @@ IEnumerable M() "); } - [Fact] + [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] [WorkItem(33564, "https://github.com/dotnet/roslyn/issues/33564")] public void LocalConstantsTypes() { @@ -648,7 +648,7 @@ this is a string constant that is too long to fit into the PDB } [WorkItem(178988, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/178988")] - [Fact] + [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] public void StringWithNulCharacter_MaxSupportedLength() { const int length = 2031; diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb index 8c223892be73d..69ee5839bde31 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb @@ -144,7 +144,7 @@ class C End Sub #End If - + Public Sub TestLocalConstantsTypes() diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb index ff467108cbbff..2446298d932b4 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb @@ -4395,7 +4395,7 @@ End Class ) End Sub - + Public Sub Constant_AllTypes() Dim source = From 2151c15091320176ef712c3b4f9bd92bb65df73b Mon Sep 17 00:00:00 2001 From: David Barbet Date: Fri, 21 Apr 2023 12:46:42 -0700 Subject: [PATCH 10/11] Remove restore on rebuild step --- azure-pipelines.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 79b992f97e7fe..6eb5960a9546e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -207,12 +207,6 @@ jobs: steps: - template: eng/pipelines/checkout-windows-task.yml - - task: PowerShell@2 - displayName: Restore - inputs: - filePath: eng/build.ps1 - arguments: -configuration Debug -prepareMachine -ci -restore -binaryLogName Restore.binlog - - powershell: .\eng\test-rebuild.ps1 -ci -configuration Release displayName: Run BuildValidator From fad8e607211e2e42b8972ecf3c37f41eadf2cbf0 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Fri, 21 Apr 2023 14:03:59 -0700 Subject: [PATCH 11/11] Revert "Disable tests on servicing branch to get building" This reverts commit 1f28c5956fd17fbc2939948ed266c08e1f20baf9. --- src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs | 10 +++++----- .../VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb | 2 +- src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs index 106852faf2128..4ec1a41d83fbc 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs @@ -16,7 +16,7 @@ namespace Microsoft.CodeAnalysis.CSharp.UnitTests.PDB { public class PDBConstantTests : CSharpTestBase { - [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] + [Fact] public void StringsWithSurrogateChar() { var source = @" @@ -84,7 +84,7 @@ public static void Main() } [WorkItem(546862, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/546862")] - [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] + [Fact] public void InvalidUnicodeString() { var source = @" @@ -145,7 +145,7 @@ public static void Main() ", format: DebugInformationFormat.PortablePdb); } - [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] + [Fact] public void AllTypes() { var source = @" @@ -466,7 +466,7 @@ IEnumerable M() "); } - [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] + [Fact] [WorkItem(33564, "https://github.com/dotnet/roslyn/issues/33564")] public void LocalConstantsTypes() { @@ -648,7 +648,7 @@ this is a string constant that is too long to fit into the PDB } [WorkItem(178988, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/178988")] - [Fact(Skip = "https://github.com/dotnet/roslyn/issues/67908")] + [Fact] public void StringWithNulCharacter_MaxSupportedLength() { const int length = 2031; diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb index 69ee5839bde31..8c223892be73d 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBConstLocalTests.vb @@ -144,7 +144,7 @@ class C End Sub #End If - + Public Sub TestLocalConstantsTypes() diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb index 0612980c96691..8039de658d6eb 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb @@ -4397,7 +4397,7 @@ End Class ) End Sub - + Public Sub Constant_AllTypes() Dim source =