diff --git a/NuGet.config b/NuGet.config index 1aafa4f960290..045dcdbc3cf46 100644 --- a/NuGet.config +++ b/NuGet.config @@ -12,7 +12,9 @@ + + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4564073db4add..4583c59c273fe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -222,7 +222,7 @@ stages: displayName: Restore inputs: filePath: eng/build.ps1 - arguments: -configuration Release -prepareMachine -ci -restore -binaryLog + arguments: -configuration Release -prepareMachine -ci -restore -binaryLog -binaryLogName Restore.binlog - task: PowerShell@2 displayName: Build diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b2b65ed7cece1..71f4641fe446d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,22 +13,18 @@ - + https://github.com/dotnet/arcade - 5a5ae41c42e475e280342b668e668ff34539e36d + df8799988af6503cbcd9544713d30732328c8c57 https://github.com/dotnet/roslyn 5d10d428050c0d6afef30a072c4ae68776621877 - + https://github.com/dotnet/arcade - 5a5ae41c42e475e280342b668e668ff34539e36d - - - https://github.com/dotnet/roslyn-analyzers - fad2bc69091f4282606ba4d3223f7efd94f2fbc8 + df8799988af6503cbcd9544713d30732328c8c57 diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index 2b7db4c5a709d..6d56827978aa2 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -301,6 +301,10 @@ function Run-MSBuild([string]$projectFilePath, [string]$buildArgs = "", [string] if ($ci) { $args += " /p:ContinuousIntegrationBuild=true" + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # in our CI builds + # https://github.com/NuGet/Home/issues/12373 + $args += " /p:RestoreUseStaticGraphEvaluation=false" } if ($bootstrapDir -ne "") { diff --git a/eng/build.ps1 b/eng/build.ps1 index 8b0e7e8e10a38..c83251fb06d1f 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -34,6 +34,7 @@ param ( [switch]$bootstrap, [string]$bootstrapConfiguration = "Release", [switch][Alias('bl')]$binaryLog, + [string]$binaryLogName = "", [switch]$buildServerLog, [switch]$ci, [switch]$collectDumps, @@ -79,6 +80,7 @@ function Print-Usage() { Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" Write-Host " -deployExtensions Deploy built vsixes (short: -d)" Write-Host " -binaryLog Create MSBuild binary log (short: -bl)" + Write-Host " -binaryLogName Name of the binary log (default Build.binlog)" Write-Host " -buildServerLog Create Roslyn build server log" Write-Host "" Write-Host "Actions:" @@ -166,6 +168,10 @@ function Process-Arguments() { $script:applyOptimizationData = $false } + if ($binaryLogName -ne "") { + $script:binaryLog = $true + } + if ($ci) { $script:binaryLog = $true if ($bootstrap) { @@ -173,6 +179,10 @@ function Process-Arguments() { } } + if ($binaryLog -and ($binaryLogName -eq "")) { + $script:binaryLogName = "Build.binlog" + } + $anyUnit = $testDesktop -or $testCoreClr if ($anyUnit -and $testVsi) { Write-Host "Cannot combine unit and VSI testing" @@ -213,7 +223,15 @@ function BuildSolution() { Write-Host "$($solution):" - $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $bl = "" + if ($binaryLog) { + $binaryLogPath = Join-Path $LogDir $binaryLogName + $bl = "/bl:" + $binaryLogPath + if ($ci -and (Test-Path $binaryLogPath)) { + Write-LogIssue -Type "error" -Message "Overwriting binary log file $($binaryLogPath)" + throw "Overwriting binary log files" + } + } if ($buildServerLog) { ${env:ROSLYNCOMMANDLINELOGFILE} = Join-Path $LogDir "Build.Server.log" @@ -240,11 +258,11 @@ function BuildSolution() { $generateDocumentationFile = if ($skipDocumentation) { "/p:GenerateDocumentationFile=false" } else { "" } $roslynUseHardLinks = if ($ci) { "/p:ROSLYNUSEHARDLINKS=true" } else { "" } - # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue # in our CI builds # https://github.com/NuGet/Home/issues/12373 $restoreUseStaticGraphEvaluation = if ($ci) { $false } else { $true } - + try { MSBuild $toolsetBuildProj ` $bl ` @@ -739,7 +757,7 @@ try { catch { if ($ci) { - echo "##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Build failed" + Write-LogIssue -Type "error" -Message "(NETCORE_ENGINEERING_TELEMETRY=Build) Build failed" } throw $_ } @@ -757,7 +775,7 @@ try { catch { if ($ci) { - echo "##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Test) Tests failed" + Write-LogIssue -Type "error" -Message "(NETCORE_ENGINEERING_TELEMETRY=Test) Tests failed" } throw $_ } diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 33a6f2d0e2481..e0420a6427550 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -67,8 +67,8 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/arm/tizen-fetch.sh b/eng/common/cross/arm/tizen-fetch.sh new file mode 100644 index 0000000000000..eabd06c4afede --- /dev/null +++ b/eng/common/cross/arm/tizen-fetch.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -e + +if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh new file mode 100644 index 0000000000000..16d1301f21e4c --- /dev/null +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -e + +if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen/ +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh new file mode 100755 index 0000000000000..64f0187e5aaa2 --- /dev/null +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -e + +if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks new file mode 100644 index 0000000000000..506d455bd4ff8 --- /dev/null +++ b/eng/common/cross/armel/tizen/tizen-dotnet.ks @@ -0,0 +1,50 @@ +lang en_US.UTF-8 +keyboard us +timezone --utc Asia/Seoul + +part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime + +rootpw tizen +desktop --autologinuser=root +user --name root --groups audio,video --password 'tizen' + +repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no +repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no + +%packages +tar +gzip + +sed +grep +gawk +perl + +binutils +findutils +util-linux +lttng-ust +userspace-rcu +procps-ng +tzdata +ca-certificates + + +### Core FX +libicu +libunwind +iputils +zlib +krb5 +libcurl +libopenssl + +%end + +%post + +### Update /tmp privilege +chmod 777 /tmp +#################################### + +%end diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index f163fb9dae966..42516bbeebc3f 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\ +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do if [[ "$path" != "Filename:" ]]; then echo "Working on: $path" - wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir" + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" fi done diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index b04d3b3806575..4d6782d99cb75 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -391,9 +391,9 @@ elif [[ "$__CodeName" == "illumos" ]]; then --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --disable-libquadmath-support --disable-shared --enable-tls make -j "$JOBS" && make install && cd .. - BaseUrl=https://pkgsrc.smartos.org + BaseUrl=https://pkgsrc.joyent.com if [[ "$__UseMirror" == 1 ]]; then - BaseUrl=https://pkgsrc.smartos.skylime.net + BaseUrl=http://pkgsrc.smartos.skylime.net fi BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" echo "Downloading manifest" @@ -402,8 +402,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do echo "Installing '$package'" - # find last occurrence of package in listing and extract its name - package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" + package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')" echo "Resolved name '$package'" wget "$BaseUrl"/"$package".tgz ar -x "$package".tgz diff --git a/eng/common/cross/x86/tizen-fetch.sh b/eng/common/cross/x86/tizen-fetch.sh new file mode 100644 index 0000000000000..fa5f88b7d6c0d --- /dev/null +++ b/eng/common/cross/x86/tizen-fetch.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -e + +if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize i686 base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs i686 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release + diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index dbf2ab4ee7d17..bcb579e37a91b 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} $xlfFiles = @() @@ -99,8 +118,7 @@ $locJson = @{ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { + if ($_.FullName.Contains($exclusion)) { $continue = $false } } @@ -115,6 +133,30 @@ $locJson = @{ } } ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) } ) } diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 0000000000000..6661fed566e49 Binary files /dev/null and b/eng/common/loc/P22DotNetHtmlLocalization.lss differ diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index c670cb7968921..eb9abaa685130 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,8 +63,8 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi + if [[ "$compiler" == "clang" ]]; then versions=( 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf51e..0c5451c11415f 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -7,11 +7,6 @@ - - - - - diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7d60..ac196e164a401 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -1,7 +1,6 @@ function Install-Gdn { param( - [Parameter(Mandatory=$true)] [string]$Path, # If omitted, install the latest version of Guardian, otherwise install that specific version. diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 7aabaa18017bf..7c164ac02f4da 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -34,7 +34,7 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool - condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) + condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot - name: AzDOProjectName @@ -60,9 +60,7 @@ jobs: - checkout: self clean: true - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml + - template: /eng/common/templates/post-build/setup-maestro-vars.yml - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 0549e4c71ce3e..4fa3d193dba14 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,9 +24,7 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' + disableComponentGovernance: '' mergeTestResults: false testRunTitle: '' testResultsFormat: '' @@ -142,14 +140,15 @@ jobs: languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + - template: /eng/common/templates/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(contains(variables['Build.SourceBranch'], 'internal/release'), eq(variables['Build.SourceBranch'], 'main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -188,6 +187,24 @@ jobs: displayName: Publish logs continueOnError: true condition: always() + - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - task: PublishBuildArtifacts@1 @@ -221,13 +238,30 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - template: /eng/common/templates/steps/generate-sbom.yml parameters: PackageVersion: ${{ parameters.packageVersion}} BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index e40bf35203b9e..8a3deef2b7274 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,20 +46,12 @@ jobs: # source-build builds run in Docker, including the default managed platform. # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: - # Main environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore-Public + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - # Servicing build environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore1ESPool-Svc-Internal + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51d9d..8dd2d355f22d2 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -14,7 +14,7 @@ parameters: # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-20220809204800-17a4aab' # Defines the platforms on which to run build jobs. One job is created for each platform, and the # object in this array is sent to the job template as 'platform'. If no platforms are specified, diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d78198..05d55c999e497 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -99,7 +99,7 @@ stages: jobs: - job: displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml new file mode 100644 index 0000000000000..babc2757d8d12 --- /dev/null +++ b/eng/common/templates/steps/component-governance.yml @@ -0,0 +1,10 @@ +parameters: + disableComponentGovernance: false + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 9dd5709f66dc7..86cf578c43144 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -17,16 +17,14 @@ steps: - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 + . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1 $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian (Overridden) - ${{ if eq(parameters.overrideGuardianVersion, '') }}: - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 + . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1 $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa5d..4cea8c33187c9 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -2,14 +2,12 @@ # PackageName - The name of the package this SBOM represents. # PackageVersion - The version of the package this SBOM represents. # ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' sbomContinueOnError: true steps: @@ -36,8 +34,6 @@ steps: BuildDropPath: ${{ parameters.buildDropPath }} PackageVersion: ${{ parameters.packageVersion }} ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - task: PublishPipelineArtifact@1 displayName: Publish SBOM manifest diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index a97a185a367a9..4624885e3bfeb 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -73,11 +73,6 @@ steps: publishArgs='--publish' fi - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ @@ -87,8 +82,7 @@ steps: $targetRidArgs \ $runtimeOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName + /p:ArcadeBuildFromSource=true displayName: Build # Upload build logs for diagnosis. diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index a7b943c2fa4c1..99c80212bac19 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,22 +1,22 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. # Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS) # Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services # team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. # How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. +# If we find alternate naming conventions in broad usage these can be added to the condition below. # # First, import the template in an arcade-ified repo to pick up the variables, e.g.: # # variables: -# - template: eng/common/templates/variables/pool-providers.yml +# - template: /eng/common/templates/variables/pool-providers.yml # # ... then anywhere specifying the pool provider use the runtime variables, # $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: @@ -26,23 +26,32 @@ # demands: ImageOverride -equals windows.vs2019.amd64 variables: -# Coalesce the target and source branches so we know when a PR targets a release branch -# If these variables are somehow missing, fall back to main (tends to have more capacity) -- name: BranchNameForPoolSelection - value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) -# Any new -Svc alternative pools should have variables added here to allow for splitting work - -# Main branch pools -- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}: + # Any new -Svc alternative pools should have variables added here to allow for splitting work - name: DncEngPublicBuildPool - value: NetCore-Public - - name: DncEngInternalBuildPool - value: NetCore1ESPool-Internal + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] -# Release branch pools -- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}: - - name: DncEngPublicBuildPool - value: NetCore-Svc-Public - name: DncEngInternalBuildPool - value: NetCore1ESPool-Svc-Internal \ No newline at end of file + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 2f27d7453cd49..b04d2a7ee9195 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -416,6 +416,13 @@ function MSBuild { export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" + + export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true + export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6 + export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000 + Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000" fi local toolset_dir="${_InitializeToolset%/*}" @@ -488,17 +495,6 @@ function MSBuild-Core { RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } -function GetDarc { - darc_path="$temp_dir/darc" - version="$1" - - if [[ -n "$version" ]]; then - version="--darcversion $version" - fi - - "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version -} - ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` diff --git a/eng/pipelines/build-windows-job.yml b/eng/pipelines/build-windows-job.yml index 8d490a209fc38..7315b19e92e22 100644 --- a/eng/pipelines/build-windows-job.yml +++ b/eng/pipelines/build-windows-job.yml @@ -42,13 +42,13 @@ jobs: displayName: Restore inputs: filePath: eng/build.ps1 - arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -restore -binaryLog ${{ parameters.restoreArguments }} + arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -restore -binaryLogName Restore.binlog ${{ parameters.restoreArguments }} - task: PowerShell@2 displayName: Build inputs: filePath: eng/build.ps1 - arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -build -binaryLog -skipDocumentation ${{ parameters.buildArguments }} + arguments: -configuration ${{ parameters.configuration }} -prepareMachine -ci -build -publish -binaryLogName Build.binlog -skipDocumentation ${{ parameters.buildArguments }} - task: PowerShell@2 displayName: Prepare Unit Tests diff --git a/eng/test-determinism.ps1 b/eng/test-determinism.ps1 index c98e405ff4fdf..6be79b1e2377e 100644 --- a/eng/test-determinism.ps1 +++ b/eng/test-determinism.ps1 @@ -52,6 +52,11 @@ function Run-Build([string]$rootDir, [string]$logFileName) { Stop-Processes + # Temporarily disable RestoreUseStaticGraphEvaluation to work around this NuGet issue + # in our CI builds + # https://github.com/NuGet/Home/issues/12373 + $restoreUseStaticGraphEvaluation = if ($ci) { $false } else { $true } + Write-Host "Building $solution using $bootstrapDir" MSBuild $toolsetBuildProj ` /p:Projects=$solution ` @@ -65,7 +70,7 @@ function Run-Build([string]$rootDir, [string]$logFileName) { /p:BootstrapBuildPath=$bootstrapDir ` /p:RunAnalyzers=false ` /p:RunAnalyzersDuringBuild=false ` - /p:RestoreUseStaticGraphEvaluation=true ` + /p:RestoreUseStaticGraphEvaluation=$restoreUseStaticGraphEvaluation ` /bl:$logFilePath Stop-Processes diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index cf7092ddd6380..e5c5cfe17f813 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -33,7 +33,7 @@ try { if (-not $noBuild) { Write-Host "Building Roslyn" - Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -build -bootstrap -ci:$ci -useGlobalNuGetCache:$useGlobalNuGetCache -configuration:$configuration -pack -binaryLog } + Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -bootstrap -ci:$ci -useGlobalNuGetCache:$useGlobalNuGetCache -configuration:$configuration -pack -binaryLog } } Subst-TempDir diff --git a/global.json b/global.json index 09e371dbd5cf6..ba30c9fe13004 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,23 @@ { "sdk": { - "version": "7.0.100", - "allowPrerelease": false, + "version": "7.0.105", + "allowPrerelease": true, "rollForward": "latestPatch" }, "tools": { - "dotnet": "7.0.100", + "dotnet": "7.0.105", + "runtimes": { + "dotnet": [ + "3.1.0" + ] + }, "vs": { "version": "17.4.1" }, "xcopy-msbuild": "17.4.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23052.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23052.2" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.23228.7", + "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.23228.7" } } diff --git a/src/VisualStudio/CSharp/Test/Microsoft.VisualStudio.LanguageServices.CSharp.UnitTests.csproj b/src/VisualStudio/CSharp/Test/Microsoft.VisualStudio.LanguageServices.CSharp.UnitTests.csproj index f665c0e02b574..751821d084341 100644 --- a/src/VisualStudio/CSharp/Test/Microsoft.VisualStudio.LanguageServices.CSharp.UnitTests.csproj +++ b/src/VisualStudio/CSharp/Test/Microsoft.VisualStudio.LanguageServices.CSharp.UnitTests.csproj @@ -46,7 +46,6 @@ - diff --git a/src/VisualStudio/Core/Test/Microsoft.VisualStudio.LanguageServices.UnitTests.vbproj b/src/VisualStudio/Core/Test/Microsoft.VisualStudio.LanguageServices.UnitTests.vbproj index e39c6cfe5643f..94ce52743db98 100644 --- a/src/VisualStudio/Core/Test/Microsoft.VisualStudio.LanguageServices.UnitTests.vbproj +++ b/src/VisualStudio/Core/Test/Microsoft.VisualStudio.LanguageServices.UnitTests.vbproj @@ -41,7 +41,6 @@ - diff --git a/src/VisualStudio/IntegrationTest/New.IntegrationTests/InProcess/ShellInProcess.cs b/src/VisualStudio/IntegrationTest/New.IntegrationTests/InProcess/ShellInProcess.cs index 825066443dccd..51975ac6fb971 100644 --- a/src/VisualStudio/IntegrationTest/New.IntegrationTests/InProcess/ShellInProcess.cs +++ b/src/VisualStudio/IntegrationTest/New.IntegrationTests/InProcess/ShellInProcess.cs @@ -16,6 +16,7 @@ using Microsoft.VisualStudio.OLE.Interop; using Microsoft.VisualStudio.Shell; using Microsoft.VisualStudio.Shell.Interop; +using Microsoft.VisualStudio.Text; using Microsoft.VisualStudio.Threading; using Microsoft.VisualStudio.Utilities; using Xunit; diff --git a/src/VisualStudio/TestUtilities2/Microsoft.VisualStudio.LanguageServices.Test.Utilities2.vbproj b/src/VisualStudio/TestUtilities2/Microsoft.VisualStudio.LanguageServices.Test.Utilities2.vbproj index fe402789f1967..f5e64d23ff76f 100644 --- a/src/VisualStudio/TestUtilities2/Microsoft.VisualStudio.LanguageServices.Test.Utilities2.vbproj +++ b/src/VisualStudio/TestUtilities2/Microsoft.VisualStudio.LanguageServices.Test.Utilities2.vbproj @@ -14,7 +14,6 @@ - @@ -73,7 +72,7 @@ - + @@ -83,4 +82,4 @@ - \ No newline at end of file +