From d86d0668bbb1c51695b7b914e60e49aec1fdefbb Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 11:20:54 -0600 Subject: [PATCH 1/9] Add official signed build --- eng/SignCheckExclusionsFile.txt | 12 ++ eng/Signing.props | 124 +++++++++++++- eng/Subsets.props | 2 +- eng/Versions.props | 3 +- eng/liveBuilds.targets | 37 ++++- .../common/upload-unsigned-artifacts-step.yml | 22 +++ eng/pipelines/coreclr/templates/build-job.yml | 48 +++--- eng/pipelines/installer/installer-matrix.yml | 4 +- eng/pipelines/installer/jobs/base-job.yml | 41 ++++- eng/pipelines/installer/jobs/bash-build.yml | 6 +- eng/pipelines/installer/jobs/osx-build.yml | 5 +- .../installer/jobs/run-publish-project.yml | 156 ------------------ .../jobs/steps/upload-job-artifacts.yml | 49 ++---- .../installer/jobs/windows-build.yml | 18 +- eng/pipelines/libraries/base-job.yml | 13 +- eng/pipelines/libraries/build-job.yml | 36 ++-- .../jobs/prepare-signed-artifacts.yml | 17 +- .../stages/publish.yml | 52 +++--- eng/pipelines/runtime-official.yml | 98 +++++++++++ src/installer/Directory.Build.targets | 7 - src/installer/pkg/Directory.Build.targets | 15 ++ src/installer/pkg/packaging/installers.proj | 6 +- .../pkg/projects/Directory.Build.props | 8 +- .../Microsoft.NETCore.App.SharedFx.sfxproj | 5 + src/installer/publish/Directory.Build.targets | 8 +- src/installer/signing/Directory.Build.props | 4 +- src/installer/signing/Directory.Build.targets | 19 ++- src/installer/signing/SignR2RBinaries.proj | 7 + .../PrepareTestAssets/PrepareTestAssets.proj | 8 + 29 files changed, 516 insertions(+), 314 deletions(-) create mode 100644 eng/SignCheckExclusionsFile.txt create mode 100644 eng/pipelines/common/upload-unsigned-artifacts-step.yml delete mode 100644 eng/pipelines/installer/jobs/run-publish-project.yml rename eng/pipelines/{installer => official}/jobs/prepare-signed-artifacts.yml (75%) rename eng/pipelines/{installer => official}/stages/publish.yml (50%) create mode 100644 eng/pipelines/runtime-official.yml create mode 100644 src/installer/signing/SignR2RBinaries.proj diff --git a/eng/SignCheckExclusionsFile.txt b/eng/SignCheckExclusionsFile.txt new file mode 100644 index 0000000000000..2db3b79a9339e --- /dev/null +++ b/eng/SignCheckExclusionsFile.txt @@ -0,0 +1,12 @@ +;; Exclusions for SignCheck. Corresponds to info in Signing.props. +;; Format: https://github.com/dotnet/arcade/blob/397316e195639450b6c76bfeb9823b40bee72d6d/src/SignCheck/Microsoft.SignCheck/Verification/Exclusion.cs#L23-L35 +;; +;; This issue tracks a way to implement exclusions via Signing.props and avoid this extra file: https://github.com/dotnet/arcade/issues/2888 + +;; The apphost and comhost are template files, modified by the SDK to produce the executable for FDE +;; and SCD apps. If they are signed, the file that the SDK produces has an invalid signature and +;; can't be signed again. More info at https://github.com/dotnet/core-setup/pull/7549. +*apphost.exe;;Template, https://github.com/dotnet/core-setup/pull/7549 +*comhost.dll;;Template, https://github.com/dotnet/core-setup/pull/7549 +*apphosttemplateapphostexe.exe;;Template, https://github.com/dotnet/core-setup/pull/7549 +*comhosttemplatecomhostdll.dll;;Template, https://github.com/dotnet/core-setup/pull/7549 diff --git a/eng/Signing.props b/eng/Signing.props index 8dcf5bde81764..3ab32f20c8eeb 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,3 +1,123 @@ - - \ No newline at end of file + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/Subsets.props b/eng/Subsets.props index ba5a44937763f..a97756a850f48 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -121,7 +121,7 @@ - + diff --git a/eng/Versions.props b/eng/Versions.props index 719943491a71a..55a8ccc7b31db 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,7 +15,8 @@ release true - true + + false true diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 0fc0a9a6ca462..10e471c9ced69 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -5,12 +5,29 @@ $(Configuration) $(OSGroup) $(Configuration) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', '$(NetCoreAppCurrent)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', '$(NetCoreAppCurrent)', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'runtime', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + + $([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework')) + + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts')) + + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'packages', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping')) + + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts')) + + $([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigurationsArtifactsPath)', 'packages', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigPackagesDir)', 'Shipping')) + $([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigPackagesDir)', 'NonShipping')) + + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'ref', '$(NetCoreAppCurrent)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'pkg', '$(NetCoreAppCurrent)', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'runtime', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'native', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) x64 x86 x64 @@ -21,6 +38,10 @@ $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','sharedFramework')) $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDirName)','sharedFramework')) @@ -51,6 +72,10 @@ $(CoreCLRCrossTargetComponentDir)PDB/*.pdb; $(CoreCLRCrossTargetComponentDir)PDB/*.dbg; $(CoreCLRCrossTargetComponentDir)PDB/*.dwarf" /> + + + runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + diff --git a/eng/pipelines/common/upload-unsigned-artifacts-step.yml b/eng/pipelines/common/upload-unsigned-artifacts-step.yml new file mode 100644 index 0000000000000..0e11cb1e75555 --- /dev/null +++ b/eng/pipelines/common/upload-unsigned-artifacts-step.yml @@ -0,0 +1,22 @@ +parameters: + name: '' + +steps: +- task: CopyFiles@2 + displayName: Prepare job-specific intermediate unsigned artifacts subdirectory + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)' + Contents: | + Shipping/**/* + NonShipping/**/* + TargetFolder: '$(Build.StagingDirectory)/UnsignedArtifacts/${{ parameters.name }}' + CleanTargetFolder: true + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + +- task: PublishBuildArtifacts@1 + displayName: Publish intermediate unsigned artifacts + inputs: + pathToPublish: '$(Build.StagingDirectory)/UnsignedArtifacts' + artifactName: IntermediateUnsignedArtifacts + artifactType: container + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 2a002786640c2..f35503a3a65ac 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -80,10 +80,11 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: officialBuildIdArg value: '-officialbuildid=$(Build.BuildNumber)' - # IBCMerge is currently Windows-only and x86/x64-only - - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'x86'))) }}: - - name: ibcOptimizeArg - value: '-ibcoptimize' + # TODO (Consolidation): Fix internal tool restore and re-enable. + # # IBCMerge is currently Windows-only and x86/x64-only + # - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'x86'))) }}: + # - name: ibcOptimizeArg + # value: '-ibcoptimize' - name: enforcePgoArg value: '' # The EnforcePGO script is only supported on Windows and is not supported on arm or arm64. @@ -107,25 +108,26 @@ jobs: - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies - # Install internal tools on official builds - # Since our internal tools are behind an authenticated feed, - # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. - # We can't do this from within the build, so we need to do this as a separate step. - - ${{ if and(and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')), eq(parameters.osGroup, 'Windows_NT')) }}: - - task: DotNetCoreInstaller@0 - inputs: - packageType: 'sdk' - version: '2.1.503' - - task: DotNetCoreCLI@2 - displayName: Restore internal tools - inputs: - command: restore - feedsToUse: config - projects: '$(Build.SourcesDirectory)/eng/common/internal/Tools.csproj' - nugetConfigPath: 'eng/internal/NuGet.config' - restoreDirectory: '$(Build.SourcesDirectory)\.packages' - verbosityRestore: 'normal' - externalFeedCredentials: 'dotnet-core-internal-tooling' + # TODO (Consolidation): Fix internal tool restore and re-enable. + # # Install internal tools on official builds + # # Since our internal tools are behind an authenticated feed, + # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. + # # We can't do this from within the build, so we need to do this as a separate step. + # - ${{ if and(and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')), eq(parameters.osGroup, 'Windows_NT')) }}: + # - task: DotNetCoreInstaller@0 + # inputs: + # packageType: 'sdk' + # version: '2.1.503' + # - task: DotNetCoreCLI@2 + # displayName: Restore internal tools + # inputs: + # command: restore + # feedsToUse: config + # projects: '$(Build.SourcesDirectory)/eng/common/internal/Tools.csproj' + # nugetConfigPath: 'eng/internal/NuGet.config' + # restoreDirectory: '$(Build.SourcesDirectory)\.packages' + # verbosityRestore: 'normal' + # externalFeedCredentials: 'dotnet-core-internal-tooling' # Build - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: diff --git a/eng/pipelines/installer/installer-matrix.yml b/eng/pipelines/installer/installer-matrix.yml index 886eee61fd40a..d9533cded8538 100644 --- a/eng/pipelines/installer/installer-matrix.yml +++ b/eng/pipelines/installer/installer-matrix.yml @@ -13,7 +13,7 @@ jobs: - Windows_NT_arm64 jobParameters: ${{ insert }}: ${{ parameters }} - skipTests: true + crossBuild: true - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -42,7 +42,6 @@ jobs: jobParameters: ${{ insert }}: ${{ parameters }} portableBuild: true - skipTests: true crossBuild: true - template: /eng/pipelines/common/platform-matrix.yml @@ -67,7 +66,6 @@ jobs: additionalMSBuildArgs: /p:OutputRid=linux-musl-arm64 portableBuild: false crossBuild: true - skipTests: true - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 34ba20e8e4c3f..e339f82aa35e0 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -3,6 +3,7 @@ parameters: osGroup: '' archType: '' osSubgroup: '' + platform: '' timeoutInMinutes: 120 condition: true container: '' @@ -23,6 +24,9 @@ parameters: buildCommandSourcesDirectory: '$(Build.SourcesDirectory)/' + isOfficialBuild: false + useOfficialAllConfigurations: false + liveCoreClrBuildConfig: '' liveLibrariesBuildConfig: '' @@ -44,16 +48,19 @@ jobs: - name: ${{ variable.key }} value: ${{ variable.value }} + - name: SkipTests + value: ${{ or(parameters.isOfficialBuild, parameters.crossBuild) }} + - name: OfficialBuildArg value: '' # Produce test-signed build for PR and Public builds - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.isOfficialBuild, false) }}: - name: SignType value: test # Set up non-PR build from internal project - - ${{ if eq(variables.isOfficialBuild, true) }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] - name: OfficialBuildArg @@ -63,16 +70,21 @@ jobs: value: >- $(CoreCLRArtifactsPathArg) $(LibrariesConfigurationArg) + $(LibrariesAllConfigurationsArtifactsPathArg) - name: CoreCLRArtifactsPathArg value: '' - name: LibrariesConfigurationArg value: '' + - name: LibrariesAllConfigurationsArtifactsPathArg + value: '' - name: CoreClrDownloadPath value: '' - name: LibrariesDownloadPath value: '' + - name: LibrariesDownloadPathAllConfigurations + value: '' - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: - name: liveCoreClrLegName @@ -84,7 +96,7 @@ jobs: - name: CoreClrDownloadPath value: 'artifacts/transport/coreclr' - name: CoreCLRArtifactsPathArg - value: /p:CoreCLRArtifactsPath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath) + value: /p:CoreCLROverridePath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath) - name: CoreClrArtifactName value: CoreCLRProduct_$(liveCoreClrLegName) @@ -102,6 +114,16 @@ jobs: - name: LibrariesConfigurationArg value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + # Download allconfigurations bits to a separate location. It overlaps with the live libraries + # download and causes conflict when unzipping to the same location. + - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}: + - name: LibrariesDownloadPathAllConfigurations + value: 'artifacts/transport/librariesallconfigurations' + - name: LibrariesAllConfigurationsArtifactsPathArg + value: /p:LibrariesAllConfigurationsArtifactsPath=${{ parameters.buildCommandSourcesDirectory }}$(LibrariesDownloadPathAllConfigurations) + - name: LibrariesArtifactNameAllConfigurations + value: libraries_bin_official_allconfigurations + dependsOn: - checkout - ${{ parameters.dependsOn }} @@ -117,6 +139,8 @@ jobs: parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}: + - libraries_build_allconfigurations_Windows_NT_x64_Release steps: @@ -159,12 +183,23 @@ jobs: displayName: 'Libraries artifacts' cleanUnpackFolder: false + - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPathAllConfigurations) + # Always use '.zip' because AllConfigurations is built on Windows but used everywhere. + artifactFileName: '$(LibrariesArtifactNameAllConfigurations).zip' + artifactName: '$(LibrariesArtifactNameAllConfigurations)' + displayName: 'Libraries artifacts (AllConfigurations)' + cleanUnpackFolder: false + - ${{ parameters.buildSteps }} - template: steps/upload-job-artifacts.yml parameters: name: ${{ coalesce(parameters.name, parameters.platform) }} skipTests: ${{ parameters.skipTests }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - script: set -x && df -h diff --git a/eng/pipelines/installer/jobs/bash-build.yml b/eng/pipelines/installer/jobs/bash-build.yml index c30ed40b2b295..f7f61277ef238 100644 --- a/eng/pipelines/installer/jobs/bash-build.yml +++ b/eng/pipelines/installer/jobs/bash-build.yml @@ -7,7 +7,6 @@ parameters: disableCrossgen: false container: '' packageDistroList: null - skipTests: false archType: x64 timeoutInMinutes: 120 pool: '' @@ -18,6 +17,8 @@ jobs: parameters: ${{ insert }}: ${{ parameters }} + skipTests: ${{ or(parameters.isOfficialBuild, parameters.crossBuild) }} + # Do not attempt to clean workspace: the agent might not be able to remove the files because # they may be owned by "root" due to the way this job uses Docker. This job does its own cleanup # as a prepare step. @@ -63,10 +64,11 @@ jobs: # This causes a newline in the arg string that causes failure. BuildArguments: >- --restore --build --ci --test + -configuration $(_BuildConfig) /p:CrossBuild=${{ parameters.crossBuild }} /p:DisableCrossgen=${{ parameters.disableCrossgen }} /p:PortableBuild=${{ parameters.portableBuild }} - /p:SkipTests=${{ parameters.skipTests }} + /p:SkipTests=$(SkipTests) $(LiveOverridePathArgs) $(CommonMSBuildArgs) ${{ parameters.additionalMSBuildArgs }} diff --git a/eng/pipelines/installer/jobs/osx-build.yml b/eng/pipelines/installer/jobs/osx-build.yml index bff6c67b21382..d09944b0b3e1b 100644 --- a/eng/pipelines/installer/jobs/osx-build.yml +++ b/eng/pipelines/installer/jobs/osx-build.yml @@ -8,15 +8,18 @@ jobs: parameters: ${{ insert }}: ${{ parameters }} + skipTests: ${{ parameters.isOfficialBuild }} + buildVariables: CommonMSBuildArgs: >- - /p:Configuration=$(_BuildConfig) /p:PortableBuild=true + /p:SkipTests=$(SkipTests) buildSteps: - script: >- $(Build.SourcesDirectory)/installer.sh --restore --build --ci --test + -configuration $(_BuildConfig) /p:StripSymbols=true $(LiveOverridePathArgs) $(CommonMSBuildArgs) diff --git a/eng/pipelines/installer/jobs/run-publish-project.yml b/eng/pipelines/installer/jobs/run-publish-project.yml deleted file mode 100644 index c4a4d3bbb9a80..0000000000000 --- a/eng/pipelines/installer/jobs/run-publish-project.yml +++ /dev/null @@ -1,156 +0,0 @@ -parameters: - projectName: '' - dependency: null - -jobs: - -- template: /eng/common/templates/post-build/setup-maestro-vars.yml - -- job: CustomPublish - displayName: Custom Publish - dependsOn: setupMaestroVars - # Only run this job if setup-maestro-vars says the current branch publishes to this channel. - # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml - condition: contains( - dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], - format('[{0}]', variables['${{ parameters.dependency.channel.bar }}'])) - pool: - name: Hosted VS2017 - # Double the default timeout. - timeoutInMinutes: 120 - workspace: - clean: all - - variables: - # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. - - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: - # Used for publishing individual leg assets to azure blob storage - - ${{ if eq(parameters.dependency.channel.public, 'true') }}: - - group: DotNet-DotNetCli-Storage - - ${{ if ne(parameters.dependency.channel.public, 'true') }}: - - group: DotNet-MSRC-Storage - # Used for dotnet/versions update - - group: DotNet-Versions-Publish - - - name: _DefaultContainerName - value: dotnet - - name: _DefaultChecksumsContainerName - value: dotnet - - - ${{ if eq(parameters.dependency.channel.public, 'true') }}: - - name: _DefaultAzureAccountName - value: dotnetcli - - name: _DefaultAzureAccessToken - value: $(dotnetcli-storage-key) - - name: _DefaultChecksumAzureAccountName - value: dotnetclichecksums - - name: _DefaultChecksumAzureAccessToken - value: $(dotnetclichecksums-storage-key) - # dotnet/versions update - - name: _GitHubUser - value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] - - name: _GitHubEmail - value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] - - name: _GitHubAuthToken - value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] - - name: _VersionsRepoOwner - value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] - - name: _VersionsRepo - value: $[ coalesce(variables.VersionsRepo, 'versions') ] - - name: _DotNetVersionsArgs - value: >- - /p:GitHubUser=$(_GitHubUser) - /p:GitHubEmail=$(_GitHubEmail) - /p:GitHubAuthToken=$(_GitHubAuthToken) - /p:VersionsRepoOwner=$(_VersionsRepoOwner) - /p:VersionsRepo=$(_VersionsRepo) - /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) - - - ${{ if ne(parameters.dependency.channel.public, 'true') }}: - - name: _DefaultAzureAccountName - value: dotnetclimsrc - - name: _DefaultAzureAccessToken - value: $(dotnetclimsrc-access-key) - - name: _DefaultChecksumAzureAccountName - value: dotnetclimsrc - - name: _DefaultChecksumsContainerName - value: dotnet-checksums - - name: _DefaultChecksumAzureAccessToken - value: $(dotnetclimsrc-access-key) - # dotnet/versions update (disabled) - - name: _DotNetVersionsArgs - value: '' - - # Blob storage publish (installers and checksums) - - name: _AzureAccountName - value: $[ coalesce(variables.AzureAccountName, '$(_DefaultAzureAccountName)') ] - - name: _ContainerName - value: $[ coalesce(variables.ContainerName, '$(_DefaultContainerName)') ] - - name: _AzureAccessToken - value: $[ coalesce(variables.AzureAccessToken, '$(_DefaultAzureAccessToken)') ] - - name: _ChecksumAzureAccountName - value: $[ coalesce(variables.ChecksumAzureAccountName, '$(_DefaultChecksumAzureAccountName)') ] - - name: _ChecksumContainerName - value: $[ coalesce(variables.ChecksumContainerName, '$(_DefaultChecksumsContainerName)') ] - - name: _ChecksumAzureAccessToken - value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(_DefaultChecksumAzureAccessToken)') ] - - - name: _CommonPublishArgs - value: >- - /p:AzureAccountName=$(_AzureAccountName) - /p:ContainerName=$(_ContainerName) - /p:AzureAccessToken=$(_AzureAccessToken) - /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName) - /p:ChecksumContainerName=$(_ChecksumContainerName) - /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) - - steps: - - - ${{ if eq(variables.isOfficialBuild, true) }}: - - task: NuGetAuthenticate@0 - - - task: DownloadBuildArtifacts@0 - displayName: Download Artifacts - inputs: - artifactName: PreparedArtifacts - downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload - - - powershell: | - $prefix = "refs/heads/" - $branch = "$(Build.SourceBranch)" - $branchName = $branch - if ($branchName.StartsWith($prefix)) - { - $branchName = $branchName.Substring($prefix.Length) - } - Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" - Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" - displayName: Find true SourceBranchName - - - script: >- - build.cmd -ci - -projects $(Build.SourcesDirectory)\src\publish\${{ parameters.projectName }}.proj - /p:Channel=${{ parameters.dependency.channel.storage }} - /p:Configuration=Release - $(_CommonPublishArgs) - $(_DotNetVersionsArgs) - /bl:$(Build.SourcesDirectory)\${{ parameters.projectName }}.binlog - displayName: Publish to custom locations - - - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact BuildLogs - inputs: - PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' - ArtifactName: Logs-CustomPublish-${{ parameters.dependency.dependsOn }}-${{ parameters.projectName }} - condition: succeededOrFailed() diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index c9660842700c7..a9c1bdcb0dd9e 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -1,40 +1,25 @@ parameters: name: '' - skipTests: false -steps: -# Upload build outputs as build artifacts only if internal and not PR, to save storage space. -- ${{ if eq(variables.isOfficialBuild, true) }}: - - task: CopyFiles@2 - displayName: Prepare job-specific Artifacts subdirectory - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)' - Contents: | - Shipping/**/* - NonShipping/**/* - TargetFolder: '$(Build.StagingDirectory)/Artifacts/${{ parameters.name }}' - CleanTargetFolder: true - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + isOfficialBuild: false - - task: PublishBuildArtifacts@1 - displayName: Publish Artifacts - inputs: - pathToPublish: '$(Build.StagingDirectory)/Artifacts' - artifactName: IntermediateUnsignedArtifacts - artifactType: container - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) +steps: +# Upload build artifacts (packages) to pipeline only if official, to save storage space. +- ${{ if eq(parameters.isOfficialBuild, true) }}: + - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml + parameters: + name: ${{ parameters.name }} -- ${{ if eq(parameters.skipTests, false) }}: - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - mergeTestResults: true - testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: always() +- task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + mergeTestResults: true + testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig) + continueOnError: true + condition: eq(variables.SkipTests, false) - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory diff --git a/eng/pipelines/installer/jobs/windows-build.yml b/eng/pipelines/installer/jobs/windows-build.yml index 9126eed56cd8c..a52b8c60be0d4 100644 --- a/eng/pipelines/installer/jobs/windows-build.yml +++ b/eng/pipelines/installer/jobs/windows-build.yml @@ -1,7 +1,7 @@ parameters: additionalMSBuildArguments: '' + crossBuild: false publishRidAgnosticPackages: false - skipTests: false archType: null timeoutInMinutes: 120 platform: '' @@ -14,10 +14,9 @@ jobs: buildVariables: CommonMSBuildArgs: >- - /p:Configuration=$(_BuildConfig) /p:TargetArchitecture=${{ parameters.archType }} /p:PortableBuild=true - /p:SkipTests=${{ parameters.skipTests }} + /p:SkipTests=$(SkipTests) $(OfficialBuildArg) MsbuildSigningArguments: >- /p:CertificateId=400 @@ -28,20 +27,9 @@ jobs: - script: >- installer.cmd -restore -build -ci -test + -configuration $(_BuildConfig) $(LiveOverridePathArgs) $(CommonMSBuildArgs) $(MsbuildSigningArguments) displayName: Build - - ${{ if eq(variables.isOfficialBuild, true) }}: - - task: NuGetCommand@2 - displayName: Push Visual Studio NuPkgs - inputs: - command: push - packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/*/VS.Redist.Common.*.nupkg' - nuGetFeedType: external - publishFeedCredentials: 'DevDiv - VS package feed' - condition: and( - succeeded(), - eq(variables['_BuildConfig'], 'Release'), - ne(variables['DisableVSPublish'], 'true')) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 14e30ad1c92a4..bba59214aa6b3 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -5,6 +5,7 @@ parameters: osSubgroup: '' framework: '' isOfficialBuild: false + isOfficialAllConfigurations: false liveCoreClrBuildConfig: '' timeoutInMinutes: 150 condition: true @@ -43,6 +44,9 @@ jobs: - _warnAsErrorArg: '' - _testScopeArg: '' + - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.testScope, '') }}: - _testScopeArg: -testscope ${{ parameters.testScope }} @@ -58,15 +62,16 @@ jobs: - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - _runtimeOSArg: -os ${{ parameters.osGroup }} - - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: + - ${{ if eq(parameters.framework, 'allConfigurations') }}: - _finalFrameworkArg: -allConfigurations + - ${{ if eq(parameters.isOfficialAllConfigurations, true) }}: + - _skipTestHostCopy: true + - librariesBuildArtifactName: 'libraries_bin_official_allconfigurations' + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - _coreClrArtifactName: '' - _coreClrDownloadPath: '' - _coreClrArtifactsPathArg: '' diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 32f51c907fdd7..c51dc1c403e14 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -5,6 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false + isOfficialAllConfigurations: false # When set to a non-empty value (Debug / Release), it determines CoreCLR # build configuration to use for building libraries and tests. Setting this @@ -30,6 +31,7 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} + isOfficialAllConfigurations: ${{ parameters.isOfficialAllConfigurations }} liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} preBuildSteps: ${{ parameters.preBuildSteps }} @@ -63,18 +65,19 @@ jobs: - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - task: DotNetCoreCLI@2 - displayName: Restore internal tools - condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) - inputs: - command: restore - feedsToUse: config - projects: 'eng/common/internal/Tools.csproj' - nugetConfigPath: 'eng/internal/NuGet.config' - restoreDirectory: '$(Build.SourcesDirectory)\.packages' - verbosityRestore: 'normal' - externalFeedCredentials: 'dotnet-core-internal-tooling' + # TODO (Consolidation): Fix internal tool restore and re-enable. + # - ${{ if eq(parameters.isOfficialBuild, true) }}: + # - task: DotNetCoreCLI@2 + # displayName: Restore internal tools + # condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) + # inputs: + # command: restore + # feedsToUse: config + # projects: 'eng/common/internal/Tools.csproj' + # nugetConfigPath: 'eng/internal/NuGet.config' + # restoreDirectory: '$(Build.SourcesDirectory)\.packages' + # verbosityRestore: 'normal' + # externalFeedCredentials: 'dotnet-core-internal-tooling' - ${{ if eq(parameters.osGroup, 'OSX') }}: - script: | @@ -141,6 +144,7 @@ jobs: inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost + condition: ne(variables['_skipTestHostCopy'], 'true') - task: CopyFiles@2 displayName: Prepare artifacts toolset folder to publish @@ -164,6 +168,14 @@ jobs: artifactName: $(librariesBuildArtifactName) displayName: Build Assets + # Save AllConfigurations artifacts using the prepare-signed-artifacts format. The + # platform-specific jobs' nupkgs automatically flow through the matching platform-specific + # Installer build, but AllConfigurations should only be uploaded once, here. + - ${{ if eq(parameters.isOfficialAllConfigurations, true) }}: + - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + - ${{ if eq(parameters.runTests, true) }}: - template: /eng/pipelines/libraries/helix.yml parameters: diff --git a/eng/pipelines/installer/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml similarity index 75% rename from eng/pipelines/installer/jobs/prepare-signed-artifacts.yml rename to eng/pipelines/official/jobs/prepare-signed-artifacts.yml index 0634047e105e3..8df2deebf9560 100644 --- a/eng/pipelines/installer/jobs/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -1,6 +1,7 @@ parameters: dependsOn: [] - PublishRidAgnosticPackagesFromJobName: '' + PublishRidAgnosticPackagesFromPlatform: '' + isOfficialBuild: false jobs: - job: PrepareSignedArtifacts @@ -14,9 +15,14 @@ jobs: workspace: clean: all + variables: + - name: SignType + value: $[ coalesce(variables.OfficialSignType, 'real') ] + steps: + - template: /eng/pipelines/common/clone-checkout-bundle-step.yml - - ${{ if eq(variables.isOfficialBuild, true) }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - task: NuGetAuthenticate@0 - task: MicroBuildSigningPlugin@2 @@ -36,9 +42,10 @@ jobs: - script: >- build.cmd -ci - -projects $(Build.SourcesDirectory)\src\publish\prepare-artifacts.proj - /p:Configuration=Release - /p:PublishRidAgnosticPackagesFromJobName=${{ parameters.PublishRidAgnosticPackagesFromJobName }} + -configuration Release + -projects $(Build.SourcesDirectory)\src\installer\publish\prepare-artifacts.proj + /p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }} + /p:OfficialBuildId=$(Build.BuildNumber) /p:SignType=$(SignType) /p:DotNetSignType=$(SignType) /bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog diff --git a/eng/pipelines/installer/stages/publish.yml b/eng/pipelines/official/stages/publish.yml similarity index 50% rename from eng/pipelines/installer/stages/publish.yml rename to eng/pipelines/official/stages/publish.yml index 29e12bf3d3a0d..3ffac1c10d45f 100644 --- a/eng/pipelines/installer/stages/publish.yml +++ b/eng/pipelines/official/stages/publish.yml @@ -1,28 +1,31 @@ parameters: - dependsOnPublishStages: [] - pipelinesPath: '' + PublishRidAgnosticPackagesFromPlatform: Windows_NT_x64 stages: -# Create extra stage per BAR channel that needs extra publish steps. -- ${{ each dependency in parameters.dependsOnPublishStages }}: - - stage: PublishBlob_${{ dependency.dependsOn }} - displayName: '${{ dependency.channel.name }} Blob Publish' - dependsOn: PrepareForPublish - variables: - - template: /eng/common/templates/post-build/common-variables.yml - jobs: - - template: /eng/pipelines/jobs/run-publish-project.yml - parameters: - projectName: publish-blobs - dependency: ${{ dependency }} +- stage: PrepareForPublish + displayName: Prepare for Publish + jobs: + + # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. + - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml + parameters: + PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} + + # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + publishUsingPipelines: true + dependsOn: PrepareSignedArtifacts + pool: + name: NetCoreInternal-Pool + queue: buildpool.windows.10.amd64.vs2017 # Stages-based publishing entry point - template: /eng/common/templates/post-build/post-build.yml parameters: validateDependsOn: - - ${{ each dependency in parameters.dependsOnPublishStages }}: - - PublishBlob_${{ dependency.dependsOn }} + - PrepareForPublish # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871 enableSymbolValidation: false # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604 @@ -32,7 +35,7 @@ stages: symbolPublishingAdditionalParameters: "'-warnAsError:$false'" # Enable SDL validation, passing through values from the 'core-setup-sdl-validation' group. SDLValidationParameters: - enable: true + enable: false # TODO: (Consolidation) Decide who owns SDL validation errors and enable. artifactNames: - PackageArtifacts - BlobArtifacts @@ -47,18 +50,3 @@ stages: -TsaRepositoryName "$(TsaRepositoryName)" -TsaCodebaseName "$(TsaCodebaseName)" -TsaPublish $True - -# Create extra stage per BAR channel that needs extra publish steps. These run after the Arcade -# stages because they depend on Arcade's NuGet package publish being complete. -- ${{ each dependency in parameters.dependsOnPublishStages }}: - - stage: PublishFinal_${{ dependency.dependsOn }} - displayName: '${{ dependency.channel.name }} Finalize' - dependsOn: - - ${{ dependency.dependsOn }} - variables: - - template: /eng/common/templates/post-build/common-variables.yml - jobs: - - template: ${{ parameters.pipelinesPath }}/jobs/run-publish-project.yml - parameters: - projectName: publish-final - dependency: ${{ dependency }} diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml new file mode 100644 index 0000000000000..cc6071ec690fd --- /dev/null +++ b/eng/pipelines/runtime-official.yml @@ -0,0 +1,98 @@ +trigger: + batch: true + branches: + include: + - master + - release/* + +pr: none + +variables: +- template: /eng/pipelines/common/variables.yml +# TODO: (Consolidation) Switch away from old signing/validation variables from former Core-Setup. +- name: TeamName + value: dotnet-core-acquisition +# Set the target blob feed for package publish during official and validation builds. +- name: _DotNetArtifactsCategory + value: .NETCore +- name: _DotNetValidationArtifactsCategory + value: .NETCoreValidation + +stages: +- stage: Build + jobs: + # + # Checkout repository + # + - template: /eng/pipelines/common/checkout-job.yml + + # + # Build CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: release + platforms: + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + + # + # Build libraries using live CoreLib from CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: ${{ variables.isOfficialBuild }} + liveCoreClrBuildConfig: release + + # + # Build libraries AllConfigurations for packages + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + jobParameters: + framework: allConfigurations + isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialAllConfigurations: true + liveCoreClrBuildConfig: release + + # + # Installer Build + # + - template: /eng/pipelines/installer/installer-matrix.yml + parameters: + liveCoreClrBuildConfig: release + liveLibrariesBuildConfig: Release + isOfficialBuild: ${{ variables.isOfficialBuild }} + useOfficialAllConfigurations: true + +- ${{ if eq(variables.isOfficialBuild, true) }}: + - template: /eng/pipelines/official/stages/publish.yml + parameters: + isOfficialBuild: ${{ variables.isOfficialBuild }} diff --git a/src/installer/Directory.Build.targets b/src/installer/Directory.Build.targets index 3d45d9ee49c92..e31f535a7b4be 100644 --- a/src/installer/Directory.Build.targets +++ b/src/installer/Directory.Build.targets @@ -11,13 +11,6 @@ - - - $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping')) - $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping')) - - diff --git a/src/installer/pkg/Directory.Build.targets b/src/installer/pkg/Directory.Build.targets index d457972bfd7f5..0abfa5bb0f3fa 100644 --- a/src/installer/pkg/Directory.Build.targets +++ b/src/installer/pkg/Directory.Build.targets @@ -1,6 +1,21 @@ + + + + + diff --git a/src/installer/pkg/packaging/installers.proj b/src/installer/pkg/packaging/installers.proj index 21736cb1953e5..029129b97c8c1 100644 --- a/src/installer/pkg/packaging/installers.proj +++ b/src/installer/pkg/packaging/installers.proj @@ -22,7 +22,7 @@ @@ -50,8 +50,8 @@ Targets="Build" /> - - + + diff --git a/src/installer/pkg/projects/Directory.Build.props b/src/installer/pkg/projects/Directory.Build.props index 3674c1506c2a2..4a54bea6c12da 100644 --- a/src/installer/pkg/projects/Directory.Build.props +++ b/src/installer/pkg/projects/Directory.Build.props @@ -55,10 +55,14 @@ - + + + diff --git a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj index 3ea98ed5d6c64..d12d993a3ff61 100644 --- a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj +++ b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj @@ -53,4 +53,9 @@ $(RestoreSources);$(LibrariesShippingPackagesDir) $(RestoreSources);$(LibrariesNonShippingPackagesDir) + + + $(RestoreSources);$(LibrariesAllConfigShippingPackagesDir) + $(RestoreSources);$(LibrariesAllConfigNonShippingPackagesDir) + diff --git a/src/installer/publish/Directory.Build.targets b/src/installer/publish/Directory.Build.targets index bb3927313199a..de17992820018 100644 --- a/src/installer/publish/Directory.Build.targets +++ b/src/installer/publish/Directory.Build.targets @@ -41,8 +41,14 @@ $(DownloadDirectory)**\VS.Redist.Common.*.nupkg" Exclude="@(DownloadedSymbolNupkgFile)" /> + diff --git a/src/installer/signing/Directory.Build.props b/src/installer/signing/Directory.Build.props index 59d1934a25486..22b57d90bf71c 100644 --- a/src/installer/signing/Directory.Build.props +++ b/src/installer/signing/Directory.Build.props @@ -5,7 +5,7 @@ $(NETCoreAppFramework) - - true + + true diff --git a/src/installer/signing/Directory.Build.targets b/src/installer/signing/Directory.Build.targets index 0afd9438c666a..43bbbcf802cc2 100644 --- a/src/installer/signing/Directory.Build.targets +++ b/src/installer/signing/Directory.Build.targets @@ -10,6 +10,14 @@ if '$()' == 'true' and points Arcade to the correct files. --> + + $([System.DateTime]::UtcNow) + + + + ArtifactsPackagesDir=$(ArtifactsPackagesDir); + OSGroup=$(OSGroup); + TargetArchitecture=$(TargetArchitecture); + NetCoreAppCurrent=$(NetCoreAppCurrent)" /> + + + $([System.DateTime]::UtcNow.Subtract($([System.DateTime]::Parse('$(SignStartTime)')))) + + + diff --git a/src/installer/signing/SignR2RBinaries.proj b/src/installer/signing/SignR2RBinaries.proj new file mode 100644 index 0000000000000..f2f60ef725faa --- /dev/null +++ b/src/installer/signing/SignR2RBinaries.proj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj index 46b4ba4710734..4be962cc58339 100644 --- a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj +++ b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj @@ -51,6 +51,14 @@ + + + + + + + + Date: Wed, 18 Dec 2019 11:55:50 -0600 Subject: [PATCH 2/9] Revert "Remove VS feed publish (for rebuild)" I'd removed this to run the same official build ID multiple times. Adding it back. --- eng/pipelines/installer/jobs/windows-build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eng/pipelines/installer/jobs/windows-build.yml b/eng/pipelines/installer/jobs/windows-build.yml index a52b8c60be0d4..95230fc86a4f7 100644 --- a/eng/pipelines/installer/jobs/windows-build.yml +++ b/eng/pipelines/installer/jobs/windows-build.yml @@ -33,3 +33,15 @@ jobs: $(MsbuildSigningArguments) displayName: Build + - ${{ if eq(parameters.isOfficialBuild, true) }}: + - task: NuGetCommand@2 + displayName: Push Visual Studio NuPkgs + inputs: + command: push + packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/*/VS.Redist.Common.*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'DevDiv - VS package feed' + condition: and( + succeeded(), + eq(variables['_BuildConfig'], 'Release'), + ne(variables['DisableVSPublish'], 'true')) From 6d8190f2c367c135398435e8fb4420ac03686c64 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 13:12:30 -0600 Subject: [PATCH 3/9] Fix CoreCLRArtifactsPath default: not libraries --- eng/liveBuilds.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 10e471c9ced69..78291b52621ac 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -7,7 +7,7 @@ $(Configuration) $([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)')) - $([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework')) From d6c749a6b9cdeada8e4fff4860ba3e5cfe54921e Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:10:02 -0600 Subject: [PATCH 4/9] Don't attempt test signing during PR validation --- eng/pipelines/installer/jobs/base-job.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index e339f82aa35e0..6d15a1bcfb5c3 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -54,10 +54,12 @@ jobs: - name: OfficialBuildArg value: '' - # Produce test-signed build for PR and Public builds - - ${{ if eq(parameters.isOfficialBuild, false) }}: - - name: SignType - value: test + # TODO (Consolidation): Enable test signing during PR validation. + # + # CoreCLR only produces the UCRT redist file in Release config. When the redist file isn't + # present, signing fails. For now, only sign in official builds which only run Release mode. + - name: SignType + value: '' # Set up non-PR build from internal project - ${{ if eq(parameters.isOfficialBuild, true) }}: From 06d8284118d9ed7443401b5025a7d8b3c41ac563 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:11:45 -0600 Subject: [PATCH 5/9] Add build trigger include/exclude paths --- eng/pipelines/runtime-official.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index cc6071ec690fd..ce2273c6361ef 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -4,6 +4,19 @@ trigger: include: - master - release/* + paths: + include: + - '*' + - docs/manpages/* + exclude: + - docs/* + - CODE-OF-CONDUCT.md + - CONTRIBUTING.md + - LICENSE.TXT + - PATENTS.TXT + - README.md + - SECURITY.md + - THIRD-PARTY-NOTICES.TXT pr: none From 4b1c7fe4f7d6794a56d47abcd332d411447cbbec Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:25:56 -0600 Subject: [PATCH 6/9] Remove unnecessary conditions (config, success) --- eng/pipelines/common/upload-unsigned-artifacts-step.yml | 2 -- eng/pipelines/installer/jobs/osx-build.yml | 1 - eng/pipelines/installer/jobs/windows-build.yml | 1 - 3 files changed, 4 deletions(-) diff --git a/eng/pipelines/common/upload-unsigned-artifacts-step.yml b/eng/pipelines/common/upload-unsigned-artifacts-step.yml index 0e11cb1e75555..c4fea2adaa634 100644 --- a/eng/pipelines/common/upload-unsigned-artifacts-step.yml +++ b/eng/pipelines/common/upload-unsigned-artifacts-step.yml @@ -11,7 +11,6 @@ steps: NonShipping/**/* TargetFolder: '$(Build.StagingDirectory)/UnsignedArtifacts/${{ parameters.name }}' CleanTargetFolder: true - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) - task: PublishBuildArtifacts@1 displayName: Publish intermediate unsigned artifacts @@ -19,4 +18,3 @@ steps: pathToPublish: '$(Build.StagingDirectory)/UnsignedArtifacts' artifactName: IntermediateUnsignedArtifacts artifactType: container - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/eng/pipelines/installer/jobs/osx-build.yml b/eng/pipelines/installer/jobs/osx-build.yml index d09944b0b3e1b..168c2ed3883de 100644 --- a/eng/pipelines/installer/jobs/osx-build.yml +++ b/eng/pipelines/installer/jobs/osx-build.yml @@ -25,4 +25,3 @@ jobs: $(CommonMSBuildArgs) $(OfficialBuildArg) displayName: Build - condition: succeeded() diff --git a/eng/pipelines/installer/jobs/windows-build.yml b/eng/pipelines/installer/jobs/windows-build.yml index 95230fc86a4f7..87d82a54f2382 100644 --- a/eng/pipelines/installer/jobs/windows-build.yml +++ b/eng/pipelines/installer/jobs/windows-build.yml @@ -43,5 +43,4 @@ jobs: publishFeedCredentials: 'DevDiv - VS package feed' condition: and( succeeded(), - eq(variables['_BuildConfig'], 'Release'), ne(variables['DisableVSPublish'], 'true')) From d951d99806ad8eee0fd402bacae5d9cc1b479781 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:32:57 -0600 Subject: [PATCH 7/9] Add issue link for internal tool restore --- eng/Versions.props | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 4 ++-- eng/pipelines/libraries/build-job.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 55a8ccc7b31db..e7f7c8b9bf1f1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,7 +15,7 @@ release true - + false diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index f35503a3a65ac..f72743395b815 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -80,7 +80,7 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: officialBuildIdArg value: '-officialbuildid=$(Build.BuildNumber)' - # TODO (Consolidation): Fix internal tool restore and re-enable. + # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # # IBCMerge is currently Windows-only and x86/x64-only # - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'x86'))) }}: # - name: ibcOptimizeArg @@ -108,7 +108,7 @@ jobs: - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies - # TODO (Consolidation): Fix internal tool restore and re-enable. + # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # # Install internal tools on official builds # # Since our internal tools are behind an authenticated feed, # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index c51dc1c403e14..5a00aa27ec449 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -65,7 +65,7 @@ jobs: - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore - # TODO (Consolidation): Fix internal tool restore and re-enable. + # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # - ${{ if eq(parameters.isOfficialBuild, true) }}: # - task: DotNetCoreCLI@2 # displayName: Restore internal tools From 842a855787ddf4317bf8bc0728854633dcb27292 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:35:47 -0600 Subject: [PATCH 8/9] Add issue link for test signing --- eng/pipelines/installer/jobs/base-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 6d15a1bcfb5c3..a950e1108df0f 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -54,7 +54,7 @@ jobs: - name: OfficialBuildArg value: '' - # TODO (Consolidation): Enable test signing during PR validation. + # TODO (Consolidation): Enable test signing during PR validation. https://github.com/dotnet/runtime/issues/1026 # # CoreCLR only produces the UCRT redist file in Release config. When the redist file isn't # present, signing fails. For now, only sign in official builds which only run Release mode. From c5fbd3dfde0cd0d48d871a1572443029f16bdd9c Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 18 Dec 2019 14:57:06 -0600 Subject: [PATCH 9/9] Fix consolidation TODO formatting, links for all --- eng/Versions.props | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 4 ++-- eng/pipelines/installer/jobs/base-job.yml | 2 +- eng/pipelines/libraries/build-job.yml | 2 +- eng/pipelines/official/stages/publish.yml | 2 +- eng/pipelines/runtime-official.yml | 2 +- src/installer/pkg/Directory.Build.targets | 2 +- src/installer/signing/Directory.Build.props | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index e7f7c8b9bf1f1..7c8f2ee2d4d4c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,7 +15,7 @@ release true - + false diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index f72743395b815..537d2e363cd29 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -80,7 +80,7 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: officialBuildIdArg value: '-officialbuildid=$(Build.BuildNumber)' - # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 + # TODO: (Consolidation) Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # # IBCMerge is currently Windows-only and x86/x64-only # - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'x86'))) }}: # - name: ibcOptimizeArg @@ -108,7 +108,7 @@ jobs: - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies - # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 + # TODO: (Consolidation) Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # # Install internal tools on official builds # # Since our internal tools are behind an authenticated feed, # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index a950e1108df0f..ff6860ed33bbc 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -54,7 +54,7 @@ jobs: - name: OfficialBuildArg value: '' - # TODO (Consolidation): Enable test signing during PR validation. https://github.com/dotnet/runtime/issues/1026 + # TODO: (Consolidation) Enable test signing during PR validation. https://github.com/dotnet/runtime/issues/1026 # # CoreCLR only produces the UCRT redist file in Release config. When the redist file isn't # present, signing fails. For now, only sign in official builds which only run Release mode. diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 5a00aa27ec449..9b1395a68cfa8 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -65,7 +65,7 @@ jobs: - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore - # TODO (Consolidation): Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 + # TODO: (Consolidation) Fix internal tool restore and re-enable. https://github.com/dotnet/runtime/issues/1024 # - ${{ if eq(parameters.isOfficialBuild, true) }}: # - task: DotNetCoreCLI@2 # displayName: Restore internal tools diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml index 3ffac1c10d45f..654e42921b0a6 100644 --- a/eng/pipelines/official/stages/publish.yml +++ b/eng/pipelines/official/stages/publish.yml @@ -35,7 +35,7 @@ stages: symbolPublishingAdditionalParameters: "'-warnAsError:$false'" # Enable SDL validation, passing through values from the 'core-setup-sdl-validation' group. SDLValidationParameters: - enable: false # TODO: (Consolidation) Decide who owns SDL validation errors and enable. + enable: false # TODO: (Consolidation) Decide who owns SDL validation errors and enable. https://github.com/dotnet/runtime/issues/1027 artifactNames: - PackageArtifacts - BlobArtifacts diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index ce2273c6361ef..d1310fdc746fc 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -22,7 +22,7 @@ pr: none variables: - template: /eng/pipelines/common/variables.yml -# TODO: (Consolidation) Switch away from old signing/validation variables from former Core-Setup. +# TODO: (Consolidation) Switch away from old signing/validation variables from former Core-Setup. https://github.com/dotnet/runtime/issues/1027 - name: TeamName value: dotnet-core-acquisition # Set the target blob feed for package publish during official and validation builds. diff --git a/src/installer/pkg/Directory.Build.targets b/src/installer/pkg/Directory.Build.targets index 0abfa5bb0f3fa..611d04ee17d35 100644 --- a/src/installer/pkg/Directory.Build.targets +++ b/src/installer/pkg/Directory.Build.targets @@ -6,7 +6,7 @@ runs after "DependsOnTargets", so hook each crossgen target individually to ensure this happens before any of them. - TODO: (Consolidation) Add this target properly to the sharedfx tooling. + TODO: (Consolidation) Add this target properly to the sharedfx tooling. https://github.com/dotnet/runtime/issues/749 --> true