From 9a79d40957994d749343ecc3ddd0078676ca553e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:21:16 -0700 Subject: [PATCH 1/8] Append job attempt number to log artifact name to get rid of file exists failure on reattempts. (#99710) --- eng/pipelines/official/jobs/prepare-signed-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml index 0398bc3a4c505..481c69dc68f6a 100644 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -2,7 +2,7 @@ parameters: dependsOn: [] PublishRidAgnosticPackagesFromPlatform: '' isOfficialBuild: false - logArtifactName: 'Logs-PrepareSignedArtifacts' + logArtifactName: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' jobs: - job: PrepareSignedArtifacts From f9e22972d90e5defad24f09be808de82fd13addc Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:41:11 -0700 Subject: [PATCH 2/8] Update dependencies from https://github.com/dotnet/arcade build (#99693) Microsoft.DotNet.ApiCompat , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 6.0.0-beta.24113.1 -> To Version 6.0.0-beta.24162.4 Co-authored-by: dotnet-maestro[bot] Co-authored-by: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> --- NuGet.config | 3 - eng/Version.Details.xml | 76 ++--- eng/Versions.props | 30 +- eng/common/SetupNugetSources.ps1 | 26 +- eng/common/templates-official/job/job.yml | 276 +++++++++++++++++ .../templates-official/job/onelocbuild.yml | 109 +++++++ .../job/publish-build-assets.yml | 121 ++++++++ .../templates-official/job/source-build.yml | 64 ++++ .../job/source-index-stage1.yml | 59 ++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 100 +++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 26 ++ .../post-build/post-build.yml | 277 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../steps/component-governance.yml | 10 + .../steps/execute-codeql.yml | 32 ++ .../steps/generate-sbom.yml | 44 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 94 ++++++ .../templates-official/steps/source-build.yml | 108 +++++++ .../variables/sdl-variables.yml | 7 + global.json | 12 +- 26 files changed, 1623 insertions(+), 75 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/NuGet.config b/NuGet.config index f64ae4985405f..44cf761484a1b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,9 +9,6 @@ - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 25a1ae7d7e28d..b053921069e29 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -26,77 +26,77 @@ - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 https://github.com/microsoft/vstest @@ -222,9 +222,9 @@ https://github.com/dotnet/xharness 9cffd0b49204fa44c7577fa1a132e39e7cf4d038 - + https://github.com/dotnet/arcade - 96facbceaca1e4eb498055c005088764d3b38dff + c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 31cd7e023c036..99ec7edafa6aa 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -42,21 +42,21 @@ 1.1.0-preview.22164.17 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 2.5.1-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 - 6.0.0-beta.24113.1 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 2.5.1-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 + 6.0.0-beta.24162.4 6.0.0-preview.1.102 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 18823840b1127..0182856edd004 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -49,11 +49,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern Write-Host "Package source $SourceName already present." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +82,17 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,24 +144,24 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") if ($dotnet5Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") if ($dotnet6Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $doc.Save($filename) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 0000000000000..616bfc38696dd --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,276 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + disableComponentGovernance: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 6.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - task: NuGetAuthenticate@1 + + - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 0000000000000..e0e9a4bc51295 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,109 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + +jobs: +- job: OneLocBuild + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + + + steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 0000000000000..9498aa412db80 --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,121 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + + displayName: Publish to Build Asset Registry + + pool: ${{ parameters.pool }} + + variables: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: _BuildConfig + value: ${{ parameters.configuration }} + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Enable cross-org NuGet feed authentication + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:Configuration=$(_BuildConfig) + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 0000000000000..96be5e6e115a6 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,64 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-mariner-2-pt + os: linux + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 0000000000000..3d8b0b966cb77 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,59 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20210614.1 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + pool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + condition: '' + dependsOn: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + + pool: ${{ parameters.pool }} + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core sdk 3.1 + inputs: + packageType: sdk + version: 3.1.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 0000000000000..0bf7ee29f4085 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.110.1 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 0000000000000..669ccd48f7f08 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,100 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 0000000000000..2159009dc8e1d --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 0000000000000..fae340f4d20d7 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,26 @@ +variables: + - group: AzureDevOps-Artifact-Feeds-Pats + - group: DotNet-Blob-Feed + - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 0000000000000..80bfa4946c695 --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,277 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + jobs: + - job: + displayName: NuGet Validation + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000000..0c87f149a4ad7 --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 0000000000000..da669030daf6e --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 0000000000000..f67a210d62f3e --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 0000000000000..babc2757d8d12 --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,10 @@ +parameters: + disableComponentGovernance: false + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 0000000000000..9b4a5ffa30a78 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 0000000000000..7fc4f358023d6 --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,44 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed + +parameters: + PackageVersion: 6.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 0000000000000..04012fed182a1 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 0000000000000..83d97a26a01ff --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 0000000000000..cd02ae1607f3b --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,94 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 0000000000000..9eb7e51456a79 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,108 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + $internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 0000000000000..1a860bd040647 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.110.1 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/global.json b/global.json index 21b5e4e68bb40..55d09b736ccef 100644 --- a/global.json +++ b/global.json @@ -1,21 +1,21 @@ { "sdk": { - "version": "6.0.126", + "version": "6.0.127", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "6.0.126" + "dotnet": "6.0.127" }, "native-tools": { "cmake": "3.16.4", "python3": "3.7.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24113.1", - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24113.1", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24113.1", - "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24113.1", + "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24162.4", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24162.4", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24162.4", + "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24162.4", "Microsoft.Build.NoTargets": "3.1.0", "Microsoft.Build.Traversal": "3.0.23", "Microsoft.NET.Sdk.IL": "6.0.0-rc.1.21415.6" From 4c12f20015f17e34aa02eabc6203b9f230a0b663 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:46:19 -0700 Subject: [PATCH 3/8] Update dependencies from https://github.com/dotnet/emsdk build (#99717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Microsoft.NET.Workload.Emscripten.Manifest-6.0.100 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.300 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.400 From Version 6.0.28 -> To Version 6.0.28 Co-authored-by: dotnet-maestro[bot] Co-authored-by: Carlos Sánchez López <1175054+carlossanlop@users.noreply.github.com> --- NuGet.config | 1 + eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NuGet.config b/NuGet.config index 44cf761484a1b..2e7a373466ee9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,6 +9,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b053921069e29..7779c6af77c48 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -10,15 +10,15 @@ https://github.com/dotnet/emsdk - ee3546a41338fd3c82167b62d7ff989934c6cb01 + bc697dd84500aeee0bcb6323e64967222f90c2aa https://github.com/dotnet/emsdk - ee3546a41338fd3c82167b62d7ff989934c6cb01 + bc697dd84500aeee0bcb6323e64967222f90c2aa https://github.com/dotnet/emsdk - ee3546a41338fd3c82167b62d7ff989934c6cb01 + bc697dd84500aeee0bcb6323e64967222f90c2aa https://github.com/dotnet/wcf From 9f6f1412a80afffb6026e64b3c46aad9700d02c5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:21:43 -0700 Subject: [PATCH 4/8] Update dependencies from https://github.com/dotnet/arcade build (#99785) Microsoft.DotNet.ApiCompat , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 6.0.0-beta.24162.4 -> To Version 6.0.0-beta.24163.13 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 1 - eng/Version.Details.xml | 76 ++++++++++++++++---------------- eng/Versions.props | 30 ++++++------- eng/common/SetupNugetSources.ps1 | 2 +- global.json | 12 ++--- 5 files changed, 60 insertions(+), 61 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2e7a373466ee9..77683e8299192 100644 --- a/NuGet.config +++ b/NuGet.config @@ -10,7 +10,6 @@ - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7779c6af77c48..22a44e7194de3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -26,77 +26,77 @@ - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 https://github.com/microsoft/vstest @@ -222,9 +222,9 @@ https://github.com/dotnet/xharness 9cffd0b49204fa44c7577fa1a132e39e7cf4d038 - + https://github.com/dotnet/arcade - c80f6c7bcfdd0e62a3012f9a97ab52be477c7d75 + 100394106fe203c112ab72bcc1576fb3753f7910 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 99ec7edafa6aa..0b5fdf4b3319e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -42,21 +42,21 @@ 1.1.0-preview.22164.17 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 2.5.1-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 - 6.0.0-beta.24162.4 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 2.5.1-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 + 6.0.0-beta.24163.13 6.0.0-preview.1.102 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 0182856edd004..4ed5c54e7331b 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -49,7 +49,7 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern Write-Host "Package source $SourceName already present." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $pwd + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source diff --git a/global.json b/global.json index 55d09b736ccef..713ad621a247d 100644 --- a/global.json +++ b/global.json @@ -1,21 +1,21 @@ { "sdk": { - "version": "6.0.127", + "version": "6.0.128", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "6.0.127" + "dotnet": "6.0.128" }, "native-tools": { "cmake": "3.16.4", "python3": "3.7.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24162.4", - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24162.4", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24162.4", - "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24162.4", + "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24163.13", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24163.13", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24163.13", + "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24163.13", "Microsoft.Build.NoTargets": "3.1.0", "Microsoft.Build.Traversal": "3.0.23", "Microsoft.NET.Sdk.IL": "6.0.0-rc.1.21415.6" From 2cf691cfc25f5531739872f895d2b4ff82560820 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:50:44 -0700 Subject: [PATCH 5/8] Update dependencies from https://github.com/dotnet/xharness build (#99787) Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 6.0.0-prerelease.23531.3 -> To Version 6.0.0-prerelease.24154.4 Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index a5a04609e8a63..e7592493d05e6 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "6.0.0-prerelease.23531.3", + "version": "6.0.0-prerelease.24154.4", "commands": [ "xharness" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 22a44e7194de3..a049b84f90e95 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -214,13 +214,13 @@ https://github.com/mono/linker c8499798a2a09639174e2f5c694d6652794cc73d - + https://github.com/dotnet/xharness - 9cffd0b49204fa44c7577fa1a132e39e7cf4d038 + e3f823d9e7dd5b42dde2fc3cdc1fb3650dd197f2 - + https://github.com/dotnet/xharness - 9cffd0b49204fa44c7577fa1a132e39e7cf4d038 + e3f823d9e7dd5b42dde2fc3cdc1fb3650dd197f2 https://github.com/dotnet/arcade diff --git a/eng/Versions.props b/eng/Versions.props index 0b5fdf4b3319e..745a6485045cf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,8 +143,8 @@ 1.0.1-prerelease-00006 17.4.0-preview-20220707-01 - 6.0.0-prerelease.23531.3 - 6.0.0-prerelease.23531.3 + 6.0.0-prerelease.24154.4 + 6.0.0-prerelease.24154.4 6.0.0-alpha.0.24115.1 6.0.0-alpha.0.23367.3 2.4.2-pre.9 From c005d624feabb922ce600dfd108fa23aa90bad69 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:43:52 -0700 Subject: [PATCH 6/8] Fix exporting certificate keys on macOS 14.4. (#99980) Apple changed the error code we get back from a failed data-key export. This caused us to not attempt to export the key using the legacy APIs and assume the key export failed. This change adds the additional error code returned from macOS 14.4. Co-authored-by: Kevin Jones --- .../Interop.SecKeyRef.cs | 6 +- .../tests/CertTests.cs | 103 ++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs index a479faf2be559..221db64e187ab 100644 --- a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs +++ b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs @@ -127,6 +127,10 @@ internal static bool TrySecKeyCopyExternalRepresentation( { const int errSecPassphraseRequired = -25260; + // macOS Sonoma 14.4 started returning errSecInvalidKeyAttributeMask when a key could not be exported + // because it must be exported with a password. + const int errSecInvalidKeyAttributeMask = -67738; + int result = AppleCryptoNative_SecKeyCopyExternalRepresentation( key, out SafeCFDataHandle data, @@ -141,7 +145,7 @@ internal static bool TrySecKeyCopyExternalRepresentation( externalRepresentation = CoreFoundation.CFGetData(data); return true; case kErrorSeeError: - if (Interop.CoreFoundation.GetErrorCode(errorHandle) == errSecPassphraseRequired) + if (Interop.CoreFoundation.GetErrorCode(errorHandle) is errSecPassphraseRequired or errSecInvalidKeyAttributeMask) { externalRepresentation = Array.Empty(); return false; diff --git a/src/libraries/System.Security.Cryptography.X509Certificates/tests/CertTests.cs b/src/libraries/System.Security.Cryptography.X509Certificates/tests/CertTests.cs index c5e62fb835e5e..3f18fb2abd3ed 100644 --- a/src/libraries/System.Security.Cryptography.X509Certificates/tests/CertTests.cs +++ b/src/libraries/System.Security.Cryptography.X509Certificates/tests/CertTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; +using System.Security.Cryptography.X509Certificates.Tests.CertificateCreation; using System.Threading; using Microsoft.DotNet.XUnitExtensions; using Test.Cryptography; @@ -24,6 +25,108 @@ public CertTests(ITestOutputHelper output) _log = output; } + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDsa() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (ECDsa certKey = cert.GetECDsaPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_RSA() + { + using (RSA ca = RSA.Create(2048)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (RSA certKey = cert.GetRSAPrivateKey()) + { + RSAParameters certParameters = certKey.ExportParameters(true); + RSAParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.P, certParameters.P); + AssertExtensions.SequenceEqual(originalParameters.Q, certParameters.Q); + } + } + } + + [Fact] + [SkipOnPlatform(PlatformSupport.MobileAppleCrypto, "DSA is not available")] + public static void PrivateKey_FromCertificate_CanExportPrivate_DSA() + { + DSAParameters originalParameters = TestData.GetDSA1024Params(); + + using (DSA ca = DSA.Create()) + { + ca.ImportParameters(originalParameters); + DSAX509SignatureGenerator gen = new DSAX509SignatureGenerator(ca); + X500DistinguishedName dn = new X500DistinguishedName("CN=potatos"); + + CertificateRequest req = new CertificateRequest( + dn, + gen.PublicKey, + HashAlgorithmName.SHA1); + + using (X509Certificate2 cert = req.Create(dn, gen, DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3), new byte[] { 1, 2, 3 })) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ca)) + using (DSA certKey = certWithKey.GetDSAPrivateKey()) + { + DSAParameters certParameters = certKey.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.X, certParameters.X); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDiffieHellman() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + using (ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest issuerRequest = new CertificateRequest( + new X500DistinguishedName("CN=root"), + ca, + HashAlgorithmName.SHA256); + + issuerRequest.CertificateExtensions.Add( + new X509BasicConstraintsExtension(true, false, 0, true)); + + CertificateRequest request = new CertificateRequest( + new X500DistinguishedName("CN=potato"), + new PublicKey(ecdh), + HashAlgorithmName.SHA256); + + request.CertificateExtensions.Add( + new X509BasicConstraintsExtension(false, false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.KeyAgreement, true)); + + DateTimeOffset notBefore = DateTimeOffset.UtcNow; + DateTimeOffset notAfter = notBefore.AddDays(30); + byte[] serial = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + + using (X509Certificate2 issuer = issuerRequest.CreateSelfSigned(notBefore, notAfter)) + using (X509Certificate2 cert = request.Create(issuer, notBefore, notAfter, serial)) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ecdh)) + using (ECDiffieHellman certKey = certWithKey.GetECDiffieHellmanPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ecdh.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + [Fact] public static void RaceUseAndDisposeDoesNotCrash() { From 8eca07b3e8302fa55787fd36d3ebffcf898cb793 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 09:36:16 -0700 Subject: [PATCH 7/8] Update dependencies from https://github.com/dotnet/emsdk build (#100049) Microsoft.NET.Workload.Emscripten.Manifest-6.0.100 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.300 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.400 From Version 6.0.28 -> To Version 6.0.29 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NuGet.config b/NuGet.config index 77683e8299192..3d2dbe74e5cc7 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a049b84f90e95..d26b17c368ea8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -8,17 +8,17 @@ https://github.com/dotnet/msquic 7312355e44fd230b7aa26c7190f3870391751476 - + https://github.com/dotnet/emsdk - bc697dd84500aeee0bcb6323e64967222f90c2aa + 5a6205aabc468d65922f79266532e0eea9ca97ec - + https://github.com/dotnet/emsdk - bc697dd84500aeee0bcb6323e64967222f90c2aa + 5a6205aabc468d65922f79266532e0eea9ca97ec - + https://github.com/dotnet/emsdk - bc697dd84500aeee0bcb6323e64967222f90c2aa + 5a6205aabc468d65922f79266532e0eea9ca97ec https://github.com/dotnet/wcf diff --git a/eng/Versions.props b/eng/Versions.props index 745a6485045cf..868aba88dc0ce 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -176,9 +176,9 @@ 11.1.0-alpha.1.21416.1 11.1.0-alpha.1.21416.1 - 6.0.28 - 6.0.28 - 6.0.28 + 6.0.29 + 6.0.29 + 6.0.29 $(MicrosoftNETWorkloadEmscriptenManifest60100Version) 1.1.87-gba258badda From 52a5f9763b549b8d4344e3d89e30d827915bd211 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:11:35 -0700 Subject: [PATCH 8/8] Update branding to 6.0.30 (#100545) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 868aba88dc0ce..f3c034be21b91 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 6.0.29 + 6.0.30 6 0 - 29 + 30 6.0.400 servicing