[6.0.4xx] Migrate to 1ES templates for internal builds

.vsts-pr.yml

@@ -21,7 +21,6 @@ variables:
   - group: DotNet-Installer-SDLValidation-Params
   - name: _PublishUsingPipelines
     value: true
-
 # Default to running tests in PRs and public CI, but not in official builds
 - name: _WindowsTestArg
   value: '-test'
@@ -32,10 +31,8 @@ variables:
     value: ''
   - name: _NonWindowsTestArg
     value: ''
-
 - name: _InternalRuntimeDownloadArgs
   value: ''
-
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNetBuilds storage account read tokens
   - name: _InternalRuntimeDownloadArgs
@@ -58,7 +55,7 @@ stages:
     - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration
       artifact: BuildConfiguration
       displayName: Publish Build Config
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Windows_NT
       pool:
@@ -115,7 +112,7 @@ stages:
             # Never run tests on arm64
             _TestArg: ''
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Windows_NT
       pool:
@@ -144,7 +141,7 @@ stages:
             _TestArg: ''
       pgoInstrument: true
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Linux
       pool:
@@ -330,7 +327,7 @@ stages:
             _BuildArchitecture: 'x64'
             _TestArg: $(_NonWindowsTestArg)
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Linux
       pool:
@@ -353,7 +350,7 @@ stages:
             _TestArg: ''
       pgoInstrument: true
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Darwin
       pool:
@@ -375,26 +372,7 @@ stages:
 
   - template: /eng/common/templates/jobs/source-build.yml
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
-
-  # For .NET 6.0, source-build only supports 6.0.1xx.  Disable tarball build validation in all other branches.
-  # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-  #   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
-  #     parameters:
-  #       dependsOn: Source_Build_Create_Tarball
-
-  # https://github.com/dotnet/core-sdk/issues/248
-  # - template: /eng/build.yml
-  #   parameters:
-  #     agentOs: FreeBSD
-  #     queue:
-  #       name: dnceng-freebsd-internal
-  #       timeoutInMinutes: 180
-  #       matrix:
-  #         Build_Release:
-  #           _BuildConfig: Release
-  #           _BuildArchitecture: 'x64'
-  #           _AdditionalBuildParameters: '/p:DisableSourceLink=true /p:DISABLE_CROSSGEN=true'
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/job/publish-build-assets.yml
@@ -413,7 +391,6 @@ stages:
             name: NetCore1ESPool-Svc-Internal
             demands: ImageOverride -equals windows.vs2017.amd64
 
-
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng\common\templates\post-build\post-build.yml
     parameters:

eng/build-pr.yml

@@ -0,0 +1,166 @@
+parameters:
+  # Agent OS identifier and used as job name
+  agentOs: ''
+
+  # Agent pool
+  pool: {}
+
+  # Additional variables
+  variables: {}
+
+  # Build strategy - matrix
+  strategy: {}
+
+  # Job timeout
+  timeoutInMinutes: 180
+
+  # Publish using pipelines
+  enablePublishUsingPipelines: true
+
+phases:
+- template: /eng/common/templates/job/job.yml
+  parameters:
+    ${{ if parameters.pgoInstrument }}:
+      name: PGO_${{ parameters.agentOs }}
+    ${{ if not(parameters.pgoInstrument) }}:
+      name: ${{ parameters.agentOs }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      enableMicrobuild: true
+    enablePublishBuildAssets: true
+    enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
+    enableTelemetry: true
+    helixRepo: dotnet/installer
+    pool: ${{ parameters.pool }}
+    ${{ if ne(parameters.strategy, '') }}:
+      strategy: ${{ parameters.strategy }}
+    workspace:
+      clean: all
+
+    variables:
+      - _PgoInstrument: ''
+      - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+        - _PackArg: '-pack'
+      - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+        - _PackArg: '--pack'
+      - ${{ if parameters.pgoInstrument }}:
+        - _PgoInstrument: '/p:PgoInstrument=true'
+        - _PackArg: ''
+      - _AgentOSName: ${{ parameters.agentOs }}
+      - _TeamName: Roslyn-Project-System
+      - _SignType: test
+      - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        - group: DotNet-Symbol-Server-PATs
+        - group: DotNet-HelixApi-Access
+        - group: DotNet-Blob-Feed
+        - _DotNetPublishToBlobFeed: true
+        - _PushToVSFeed: true
+        - _SignType: real
+        - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+                  /p:DotNetSignType=$(_SignType)
+                  /p:TeamName=$(_TeamName)
+                  /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+                  $(_PgoInstrument)
+
+    steps:
+    - checkout: self
+      clean: true
+    - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - script: build.cmd
+                  $(_TestArg) $(_PackArg)
+                  -publish -ci -sign
+                  -Configuration $(_BuildConfig)
+                  -Architecture $(_BuildArchitecture)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+        env:
+          DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE)
+
+    - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+    - ${{ if eq(parameters.agentOs, 'Linux') }}:
+      - script: ./build.sh
+                  $(_TestArg) $(_PackArg)
+                  --publish --ci
+                  --noprettyprint
+                  --configuration $(_BuildConfig)
+                  $(_DockerParameter)
+                  --architecture $(_BuildArchitecture)
+                  $(_LinuxPortable)
+                  $(_RuntimeIdentifier)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+
+    - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
+      - script: ./build.sh
+                  $(_TestArg)
+                  --pack --publish --ci
+                  --noprettyprint
+                  --configuration $(_BuildConfig)
+                  --architecture $(_BuildArchitecture)
+                  $(_RuntimeIdentifier)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT'), ne(variables['PostBuildSign'], 'true')) }}:
+      - task: NuGetCommand@2
+        displayName: Push Visual Studio NuPkgs
+        inputs:
+          command: push
+          packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping/VS.*.nupkg'
+          nuGetFeedType: external
+          publishFeedCredentials: 'DevDiv - VS package feed'
+        condition: and(succeeded(), ne(variables['PostBuildSign'], true), eq(variables['_PushToVSFeed'], 'true'), eq(variables['_DotNetPublishToBlobFeed'], 'true'), or(eq(variables['_BuildArchitecture'], 'x64'), eq(variables['_BuildArchitecture'], 'x86')))
+
+    - task: PublishTestResults@2
+      displayName: Publish Test Results
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
+        testRunTitle: '$(_AgentOSName)_$(Agent.JobName)'
+        platform: '$(BuildPlatform)'
+        configuration: '$(_BuildConfig)'
+      condition: ne(variables['_TestArg'], '')
+
+    - task: CopyFiles@2
+      displayName: Gather Logs
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)/artifacts'
+        Contents: |
+          log/$(_BuildConfig)/**/*
+          TestResults/$(_BuildConfig)/**/*
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+      continueOnError: true
+      condition: always()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Logs to VSTS
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)'
+        publishLocation: Container
+      continueOnError: true
+      condition: always()

eng/build.yml

@@ -18,7 +18,7 @@ parameters:
   enablePublishUsingPipelines: true
 
 phases:
-- template: /eng/common/templates/job/job.yml
+- template: /eng/common/templates-official/job/job.yml
   parameters:
     ${{ if parameters.pgoInstrument }}:
       name: PGO_${{ parameters.agentOs }}
@@ -28,9 +28,6 @@ phases:
     ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
       enableMicrobuild: true
     enablePublishBuildAssets: true
-    # Don't use built-in arcade logic for publishing artifacts and test results, as we manually do this and give better names
-    #enablePublishBuildArtifacts: true
-    #enablePublishTestResults: true
     enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
     enableTelemetry: true
     helixRepo: dotnet/installer
@@ -49,12 +46,10 @@ phases:
       - ${{ if parameters.pgoInstrument }}:
         - _PgoInstrument: '/p:PgoInstrument=true'
         - _PackArg: ''
-
       - _AgentOSName: ${{ parameters.agentOs }}
       - _TeamName: Roslyn-Project-System
       - _SignType: test
       - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
-
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
         - group: DotNet-HelixApi-Access
@@ -101,6 +96,7 @@ phases:
             arguments: $(Build.SourcesDirectory)/NuGet.config $Token
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
     - ${{ if eq(parameters.agentOs, 'Linux') }}:
       - script: ./build.sh
                   $(_TestArg) $(_PackArg)
@@ -130,10 +126,9 @@ phases:
         displayName: Build
 
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT'), ne(variables['PostBuildSign'], 'true')) }}:
-      - task: NuGetCommand@2
+      - task: 1ES.PublishNuget@1
         displayName: Push Visual Studio NuPkgs
         inputs:
-          command: push
           packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping/VS.*.nupkg'
           nuGetFeedType: external
           publishFeedCredentials: 'DevDiv - VS package feed'
@@ -154,13 +149,13 @@ phases:
       inputs:
         SourceFolder: '$(Build.SourcesDirectory)/artifacts'
         Contents: |
-         log/$(_BuildConfig)/**/*
-         TestResults/$(_BuildConfig)/**/*
+          log/$(_BuildConfig)/**/*
+          TestResults/$(_BuildConfig)/**/*
         TargetFolder: '$(Build.ArtifactStagingDirectory)'
       continueOnError: true
       condition: always()
 
-    - task: PublishBuildArtifacts@1
+    - task: 1ES.PublishBuildArtifacts@1
       displayName: Publish Logs to VSTS
       inputs:
         PathtoPublish: '$(Build.ArtifactStagingDirectory)'

eng/source-build-tarball-build-official.yml

@@ -1,5 +1,6 @@
 trigger: none
 pr: none
+
 resources:
   pipelines:
   - pipeline: installer-build-resource
@@ -17,10 +18,25 @@ resources:
         - internal/release/6.0.4xx
       stages:
       - build
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
 
-stages:
-- stage: build
-  jobs:
-  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
-    parameters:
-      installerBuildResourceId: installer-build-resource
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    sdl:
+      sourceAnalysisPool:
+        name: NetCore1ESPool-Svc-Internal
+        image: 1es-windows-2022
+        os: windows
+    customBuildTags:
+    - ES365AIMigrationTooling
+    stages:
+    - stage: build
+      jobs:
+      - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml@self
+        parameters:
+          installerBuildResourceId: installer-build-resource