CI: run helix, and non-helix tests in separate jobs (#5069)

These will the build jobs now on the public pipeline: - `Windows` - runs non-helix tests - and package signing, and workload build - `Windows_helix_tests` - runs helix tests - `Linux` - runs non-helix tests - `Linux_helix_tests` - runs helix tests Code-coverage is unaffected, and runs after all the build jobs are done. The internal pipeline is effectively unchanged, running both helix, and non-helix tests on the Windows job.
dotnet · Jul 29, 2024 · c3b47ef · c3b47ef
1 parent 7eefa05
commit c3b47ef
Show file tree

Hide file tree

Showing 3 changed files with 112 additions and 97 deletions.
diff --git a/eng/pipelines/azure-pipelines-public.yml b/eng/pipelines/azure-pipelines-public.yml
@@ -81,7 +81,8 @@ stages:
       artifacts:
         publish:
           artifacts: false
-          logs: true
+          logs:
+            name: 'Logs_Build_$(Agent.JobName)_$(_BuildConfig)'
           manifests: true
       enableMicrobuild: true
       enablePublishUsingPipelines: true
@@ -95,64 +96,69 @@ stages:
 
       jobs:
 
-      - job: Windows
-        # timeout accounts for wait times for helix agents up to 30mins
-        timeoutInMinutes: 90
-
-        pool:
-          name: $(DncEngPublicBuildPool)
-          demands: ImageOverride -equals windows.vs2022preview.amd64.open
-
-        variables:
-          - name: _buildScript
-            value: $(Build.SourcesDirectory)/build.cmd -ci
-
-        preSteps:
-          - checkout: self
-            fetchDepth: 1
-            clean: true
-
-        steps:
-          - template: /eng/pipelines/templates/BuildAndTest.yml
-            parameters:
-              runAsPublic: true
-              dotnetScript: $(Build.SourcesDirectory)/dotnet.cmd
-              buildScript: $(_buildScript)
-              buildConfig: $(_BuildConfig)
-              repoArtifactsPath: $(Build.Arcade.ArtifactsPath)
-              repoLogPath: $(Build.Arcade.LogsPath)
-              repoTestResultsPath: $(Build.Arcade.TestResultsPath)
-              isWindows: true
-
-      - job: Linux
-        # timeout accounts for wait times for helix agents up to 30mins
-        timeoutInMinutes: 90
-
-        pool:
-          name: $(DncEngPublicBuildPool)
-          demands: ImageOverride -equals build.ubuntu.2004.amd64.open
-
-        variables:
-          - name: _buildScript
-            value: $(Build.SourcesDirectory)/build.sh --ci
-
-        preSteps:
-          - checkout: self
-            fetchDepth: 1
-            clean: true
-
-        steps:
-        - template: /eng/pipelines/templates/BuildAndTest.yml
-          parameters:
-            runAsPublic: true
-            dotnetScript: $(Build.SourcesDirectory)/dotnet.sh
-            buildScript: $(_buildScript)
-            buildConfig: $(_BuildConfig)
-            repoArtifactsPath: $(Build.Arcade.ArtifactsPath)
-            repoLogPath: $(Build.Arcade.LogsPath)
-            repoTestResultsPath: $(Build.Arcade.TestResultsPath)
-            isWindows: false
-
+      - ${{ each testVariant in split(',_helix_tests', ',') }}:
+        - job: Windows${{ testVariant }}
+
+          # timeout accounts for wait times for helix agents up to 30mins
+          timeoutInMinutes: 90
+
+          pool:
+            name: $(DncEngPublicBuildPool)
+            demands: ImageOverride -equals windows.vs2022preview.amd64.open
+
+          variables:
+            - name: _buildScript
+              value: $(Build.SourcesDirectory)/build.cmd -ci
+
+          preSteps:
+            - checkout: self
+              fetchDepth: 1
+              clean: true
+
+          steps:
+            - template: /eng/pipelines/templates/BuildAndTest.yml
+              parameters:
+                runAsPublic: true
+                dotnetScript: $(Build.SourcesDirectory)/dotnet.cmd
+                buildScript: $(_buildScript)
+                buildConfig: $(_BuildConfig)
+                repoArtifactsPath: $(Build.Arcade.ArtifactsPath)
+                repoLogPath: $(Build.Arcade.LogsPath)
+                repoTestResultsPath: $(Build.Arcade.TestResultsPath)
+                isWindows: true
+                runHelixTests: ${{ contains(testVariant, 'helix') }}
+
+      - ${{ each testVariant in split(',_helix_tests', ',') }}:
+        - job: Linux${{ testVariant }}
+
+          # timeout accounts for wait times for helix agents up to 30mins
+          timeoutInMinutes: 90
+
+          pool:
+            name: $(DncEngPublicBuildPool)
+            demands: ImageOverride -equals build.ubuntu.2004.amd64.open
+
+          variables:
+            - name: _buildScript
+              value: $(Build.SourcesDirectory)/build.sh --ci
+
+          preSteps:
+            - checkout: self
+              fetchDepth: 1
+              clean: true
+
+          steps:
+            - template: /eng/pipelines/templates/BuildAndTest.yml
+              parameters:
+                runAsPublic: true
+                dotnetScript: $(Build.SourcesDirectory)/dotnet.sh
+                buildScript: $(_buildScript)
+                buildConfig: $(_BuildConfig)
+                repoArtifactsPath: $(Build.Arcade.ArtifactsPath)
+                repoLogPath: $(Build.Arcade.LogsPath)
+                repoTestResultsPath: $(Build.Arcade.TestResultsPath)
+                isWindows: false
+                runHelixTests: ${{ contains(testVariant, 'helix') }}
 
 # ----------------------------------------------------------------
 # This stage performs quality gates checks
@@ -195,3 +201,6 @@ stages:
           displayName: Init toolset
 
         - template: /eng/pipelines/templates/VerifyCoverageReport.yml
+          parameters:
+            # matches what is used in the conditions for the build/test jobs
+            testVariants: ',_helix_tests'
diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml
@@ -16,7 +16,7 @@ parameters:
     type: string
   - name: dotnetScript
     type: string
-  - name: skipTests
+  - name: runHelixTests
     type: boolean
     default: false
 
@@ -30,7 +30,7 @@ steps:
             $(_OfficialBuildIdArgs)
     displayName: Build
 
-  - ${{ if ne(parameters.skipTests, 'true') }}:
+  - ${{ if or(ne(parameters.runAsPublic, 'true'), ne(parameters.runHelixTests, 'true')) }}:
     - ${{ if ne(parameters.isWindows, 'true') }}:
       - script: mkdir ${{ parameters.repoArtifactsPath }}/devcert-scripts &&
                 cd ${{ parameters.repoArtifactsPath }}/devcert-scripts &&
@@ -55,6 +55,7 @@ steps:
 
       displayName: Run non-helix tests
 
+  - ${{ if or(ne(parameters.runAsPublic, 'true'), eq(parameters.runHelixTests, 'true')) }}:
     - script: ${{ parameters.buildScript }}
               /p:Configuration=${{ parameters.buildConfig }}
               $(_OfficialBuildIdArgs)
@@ -86,29 +87,30 @@ steps:
         HelixBuild: $(Build.BuildNumber)
         HelixAccessToken: $(HelixApiAccessToken)
 
-    - task: CopyFiles@2
-      inputs:
-        Contents: '${{ parameters.repoArtifactsPath }}/**/*.cobertura.xml'
-        TargetFolder: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
-        flattenFolders: true
-      displayName: Gather code coverage results
+  - task: CopyFiles@2
+    inputs:
+      Contents: '${{ parameters.repoArtifactsPath }}/**/*.cobertura.xml'
+      TargetFolder: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
+      flattenFolders: true
+    displayName: Gather code coverage results
 
-    - ${{ if eq(parameters.runAsPublic, 'true') }}:
-      - task: PublishPipelineArtifact@1
-        displayName: Publish coverage results (cobertura.xml)
-        inputs:
-          targetPath: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
-          artifactName: '$(Agent.JobName)_CodeCoverageResults'
-          publishLocation: 'pipeline'
+  - ${{ if eq(parameters.runAsPublic, 'true') }}:
+    - task: PublishPipelineArtifact@1
+      displayName: Publish coverage results (cobertura.xml)
+      inputs:
+        targetPath: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
+        artifactName: '$(Agent.JobName)_CodeCoverageResults'
+        publishLocation: 'pipeline'
 
-    - ${{ if ne(parameters.runAsPublic, 'true') }}:
-      - task: 1ES.PublishPipelineArtifact@1
-        displayName: Publish code coverage results
-        inputs:
-          targetPath: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
-          artifactName: '$(Agent.JobName)_CodeCoverageResults'
+  - ${{ if ne(parameters.runAsPublic, 'true') }}:
+    - task: 1ES.PublishPipelineArtifact@1
+      displayName: Publish code coverage results
+      inputs:
+        targetPath: '${{ parameters.repoArtifactsPath }}/CodeCoverage'
+        artifactName: '$(Agent.JobName)_CodeCoverageResults'
 
-  - ${{ if eq(parameters.isWindows, 'true') }}:
+  # Run on windows, for internal pipeline, or public+non-helix-tests job
+  - ${{ if and(eq(parameters.isWindows, 'true'), or(ne(parameters.runAsPublic, 'true'), ne(parameters.runHelixTests, 'true'))) }}:
     - script: ${{ parameters.buildScript }}
               -pack
               -sign $(_SignArgs)

diff --git a/eng/pipelines/templates/VerifyCoverageReport.yml b/eng/pipelines/templates/VerifyCoverageReport.yml
@@ -1,22 +1,26 @@
+parameters:
+  testVariants: ''
+
 steps:
 
-  # This downloads *cobertura.xml from Windows_CodeCoverageResults artifact to the root of the repo
-  - task: DownloadPipelineArtifact@2
-    displayName: Download Windows code coverage reports
-    inputs:
-      buildType: 'current'
-      artifactName: Windows_CodeCoverageResults
-      itemPattern: '*.cobertura.xml'
-      targetPath: $(Build.SourcesDirectory)/Windows_CodeCoverageResults
+  - ${{ each testVariant in split(parameters.testVariants, ',') }}:
+    # This downloads *cobertura.xml from Windows_CodeCoverageResults artifact to the root of the repo
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Windows${{ testVariant }} code coverage reports
+      inputs:
+        buildType: 'current'
+        artifactName: Windows${{ testVariant }}_CodeCoverageResults
+        itemPattern: '*.cobertura.xml'
+        targetPath: $(Build.SourcesDirectory)/Windows_CodeCoverageResults
 
-  # This downloads *cobertura.xml from Linux_CodeCoverageResults artifact to the root of the repo
-  - task: DownloadPipelineArtifact@2
-    displayName: Download Linux code coverage reports
-    inputs:
-      buildType: 'current'
-      artifactName: Linux_CodeCoverageResults
-      itemPattern: '*.cobertura.xml'
-      targetPath: $(Build.SourcesDirectory)/Linux_CodeCoverageResults
+    # This downloads *cobertura.xml from Linux_CodeCoverageResults artifact to the root of the repo
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Linux${{ testVariant }} code coverage reports
+      inputs:
+        buildType: 'current'
+        artifactName: Linux${{ testVariant }}_CodeCoverageResults
+        itemPattern: '*.cobertura.xml'
+        targetPath: $(Build.SourcesDirectory)/Linux_CodeCoverageResults
 
   # Merge the downloaded files (Windows_cobertura.xml and Linux_cobertura.xml) as one (merged.cobertura.xml)
   - script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage merge