[EngSys] Combine Build Build and Build Test pipelines (#15085)

* only finds tests with a Test method * adding conditional to ci.yml files * removing the need for build.yml file * changing files to use one archetype-sdk-client.yml file * bad conditional * passing parameter through * checking for explicit true * moving coverage installation into test section * had to fix profiles ci.yml file * fixing hang up in gofmt when there are no go files in dir * changing echo to fmt * error check on samples * removing build.yml file * bens comments and lint * undoing change * simplifying
Azure · Jul 21, 2021 · 1af91d4 · 1af91d4
1 parent e3f76d0
commit 1af91d4
Show file tree

Hide file tree

Showing 26 changed files with 108 additions and 132 deletions.
diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml b/eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
@@ -42,12 +42,12 @@ stages:
       - template: ../steps/create-go-workspace.yml
 
       - template: ../steps/set-scope.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
 
-      - template: ../steps/build.yml
-        parameters: 
+      - template: ../steps/build-test.yml
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
           Scope: $(SCOPE)
@@ -70,18 +70,18 @@ stages:
       - template: ../steps/create-go-workspace.yml
 
       - template: ../steps/set-scope.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
 
       - template: ../steps/analyze.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
           Scope: $(SCOPE)
           LintVersion: $(GoLintCLIVersion)
 
-# Below stage won't work until the release stage is added/necessary. 
+# Below stage won't work until the release stage is added/necessary.
 # "Releasing" is just the package in the repository on github, but there may be some other metadata related
 # tasks that become necessary later on.
 

diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml
@@ -1,5 +1,6 @@
 parameters:
   ServiceDirectory: ''
+  RunTests: false
 
 stages:
   - stage: Build
@@ -42,17 +43,18 @@ stages:
       - template: ../steps/create-go-workspace.yml
 
       - template: ../steps/set-scope.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
 
       - template: ../steps/build-test.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
           Scope: $(SCOPE)
           Image: $(vm.image)
           GoVersion: $(go.version)
+          RunTests: ${{ parameters.RunTests }}
 
     - job: Analyze
       displayName: Analyze
@@ -70,18 +72,18 @@ stages:
       - template: ../steps/create-go-workspace.yml
 
       - template: ../steps/set-scope.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
 
       - template: ../steps/analyze.yml
-        parameters: 
+        parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GoWorkspace: $(GO_WORKSPACE_PATH)
           Scope: $(SCOPE)
           LintVersion: $(GoLintCLIVersion)
 
-# Below stage won't work until the release stage is added/necessary. 
+# Below stage won't work until the release stage is added/necessary.
 # "Releasing" is just the package in the repository on github, but there may be some other metadata related
 # tasks that become necessary later on.
 

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
@@ -37,9 +37,9 @@ steps:
     failOnStderr: true
     workingDirectory: 'sdk/${{parameters.ServiceDirectory}}'
 
-  - script: |
-      echo check source file formatting in $(pwd)
-      gofmt -s -l -d $(find . -name '*.go' -print) >&2
+  - pwsh: |
+      Write-Host "##[command]Check source file formatting in $(pwd)"
+      gofmt -s -l -d .
     displayName: 'Format Check'
     condition: succeededOrFailed()
     failOnStderr: true

diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml
@@ -4,86 +4,76 @@ parameters:
   Scope: 'sdk/...'
   Image: ''
   GoVersion: ''
+  RunTests: false
 
 steps:
 
-  - pwsh: |
-      go get github.com/jstemmer/go-junit-report
-      go get github.com/axw/gocov/gocov
-      go get github.com/AlekSi/gocov-xml
-      go get github.com/matm/gocov-html
-      go get -u github.com/wadey/gocovmerge
-    displayName: "Install Coverage and Junit Dependencies"
-    workingDirectory: '${{parameters.GoWorkspace}}'
-
-  - pwsh: |
-      $modDirs = (./eng/scripts/get_module_dirs.ps1 -serviceDir $(SCOPE))
-      foreach ($md in $modDirs) {
-        pushd $md
-        Write-Host "##[command]Executing go build -v ./... in $md"
-        go build -v ./...
-      }
+  - pwsh: ./eng/scripts/build.ps1
     displayName: 'Build'
     workingDirectory: '${{parameters.GoWorkspace}}'
     env:
       GO111MODULE: 'on'
 
-  - pwsh: |
-      $modDirs = (./eng/scripts/get_module_dirs.ps1 -serviceDir $(SCOPE))
-      foreach ($md in $modDirs) {
-        pushd $md
-        Write-Host "##[command]Executing go vet ./... in $md"
-        go vet ./...
-      }
+  - pwsh: ./eng/scripts/build.ps1 -vet -skipBuild
     displayName: 'Vet'
     workingDirectory: '${{parameters.GoWorkspace}}'
     env:
       GO111MODULE: 'on'
 
-  - pwsh: |
-      $testDirs = (./eng/scripts/get_test_dirs.ps1 -serviceDir $(SCOPE))
-      foreach ($td in $testDirs) {
-        pushd $td
-        Write-Host "##[command]Executing go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic $td | go-junit-report -set-exit-code > report.xml"
-        go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic . | go-junit-report -set-exit-code > report.xml
-        # if no tests were actually run (e.g. examples) delete the coverage file so it's omitted from the coverage report
-        if (Select-String -path ./report.xml -pattern '<testsuites></testsuites>' -simplematch -quiet) {
-          Write-Host "##[command]Deleting empty coverage file"
-          rm coverage.txt
+  - ${{ if eq(parameters.RunTests, 'true') }}:
+    - pwsh: |
+        go get github.com/jstemmer/go-junit-report
+        go get github.com/axw/gocov/gocov
+        go get github.com/AlekSi/gocov-xml
+        go get github.com/matm/gocov-html
+        go get -u github.com/wadey/gocovmerge
+      displayName: "Install Coverage and Junit Dependencies"
+      workingDirectory: '${{parameters.GoWorkspace}}'
+
+    - pwsh: |
+        $testDirs = (./eng/scripts/get_test_dirs.ps1 -serviceDir $(SCOPE))
+        foreach ($td in $testDirs) {
+          pushd $td
+          Write-Host "##[command]Executing go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic $td | go-junit-report -set-exit-code > report.xml"
+          go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic . | go-junit-report -set-exit-code > report.xml
+          # if no tests were actually run (e.g. examples) delete the coverage file so it's omitted from the coverage report
+          if (Select-String -path ./report.xml -pattern '<testsuites></testsuites>' -simplematch -quiet) {
+            Write-Host "##[command]Deleting empty coverage file"
+            rm coverage.txt
+          }
         }
-      }
-    displayName: 'Run Tests'
-    workingDirectory: '${{parameters.GoWorkspace}}'
-    env:
-      GO111MODULE: 'on'
+      displayName: 'Run Tests'
+      workingDirectory: '${{parameters.GoWorkspace}}'
+      env:
+        GO111MODULE: 'on'
 
-  - pwsh: |
-      $coverageFiles = [Collections.Generic.List[String]]@()
-      Get-Childitem -recurse -path $(SCOPE) -filter coverage.txt | foreach-object {
-        $covFile = $_.FullName
-        Write-Host "Adding $covFile to the list of code coverage files"
-        $coverageFiles.Add($covFile)
-      }
-      gocovmerge $coverageFiles > mergedCoverage.txt
-      gocov convert ./mergedCoverage.txt > ./coverage.json
-      # gocov converts rely on standard input
-      Get-Content ./coverage.json | gocov-xml > ./coverage.xml
-      Get-Content ./coverage.json | gocov-html > ./coverage.html
-    displayName: 'Generate Coverage XML'
-    workingDirectory: '${{parameters.GoWorkspace}}sdk'
+    - pwsh: |
+        $coverageFiles = [Collections.Generic.List[String]]@()
+        Get-Childitem -recurse -path $(SCOPE) -filter coverage.txt | foreach-object {
+          $covFile = $_.FullName
+          Write-Host "Adding $covFile to the list of code coverage files"
+          $coverageFiles.Add($covFile)
+        }
+        gocovmerge $coverageFiles > mergedCoverage.txt
+        gocov convert ./mergedCoverage.txt > ./coverage.json
+        # gocov converts rely on standard input
+        Get-Content ./coverage.json | gocov-xml > ./coverage.xml
+        Get-Content ./coverage.json | gocov-html > ./coverage.html
+      displayName: 'Generate Coverage XML'
+      workingDirectory: '${{parameters.GoWorkspace}}sdk'
 
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    inputs:
-      testRunner: JUnit
-      testResultsFiles: '${{parameters.GoWorkspace}}sdk/**/report.xml'
-      testRunTitle: 'Go ${{ parameters.GoVersion }} on ${{ parameters.Image }}'
-      failTaskOnFailedTests: true
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testRunner: JUnit
+        testResultsFiles: '${{parameters.GoWorkspace}}sdk/**/report.xml'
+        testRunTitle: 'Go ${{ parameters.GoVersion }} on ${{ parameters.Image }}'
+        failTaskOnFailedTests: true
 
-  - task: PublishCodeCoverageResults@1
-    condition: succeededOrFailed()
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: '${{parameters.GoWorkspace}}sdk/coverage.xml'
-      additionalCodeCoverageFiles: '${{parameters.GoWorkspace}}sdk/coverage.html'
-      failIfCoverageEmpty: true
+    - task: PublishCodeCoverageResults@1
+      condition: succeededOrFailed()
+      inputs:
+        codeCoverageTool: Cobertura
+        summaryFileLocation: '${{parameters.GoWorkspace}}sdk/coverage.xml'
+        additionalCodeCoverageFiles: '${{parameters.GoWorkspace}}sdk/coverage.html'
+        failIfCoverageEmpty: true
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
diff --git a/eng/scripts/build.ps1 b/eng/scripts/build.ps1
@@ -19,9 +19,9 @@ foreach ($sdk in (./eng/scripts/get_module_dirs.ps1 -serviceDir 'sdk/...')) {
 }
 
 $keys = $sdks.Keys | Sort-Object;
-if (![string]::IsNullOrWhiteSpace($filter)) { 
+if (![string]::IsNullOrWhiteSpace($filter)) {
     Write-Host "Using filter: $filter"
-    $keys = $keys.Where( { $_ -match $filter }) 
+    $keys = $keys.Where( { $_ -match $filter })
 }
 
 $keys | ForEach-Object { $sdks[$_] } | ForEach-Object {

diff --git a/eng/scripts/get_test_dirs.ps1 b/eng/scripts/get_test_dirs.ps1
@@ -7,9 +7,13 @@ $testDirs = [Collections.Generic.List[String]]@()
 # find each directory under $serviceDir that contains Go test files
 Get-Childitem -recurse -path $serviceDir -filter *_test.go | foreach-object {
   $cdir = $_.Directory
-  if (!$testDirs.Contains($cdir)) {
-    Write-Host "Adding $cdir to list of test directories"
-    $testDirs.Add($cdir)
+  $tests = Select-String -Path $_ 'Test' -AllMatches
+
+  if ($tests.Count -gt 0) {
+    if (!$testDirs.Contains($cdir)) {
+      Write-Host "Adding $cdir to list of test directories"
+      $testDirs.Add($cdir)
+    }
   }
 }
 

diff --git a/sdk/agfood/armagfood/ci.yml b/sdk/agfood/armagfood/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/agfood/armagfood/
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'agfood/armagfood'
diff --git a/sdk/agrifood/armagrifood/ci.yml b/sdk/agrifood/armagrifood/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/agrifood/armagrifood
 
 stages:
-- template: /eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'agrifood/armagrifood'
diff --git a/sdk/authorization/armauthorization/ci.yml b/sdk/authorization/armauthorization/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/authorization/armauthorization
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'authorization/armauthorization'
diff --git a/sdk/azcore/ci.yml b/sdk/azcore/ci.yml
@@ -8,9 +8,10 @@ pr:
   paths:
     include:
     - sdk/azcore/
-    
-    
+
+
 stages:
 - template: ../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'azcore'
+    RunTests: true
diff --git a/sdk/azidentity/ci.yml b/sdk/azidentity/ci.yml
@@ -13,3 +13,4 @@ stages:
 - template: ../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'azidentity'
+    RunTests: true
diff --git a/sdk/compute/armcompute/ci.yml b/sdk/compute/armcompute/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/compute/armcompute/
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'compute/armcompute'
diff --git a/sdk/containerregistry/armcontainerregistry/ci.yml b/sdk/containerregistry/armcontainerregistry/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/containerregistry/armcontainerregistry/
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'containerregistry/armcontainerregistry'
diff --git a/sdk/containerservice/armcontainerservice/ci.yml b/sdk/containerservice/armcontainerservice/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/containerservice/armcontainerservice
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'containerservice/armcontainerservice'
diff --git a/sdk/keyvault/armkeyvault/ci.yml b/sdk/keyvault/armkeyvault/ci.yml
@@ -10,6 +10,6 @@ pr:
       - sdk/keyvault/armkeyvault
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'keyvault/armkeyvault'
diff --git a/sdk/monitor/armmonitor/ci.yml b/sdk/monitor/armmonitor/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/monitor/armmonitor
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'monitor/armmonitor'
diff --git a/sdk/network/armnetwork/ci.yml b/sdk/network/armnetwork/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/network/armnetwork/
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'network/armnetwork'
diff --git a/sdk/privatedns/armprivatedns/ci.yml b/sdk/privatedns/armprivatedns/ci.yml
@@ -10,6 +10,6 @@ pr:
     - sdk/privatedns/armprivatedns
 
 stages:
-- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client-samples.yml
+- template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: 'privatedns/armprivatedns'