Use sparse checkout for eng/common sync PRs (#1594)

We're spending 10+ minutes sequentially cloning repos in order to create the patch files for our sync PRs, when those files are localized to a single directory. We could eliminate ~99% of this time by doing a sparse checkout instead.
Azure · May 12, 2021 · d60b181 · d60b181
1 parent 0d86533
commit d60b181
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 19 deletions.
diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml
@@ -13,7 +13,7 @@ parameters:
   PRTitle: not-specified
   PRBody: ''
   ScriptDirectory: eng/common/scripts
-  GHReviewersVariable: '' 
+  GHReviewersVariable: ''
   GHTeamReviewersVariable: ''
   GHAssignessVariable: ''
   # Multiple labels seperated by comma, e.g. "bug, APIView"
@@ -33,7 +33,7 @@ steps:
     PushArgs: ${{ parameters.PushArgs }}
     WorkingDirectory: ${{ parameters.WorkingDirectory }}
     ScriptDirectory: ${{ parameters.ScriptDirectory }}
-    SkipCheckingForChanges: ${{ parameters.SkipCheckingForChanges }} 
+    SkipCheckingForChanges: ${{ parameters.SkipCheckingForChanges }}
 
 - task: PowerShell@2
   displayName: Create pull request

diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml
@@ -20,6 +20,7 @@ steps:
     - pwsh: |
         $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}"
         New-Item $dir -ItemType Directory -Force
+      displayName: Create ${{ repo.Name }} directories
 
     - pwsh: |
         git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} .

diff --git a/eng/pipelines/eng-common-sync.yml b/eng/pipelines/eng-common-sync.yml
@@ -11,7 +11,7 @@ parameters:
   default: eng/common
 - name: BaseBranchName
   type: string
-  default: $(Build.SourceBranchName)
+  default: $(Build.SourceBranch)
 - name: Repos
   type: object
   default:
@@ -47,6 +47,11 @@ stages:
           displayName: Sync ${{ parameters.DirectoryToSync }} Directory
 
           steps:
+            - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+              parameters:
+                Paths:
+                  - ${{ parameters.DirectoryToSync }}
+
             - pwsh: |
                 Set-PsDebug -Trace 1
                 $patchDir = "$(Build.ArtifactStagingDirectory)/patchfiles"
@@ -135,13 +140,13 @@ stages:
                         Add-Content -Path "$(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}" -Value $PRData
                       displayName: Write Sync PR Data to Artifact File
                       condition: succeeded()
-                  
+
                   - task: PublishPipelineArtifact@1
                     condition: succeeded()
-                    displayName: Publish ${{ parameters.PRDataFileName }} 
+                    displayName: Publish ${{ parameters.PRDataFileName }}
                     inputs:
                       artifactName: ${{ parameters.ArtifactName }}
-                      path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }} 
+                      path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}
 
     - stage: VerifyAndMerge
       jobs:
@@ -213,4 +218,4 @@ stages:
                 DirectoryToSync: ${{ parameters.DirectoryToSync }}
                 UpstreamBranchName: "sync-${{ parameters.DirectoryToSync }}"
                 BaseBranchName: ${{ parameters.BaseBranchName }}
-                Repos: ${{ parameters.Repos }}
+                Repos: ${{ parameters.Repos }}
diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml
@@ -11,14 +11,21 @@ parameters:
   PushArgs: -f
 
 steps:
+- template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
 - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+    parameters:
+      SkipDefaultCheckout: true
+      Paths:
+        - ${{ parameters.DirectoryToSync }}
+      Repositories:
+        - ${{ each repo in parameters.Repos }}:
+          - Name: ${{ parameters.PROwner }}/${{ repo }}
+            Commitish: ${{ parameters.BaseBranchName }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }}
+
   - ${{ each repo in parameters.Repos }}:
-    - pwsh: |
-        Set-PsDebug -Trace 1
-        git clone --branch ${{ parameters.BaseBranchName }} https://github.com/azure/${{ repo }}
-      displayName: Clone ${{ repo }}
-      workingDirectory: $(System.DefaultWorkingDirectory)
-    - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
     - pwsh: |
         $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}"
         if (Test-Path '$(PatchFilesLocation)')
@@ -31,7 +38,7 @@ steps:
           {
              Write-Host $file.FullName
              git -c user.name="azure-sdk" -c user.email="[email protected]" am -3 $file.FullName
-             if ($lastExitCode -ne 0) { 
+             if ($lastExitCode -ne 0) {
                git -c user.name="azure-sdk" -c user.email="[email protected]" am --show-current-patch=diff
                Write-Error "##vso[task.LogIssue type=warning;]Failed to properly apply patch files to [https://github.com/azure/${{ repo }}]"
                exit 1
@@ -121,24 +128,39 @@ steps:
         -AuthToken "$(azuresdk-github-pat)"
 
 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+    parameters:
+      SkipDefaultCheckout: true
+      Paths:
+        - ${{ parameters.DirectoryToSync }}
+      Repositories:
+        - Name: $(Build.Repository.Name)
+          Commitish: $(Build.SourceVersion)
+          WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
+        - ${{ each repo in parameters.Repos }}:
+          - Name: ${{ parameters.PROwner }}/${{ repo }}
+            Commitish: ${{ parameters.BaseBranchName }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }}
+
   - ${{ each repo in parameters.Repos }}:
     - pwsh: |
         Set-PsDebug -Trace 1
-        git clone --branch ${{ parameters.BaseBranchName }} https://github.com/azure/${{ repo }}
         $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}"
-        if (!(Test-Path $repoPath)) { mkdir $repoPath }
-        Remove-Item -v -r $repoPath
-        Copy-Item -v -r $(Build.SourcesDirectory)/${{ parameters.DirectoryToSync }} $repoPath
+        Remove-Item -v -r -ErrorAction Ignore $repoPath
+        Copy-Item -v -r `
+          $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.DirectoryToSync }} `
+          $repoPath
         Get-ChildItem -r $repoPath
       displayName: Copy ${{ parameters.DirectoryToSync }} from azure-sdk-tools to ${{ repo }}
       workingDirectory: $(System.DefaultWorkingDirectory)
 
     - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
       parameters:
         RepoName: ${{ repo }}
+        BaseBranchName: ${{ parameters.BaseBranchName }}
         PRBranchName: sync-${{ parameters.DirectoryToSync }}
         CommitMsg: ${{ parameters.CommitMessage }}
         PRTitle: ${{ parameters.CommitMessage }}
         PushArgs: -f
         WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }}
-        ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts
+        ScriptDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.ScriptDirectory }}