From 1d1a9563bd66bbce592f2f124d17f622bd676b81 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 5 May 2021 20:31:15 -0400 Subject: [PATCH 1/8] Use sparse checkout for eng/common sync PRs --- eng/pipelines/eng-common-sync.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/eng-common-sync.yml b/eng/pipelines/eng-common-sync.yml index e32656f3f0a..ddaab8d4f78 100644 --- a/eng/pipelines/eng-common-sync.yml +++ b/eng/pipelines/eng-common-sync.yml @@ -47,6 +47,11 @@ stages: displayName: Sync ${{ parameters.DirectoryToSync }} Directory steps: + - template: /eng/common/pipelines/steps/sparse-checkout.yml + parameters: + Paths: + - ${{ parameters.DirectoryToSync }} + - pwsh: | Set-PsDebug -Trace 1 $patchDir = "$(Build.ArtifactStagingDirectory)/patchfiles" From 49aa992fa7d102002521fe221912850d97d362b3 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 5 May 2021 21:35:33 -0400 Subject: [PATCH 2/8] Use sparse checkout for directory sync stage --- eng/pipelines/eng-common-sync.yml | 10 +++---- .../templates/steps/sync-directory.yml | 27 ++++++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/eng-common-sync.yml b/eng/pipelines/eng-common-sync.yml index ddaab8d4f78..90f4be3314d 100644 --- a/eng/pipelines/eng-common-sync.yml +++ b/eng/pipelines/eng-common-sync.yml @@ -47,7 +47,7 @@ stages: displayName: Sync ${{ parameters.DirectoryToSync }} Directory steps: - - template: /eng/common/pipelines/steps/sparse-checkout.yml + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: Paths: - ${{ parameters.DirectoryToSync }} @@ -140,13 +140,13 @@ stages: Add-Content -Path "$(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}" -Value $PRData displayName: Write Sync PR Data to Artifact File condition: succeeded() - + - task: PublishPipelineArtifact@1 condition: succeeded() - displayName: Publish ${{ parameters.PRDataFileName }} + displayName: Publish ${{ parameters.PRDataFileName }} inputs: artifactName: ${{ parameters.ArtifactName }} - path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }} + path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }} - stage: VerifyAndMerge jobs: @@ -218,4 +218,4 @@ stages: DirectoryToSync: ${{ parameters.DirectoryToSync }} UpstreamBranchName: "sync-${{ parameters.DirectoryToSync }}" BaseBranchName: ${{ parameters.BaseBranchName }} - Repos: ${{ parameters.Repos }} \ No newline at end of file + Repos: ${{ parameters.Repos }} diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index ec448d75072..6859411d97c 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -12,12 +12,16 @@ parameters: steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - ${{ parameters.DirectoryToSync }} + Repositories: + - ${{ each repo in parameters.Repos }}: + - Name: Azure/${{ repo }} + WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} + - ${{ each repo in parameters.Repos }}: - - pwsh: | - Set-PsDebug -Trace 1 - git clone --branch ${{ parameters.BaseBranchName }} https://github.com/azure/${{ repo }} - displayName: Clone ${{ repo }} - workingDirectory: $(System.DefaultWorkingDirectory) - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - pwsh: | $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}" @@ -31,7 +35,7 @@ steps: { Write-Host $file.FullName git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" am -3 $file.FullName - if ($lastExitCode -ne 0) { + if ($lastExitCode -ne 0) { git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" am --show-current-patch=diff Write-Error "##vso[task.LogIssue type=warning;]Failed to properly apply patch files to [https://github.com/azure/${{ repo }}]" exit 1 @@ -121,10 +125,19 @@ steps: -AuthToken "$(azuresdk-github-pat)" - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - ${{ parameters.DirectoryToSync }} + Repositories: + - ${{ each repo in parameters.Repos }}: + - Name: Azure/${{ repo }} + Commitish: ${{ parameters.BaseBranchName }} + WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} + - ${{ each repo in parameters.Repos }}: - pwsh: | Set-PsDebug -Trace 1 - git clone --branch ${{ parameters.BaseBranchName }} https://github.com/azure/${{ repo }} $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}" if (!(Test-Path $repoPath)) { mkdir $repoPath } Remove-Item -v -r $repoPath From e8425083ecf3dbf384dbb1256f9b796104aa8222 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 5 May 2021 21:49:45 -0400 Subject: [PATCH 3/8] Use Build.SourceBranch as default base branch for eng-common-sync --- eng/pipelines/eng-common-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/eng-common-sync.yml b/eng/pipelines/eng-common-sync.yml index 90f4be3314d..9294f1203c9 100644 --- a/eng/pipelines/eng-common-sync.yml +++ b/eng/pipelines/eng-common-sync.yml @@ -11,7 +11,7 @@ parameters: default: eng/common - name: BaseBranchName type: string - default: $(Build.SourceBranchName) + default: $(Build.SourceBranch) - name: Repos type: object default: From 2d76cdff2b31b198c46cb1e5518d6cad55661ccc Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 5 May 2021 22:12:36 -0400 Subject: [PATCH 4/8] fix clone pathing --- eng/pipelines/templates/steps/sync-directory.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index 6859411d97c..883126b9b02 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -130,6 +130,9 @@ steps: Paths: - ${{ parameters.DirectoryToSync }} Repositories: + - Name: $(Build.Repository.Name) + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) - ${{ each repo in parameters.Repos }}: - Name: Azure/${{ repo }} Commitish: ${{ parameters.BaseBranchName }} @@ -139,9 +142,10 @@ steps: - pwsh: | Set-PsDebug -Trace 1 $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}" - if (!(Test-Path $repoPath)) { mkdir $repoPath } - Remove-Item -v -r $repoPath - Copy-Item -v -r $(Build.SourcesDirectory)/${{ parameters.DirectoryToSync }} $repoPath + Remove-Item -v -r -ErrorAction Ignore $repoPath + Copy-Item -v -r ` + $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.DirectoryToSync }} ` + $repoPath Get-ChildItem -r $repoPath displayName: Copy ${{ parameters.DirectoryToSync }} from azure-sdk-tools to ${{ repo }} workingDirectory: $(System.DefaultWorkingDirectory) @@ -154,4 +158,4 @@ steps: PRTitle: ${{ parameters.CommitMessage }} PushArgs: -f WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} - ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + ScriptDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/eng/common/scripts From ec414e21d3fee04af77b44b82f2b553fff3d733d Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 5 May 2021 22:18:55 -0400 Subject: [PATCH 5/8] Update sparse checkout create directory step name --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index b7dfa79a807..cf81555d775 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -20,6 +20,7 @@ steps: - pwsh: | $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}" New-Item $dir -ItemType Directory -Force + displayName: Create ${{ repo.Name }} directories - pwsh: | git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} . From 3327c77680406aa8c1448c97a13b73c0e85a58eb Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 10 May 2021 16:26:15 -0400 Subject: [PATCH 6/8] plumb BaseBranchName to create pull request --- eng/common/pipelines/templates/steps/create-pull-request.yml | 4 ++-- eng/pipelines/templates/steps/sync-directory.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml index 940d4672c86..df364d44786 100644 --- a/eng/common/pipelines/templates/steps/create-pull-request.yml +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -13,7 +13,7 @@ parameters: PRTitle: not-specified PRBody: '' ScriptDirectory: eng/common/scripts - GHReviewersVariable: '' + GHReviewersVariable: '' GHTeamReviewersVariable: '' GHAssignessVariable: '' # Multiple labels seperated by comma, e.g. "bug, APIView" @@ -33,7 +33,7 @@ steps: PushArgs: ${{ parameters.PushArgs }} WorkingDirectory: ${{ parameters.WorkingDirectory }} ScriptDirectory: ${{ parameters.ScriptDirectory }} - SkipCheckingForChanges: ${{ parameters.SkipCheckingForChanges }} + SkipCheckingForChanges: ${{ parameters.SkipCheckingForChanges }} - task: PowerShell@2 displayName: Create pull request diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index 883126b9b02..07582ac402b 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -153,6 +153,7 @@ steps: - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: RepoName: ${{ repo }} + BaseBranchName: ${{ parameters.BaseBranchName }} PRBranchName: sync-${{ parameters.DirectoryToSync }} CommitMsg: ${{ parameters.CommitMessage }} PRTitle: ${{ parameters.CommitMessage }} From ca37020361576e84856a5c264ae024d670dd9b9f Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 10 May 2021 17:15:57 -0400 Subject: [PATCH 7/8] Apply parameters everywhere in directory sync template --- eng/pipelines/templates/steps/sync-directory.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index 07582ac402b..432c8221222 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -11,6 +11,8 @@ parameters: PushArgs: -f steps: +- template: /eng/common/pipelines/templates/steps/set-default-branch.yml + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -18,11 +20,11 @@ steps: - ${{ parameters.DirectoryToSync }} Repositories: - ${{ each repo in parameters.Repos }}: - - Name: Azure/${{ repo }} + - Name: ${{ parameters.PROwner }}/${{ repo }} + Commitish: ${{ parameters.BaseBranchName }} WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} - ${{ each repo in parameters.Repos }}: - - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - pwsh: | $repoPath = "${{ repo }}/${{ parameters.DirectoryToSync }}" if (Test-Path '$(PatchFilesLocation)') @@ -134,7 +136,7 @@ steps: Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) - ${{ each repo in parameters.Repos }}: - - Name: Azure/${{ repo }} + - Name: ${{ parameters.PROwner }}/${{ repo }} Commitish: ${{ parameters.BaseBranchName }} WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} @@ -159,4 +161,4 @@ steps: PRTitle: ${{ parameters.CommitMessage }} PushArgs: -f WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }} - ScriptDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/eng/common/scripts + ScriptDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.ScriptDirectory }} From 6e74ed291abe63c5dd1757918486e60059026203 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 10 May 2021 17:19:41 -0400 Subject: [PATCH 8/8] Skip redundant checkouts --- eng/pipelines/templates/steps/sync-directory.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index 432c8221222..9a66c28f7a6 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -16,6 +16,7 @@ steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: + SkipDefaultCheckout: true Paths: - ${{ parameters.DirectoryToSync }} Repositories: @@ -129,6 +130,7 @@ steps: - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: + SkipDefaultCheckout: true Paths: - ${{ parameters.DirectoryToSync }} Repositories: