diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index b580d73f1406..b875a806b143 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -13,7 +13,7 @@ parameters: # "TestApplicationId": "", # "TestApplicationSecret": "", # "ProvisionerApplicationId": "", -# "ProvisoinerApplicationSecret": "", +# "ProvisionerApplicationSecret": "", # "Environment": "AzureCloud | AzureGov | AzureChina | " # } diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index 1565e7d4bd0f..767f8c8c516b 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -32,5 +32,5 @@ steps: -Force ` -Verbose displayName: Remove test resources - condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed()) + condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '') continueOnError: true diff --git a/eng/common/pipelines/templates/steps/cosmos-emulator.yml b/eng/common/pipelines/templates/steps/cosmos-emulator.yml new file mode 100644 index 000000000000..564f3e429bd7 --- /dev/null +++ b/eng/common/pipelines/templates/steps/cosmos-emulator.yml @@ -0,0 +1,16 @@ +parameters: + EmulatorMsiUrl: "https://aka.ms/cosmosdb-emulator" + StartParameters: '' + +steps: + - powershell: | + $targetDir = $env:temp + Write-Host "Downloading and extracting Cosmos DB Emulator - ${{ parameters.EmulatorMsiUrl }}" + Write-Host "Target Dir: $targetDir" + msiexec /a ${{ parameters.EmulatorMsiUrl }} TARGETDIR=$targetDir /qn | wait-process + displayName: Download and Extract Public Cosmos DB Emulator + - powershell: | + Write-Host "Launching Cosmos DB Emulator" + Import-Module "$env:temp\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator" + Start-CosmosDbEmulator -NoUI ${{ parameters.StartParameters }} + displayName: Start Cosmos DB Emulator \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml index 790de92bcbe5..10af61de1100 100644 --- a/eng/common/pipelines/templates/steps/create-pull-request.yml +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -12,6 +12,8 @@ parameters: WorkingDirectory: $(System.DefaultWorkingDirectory) PRTitle: not-specified ScriptDirectory: eng/common/scripts + GHReviewersVariable: '' + GHTeamReviewersVariable: '' steps: @@ -63,3 +65,18 @@ steps: -PRBranch "${{ parameters.PRBranchName }}" -AuthToken "$(azuresdk-github-pat)" -PRTitle "${{ parameters.PRTitle }}" + +- task: PowerShell@2 + displayName: Tag a Reviewer on PR + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: ${{ parameters.WorkingDirectory }} + filePath: ${{ parameters.ScriptDirectory }}/add-pullrequest-reviewers.ps1 + arguments: > + -RepoOwner "${{ parameters.RepoOwner }}" + -RepoName "${{ parameters.RepoName }}" + -AuthToken "$(azuresdk-github-pat)" + -GitHubUsers "$(${{ parameters.GHReviewersVariable }})" + -GitHubTeams "$(${{ parameters.GHTeamReviewersVariable }})" + -PRNumber "$(Submitted.PullRequest.Number)" diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index 6ff9f84ff3ee..a9ff8c4e17b4 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -12,6 +12,8 @@ parameters: ArtifactName: '' Language: '' DocRepoDestinationPath: '' #usually docs-ref-services/ + GHReviewersVariable: '' + GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step steps: - pwsh: | @@ -56,3 +58,5 @@ steps: BaseBranchName: smoke-test WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} + GHReviewersVariable: ${{ parameters.GHReviewersVariable }} + GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} diff --git a/eng/common/pipelines/templates/steps/get-pr-owners.yml b/eng/common/pipelines/templates/steps/get-pr-owners.yml new file mode 100644 index 000000000000..a80d5b83b2de --- /dev/null +++ b/eng/common/pipelines/templates/steps/get-pr-owners.yml @@ -0,0 +1,46 @@ +parameters: + TargetVariable: '' + ServiceDirectory: '' + +steps: + - pwsh: | + git clone https://github.com/Azure/azure-sdk-tools.git $(Build.SourcesDirectory)/tools_repo + cd $(Build.SourcesDirectory)/tools_repo + git checkout 564ad63ae72d18422533fa1da9d396e7703c1cb5 + displayName: Setup Identity Resolver + + - pwsh: | + $result = dotnet run -v q -- ` + --aad-app-id-var APP_ID ` + --aad-app-secret-var APP_SECRET ` + --aad-tenant-var AAD_TENANT ` + --kusto-url-var KUSTO_URL ` + --kusto-database-var KUSTO_DB ` + --kusto-table-var KUSTO_TABLE ` + --identity "$(Build.QueuedBy)" + $resolvedIdentity = $result[-1] | ConvertFrom-Json + + Write-Host $resolvedIdentity + + Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)" + displayName: 'Resolving Queuing User' + workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver + env: + APP_ID: $(notification-aad-app-id) + APP_SECRET: $(notification-aad-secret) + AAD_TENANT: $(notification-aad-tenant) + KUSTO_URL: $(notification-kusto-url) + KUSTO_DB: $(notification-kusto-db) + KUSTO_TABLE: $(notification-kusto-table) + + - pwsh: | + Remove-Item -Force -Recurse $(Build.SourcesDirectory)/tools_repo + displayName: Clean Up Cloned Tools Repo + + - pwsh: | + $originalValue = "$(${{ parameters.TargetVariable }})" + $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory) + if ($result) { + Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result" + } + displayName: Add CodeOwners if Present \ No newline at end of file diff --git a/eng/common/scripts/Submit-PullRequest.ps1 b/eng/common/scripts/Submit-PullRequest.ps1 index ef2a6f545061..5edabc599a99 100644 --- a/eng/common/scripts/Submit-PullRequest.ps1 +++ b/eng/common/scripts/Submit-PullRequest.ps1 @@ -58,6 +58,9 @@ $resp | Write-Verbose if ($resp.Count -gt 0) { Write-Host -f green "Pull request already exists $($resp[0].html_url)" + + # setting variable to reference the pull request by number + Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp[0].number)" } else { $data = @{ @@ -80,4 +83,7 @@ else { $resp | Write-Verbose Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)" + + # setting variable to reference the pull request by number + Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp.number)" } diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 index c78ce9020ebb..6b5b5606561b 100644 --- a/eng/common/scripts/Verify-Links.ps1 +++ b/eng/common/scripts/Verify-Links.ps1 @@ -125,7 +125,18 @@ function CheckLink ([System.Uri]$linkUri) } else { try { - $response = Invoke-WebRequest -Uri $linkUri + $headRequestSucceeded = $true + try { + # Attempt HEAD request first + $response = Invoke-WebRequest -Uri $linkUri -Method HEAD + } + catch { + $headRequestSucceeded = $false + } + if (!$headRequestSucceeded) { + # Attempt a GET request if the HEAD request failed. + $response = Invoke-WebRequest -Uri $linkUri -Method GET + } $statusCode = $response.StatusCode if ($statusCode -ne 200) { Write-Host "[$statusCode] while requesting $linkUri" diff --git a/eng/common/scripts/add-pullrequest-reviewers.ps1 b/eng/common/scripts/add-pullrequest-reviewers.ps1 new file mode 100644 index 000000000000..00460ce701f0 --- /dev/null +++ b/eng/common/scripts/add-pullrequest-reviewers.ps1 @@ -0,0 +1,79 @@ +param( + [Parameter(Mandatory = $true)] + $RepoOwner, + + [Parameter(Mandatory = $true)] + $RepoName, + + [Parameter(Mandatory = $false)] + $GitHubUsers = "", + + [Parameter(Mandatory = $false)] + $GitHubTeams = "", + + [Parameter(Mandatory = $true)] + $PRNumber, + + [Parameter(Mandatory = $true)] + $AuthToken +) + +# at least one of these needs to be populated +if (-not $GitHubUsers -and -not $GitHubTeams) { + Write-Host "No user provided for addition, exiting." + exit 0 +} + +$userAdditions = @($GitHubUsers.Split(",") | % { $_.Trim() } | ? { return $_ }) +$teamAdditions = @($GitHubTeams.Split(",") | % { $_.Trim() } | ? { return $_ }) + +$headers = @{ + Authorization = "bearer $AuthToken" +} +$uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers" + +try { + $resp = Invoke-RestMethod -Headers $headers $uri -MaximumRetryCount 3 +} +catch { + Write-Error "Invoke-RestMethod [$uri] failed with exception:`n$_" + exit 1 +} + +# the response object takes this form: https://developer.github.com/v3/pulls/review_requests/#response-1 +# before we can push a new reviewer, we need to pull the simple Ids out of the complex objects that came back in the response +$userReviewers = @($resp.users | % { return $_.login }) +$teamReviewers = @($resp.teams | % { return $_.slug }) + +if (!$usersReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $usersReviewers.Clone() } +$modifiedUserReviewers += ($modifiedUserReviewers | ? { !$usersReviews.Contains($_) }) + +if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() } +$modifiedTeamReviewers += ($modifiedUserReviewers | ? { !$teamReviewers.Contains($_) }) + +$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers +$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers + +# Compare-Object returns values when there is a difference between the comparied objects. +# we only want to run the update if there IS a difference. +if ($detectedUserDiffs -or $detectedTeamDiffs) { + $postResp = @{} + + if ($modifiedUserReviewers) { $postResp["reviewers"] = $modifiedUserReviewers } + if ($modifiedTeamReviewers) { $postResp["team_reviewers"] = $modifiedTeamReviewers } + + $postResp = $postResp | ConvertTo-Json + + try { + $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3 + $resp | Write-Verbose + } + catch { + Write-Error "Unable to update PR reviewers. `n$_" + } +} +else { + $results = $GitHubUsers + $GitHubTeams + Write-Host "Reviewers $results already added. Exiting." + exit(0) +} diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index e48a526fe9e5..40b99e632de7 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -347,6 +347,38 @@ function ParseCArtifact($pkg, $workingDirectory) { } } +function ParseCppArtifact($pkg, $workingDirectory) { + $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON + $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName + $releaseNotes = "" + $readmeContent = "" + + $pkgVersion = $packageInfo.version + $pkgName = $packageInfo.name + + $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0] + if ($changeLogLoc) + { + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion + } + + $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0] + if ($readmeContentLoc) { + $readmeContent = Get-Content -Raw $readmeContentLoc + } + + return New-Object PSObject -Property @{ + PackageId = $pkgName + PackageVersion = $pkgVersion + # Artifact info is always considered deployable for now becasue it is not + # deployed anywhere. Dealing with duplicate tags happens downstream in + # CheckArtifactShaAgainstTagsList + Deployable = $true + ReleaseNotes = $releaseNotes + } +} + + # Returns the pypi publish status of a package id and version. function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { try { @@ -374,9 +406,10 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { # Retrieves the list of all tags that exist on the target repository function GetExistingTags($apiUrl) { try { - return (Invoke-WebRequest-WithHandling -Method "GET" -url "$apiUrl/git/refs/tags" ) | % { $_.ref.Replace("refs/tags/", "") } + return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } } catch { + Write-Host $_ $statusCode = $_.Exception.Response.StatusCode.value__ $statusDescription = $_.Exception.Response.StatusDescription @@ -386,7 +419,7 @@ function GetExistingTags($apiUrl) { # Return an empty list if there are no tags in the repo if ($statusCode -eq 404) { - return @() + return ,@() } exit(1) @@ -424,6 +457,10 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a $ParsePkgInfoFn = "ParseCArtifact" $packagePattern = "*.json" } + "CPP" { + $ParsePkgInfoFn = "ParseCppArtifact" + $packagePattern = "*.json" + } default { Write-Host "Unrecognized Language: $language" exit(1) diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index 0093c38e7f57..a0893099293a 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -350,4 +350,10 @@ if ($Language -eq "c") # used to publish multiple docs packages in a single invocation. $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version +} + +if ($Language -eq "cpp") +{ + $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json) + Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version } \ No newline at end of file diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index 1667cddd7e22..83f2caa5cf41 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -7,7 +7,7 @@ param ( $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary) $packageRepository, # used to indicate destination against which we will check the existing version. - # valid options: PyPI, Nuget, NPM, Maven, C + # valid options: PyPI, Nuget, NPM, Maven, C, CPP # used by CreateTags $releaseSha, # the SHA for the artifacts. DevOps: $(Release.Artifacts..SourceVersion) or $(Build.SourceVersion) diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 new file mode 100644 index 000000000000..edbae4cb18c6 --- /dev/null +++ b/eng/common/scripts/get-codeowners.ps1 @@ -0,0 +1,40 @@ +param ( + $TargetDirectory, # should be in relative form from root of repo. EG: sdk/servicebus + $RootDirectory # ideally $(Build.SourcesDirectory) +) + +$codeOwnersLocation = Join-Path $RootDirectory -ChildPath ".github/CODEOWNERS" + +if (!(Test-Path $codeOwnersLocation)) { + Write-Host "Unable to find CODEOWNERS file in target directory $RootDirectory" + exit 1 +} + +$codeOwnersContent = Get-Content $codeOwnersLocation + +$ownedFolders = @{} + +foreach ($contentLine in $codeOwnersContent) { + if (-not $contentLine.StartsWith("#") -and $contentLine){ + $splitLine = $contentLine -split "\s+" + + # CODEOWNERS file can also have labels present after the owner aliases + # gh aliases start with @ in codeowners. don't pass on to API calls + $ownedFolders[$splitLine[0].ToLower()] = ($splitLine[1..$($splitLine.Length)] ` + | ? { $_.StartsWith("@") } ` + | % { return $_.substring(1) }) -join "," + } +} + +$results = $ownedFolders[$TargetDirectory.ToLower()] + +if ($results) { + Write-Host "Discovered code owners for path $TargetDirectory are $results." + return $results +} +else { + Write-Host "Unable to match path $TargetDirectory in CODEOWNERS file located at $codeOwnersLocation." + Write-Host $ownedFolders | ConvertTo-Json + return "" +} + diff --git a/eng/common/scripts/modules/Package-Properties.psm1 b/eng/common/scripts/modules/Package-Properties.psm1 index b9a8eb11780d..b0572a71d449 100644 --- a/eng/common/scripts/modules/Package-Properties.psm1 +++ b/eng/common/scripts/modules/Package-Properties.psm1 @@ -64,12 +64,6 @@ class PackageProps } } -$ProgressPreference = "SilentlyContinue" - - -Register-PSRepository -Default -ErrorAction:SilentlyContinue -Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser - function Extract-PkgProps ($pkgPath, $serviceName, $pkgName, $lang) { if ($lang -eq "net") @@ -261,6 +255,9 @@ function Operate-OnPackages ($activePkgList, $serviceName, $language, $repoRoot, function Get-PkgListFromYml ($ciYmlPath) { + $ProgressPreference = "SilentlyContinue" + Register-PSRepository -Default -ErrorAction:SilentlyContinue + Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser $ciYmlContent = Get-Content $ciYmlPath -Raw $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered if ($ciYmlObj.Contains("stages"))