From 08b2f0df857ae79c206c5ae3bedfbd456843cbe9 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Wed, 30 Jun 2021 08:43:18 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 1725 (#14933) * Bring changes from JS docs metadata * Move business logic inside Update-DocsMsMetadata.ps1 * Update with the latest changes in JS PR * Update from latest PR feedback * Add check for empty path to prevent crashes when creating relative paths Co-authored-by: Daniel Jurek --- .../steps/enable-long-path-support.yml | 10 ++ .../steps/set-daily-docs-branch-name.yml | 14 ++ .../templates/steps/set-default-branch.yml | 5 +- .../steps/update-docsms-metadata.yml | 95 +++++++++++ eng/common/scripts/Invoke-DevOpsAPI.ps1 | 7 +- eng/common/scripts/Package-Properties.ps1 | 1 + eng/common/scripts/Queue-Pipeline.ps1 | 15 +- .../scripts/Save-Package-Properties.ps1 | 74 ++++++++- eng/common/scripts/Update-DocsMsMetadata.ps1 | 155 ++++++++++++++++++ eng/common/scripts/Update-DocsMsPackages.ps1 | 115 ++++++++++++- eng/common/scripts/common.ps1 | 2 + 11 files changed, 474 insertions(+), 19 deletions(-) create mode 100644 eng/common/pipelines/templates/steps/enable-long-path-support.yml create mode 100644 eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml create mode 100644 eng/common/pipelines/templates/steps/update-docsms-metadata.yml create mode 100644 eng/common/scripts/Update-DocsMsMetadata.ps1 diff --git a/eng/common/pipelines/templates/steps/enable-long-path-support.yml b/eng/common/pipelines/templates/steps/enable-long-path-support.yml new file mode 100644 index 000000000000..1e0db13d754a --- /dev/null +++ b/eng/common/pipelines/templates/steps/enable-long-path-support.yml @@ -0,0 +1,10 @@ +steps: +- pwsh: | + if ($IsWindows) { + REG ADD HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /f /v LongPathsEnabled /t REG_DWORD /d 1 + git config --system core.longpaths true + } + else { + Write-Host "This script is not executing on Windows, skipping registry modification." + } + displayName: Enable long path support if necessary diff --git a/eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml b/eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml new file mode 100644 index 000000000000..1548a9326f7a --- /dev/null +++ b/eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml @@ -0,0 +1,14 @@ +parameters: + - name: DailyBranchVariableName + type: string + default: TargetBranchName + +steps: + - pwsh: | + $branchName = $env:DAILYDOCSBRANCHNAMEOVERRIDE + if (!$branchName) { + $branchName = "daily/$(Get-Date -Format 'yyyy-MM-dd')" + } + Write-Host "Daily Branch Name: $branchName" + Write-Host "##vso[task.setvariable variable=${{ parameters.DailyBranchVariableName }};]$branchName" + displayName: Set daily docs branch name in $(${{ parameters.DailyBranchVariableName }}) diff --git a/eng/common/pipelines/templates/steps/set-default-branch.yml b/eng/common/pipelines/templates/steps/set-default-branch.yml index 491edaa8b68d..31bf28936164 100644 --- a/eng/common/pipelines/templates/steps/set-default-branch.yml +++ b/eng/common/pipelines/templates/steps/set-default-branch.yml @@ -1,6 +1,7 @@ parameters: WorkingDirectory: '$(System.DefaultWorkingDirectory)' RemoteRepo: 'origin' + DefaultBranchVariableName: DefaultBranch steps: - pwsh: | $setDefaultBranch = (git remote show ${{ parameters.RemoteRepo }} | Out-String) -replace "(?ms).*HEAD branch: (\w+).*", '$1' @@ -8,8 +9,8 @@ steps: Write-Host "Not able to fetch the default branch from git command. Set to main." $setDefaultBranch = 'main' } - Write-Host "Setting DefaultBranch=$setDefaultBranch" - Write-Host "##vso[task.setvariable variable=DefaultBranch]$setDefaultBranch" + Write-Host "Setting ${{ parameters.DefaultBranchVariableName }}=$setDefaultBranch" + Write-Host "##vso[task.setvariable variable=${{ parameters.DefaultBranchVariableName }}]$setDefaultBranch" displayName: "Setup Default Branch" workingDirectory: ${{ parameters.workingDirectory }} ignoreLASTEXITCODE: true diff --git a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml new file mode 100644 index 000000000000..341169b37348 --- /dev/null +++ b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml @@ -0,0 +1,95 @@ +parameters: + - name: PackageInfoLocations + type: object + default: [] + - name: RepoId + type: string + default: $(Build.Repository.Name) + - name: WorkingDirectory + type: string + default: '' + - name: ScriptDirectory + type: string + default: eng/common/scripts + - name: TargetDocRepoName + type: string + default: '' + - name: TargetDocRepoOwner + type: string + - name: Language + type: string + default: '' + - name: DailyDocsBuild + type: boolean + default: false + - name: SparseCheckoutPaths + type: object + default: + - '**' + +steps: +- template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml + +- pwsh: | + Write-Host "###vso[task.setvariable variable=DocRepoLocation]${{ parameters.WorkingDirectory }}/doc" + displayName: Set $(DocRepoLocation) + +- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Repositories: + - Name: ${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }} + WorkingDirectory: $(DocRepoLocation) + Paths: ${{ parameters.SparseCheckoutPaths }} + +# If performing a daily docs build set the $(TargetBranchName) to a daily branch +# name and attempt to checkout the daily docs branch. If the branch doesn't +# exist, create it +- ${{ if eq(parameters.DailyDocsBuild, 'true') }}: + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + + - pwsh: | + $ErrorActionPreference = "Continue" + $RemoteName = "origin" + $BranchName = "$(TargetBranchName)" + # Fetch and checkout remote branch if it already exists otherwise create a new branch. + git ls-remote --exit-code --heads $RemoteName $BranchName + if ($LASTEXITCODE -eq 0) { + Write-Host "git fetch $RemoteName $BranchName" + git fetch $RemoteName $BranchName + Write-Host "git checkout $BranchName." + git checkout $BranchName + } else { + Write-Host "git checkout -b $BranchName." + git checkout -b $BranchName + } + displayName: Checkout daily docs branch if it exists + workingDirectory: $(DocRepoLocation) + +# If NOT performing a daily docs build, set the $(TargetBranchName) to the +# default branch of the documentation repository. +- ${{ if ne(parameters.DailyDocsBuild, 'true') }}: + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(DocRepoLocation) + DefaultBranchVariableName: TargetBranchName + +- pwsh: | + $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"') + $packageInfoLocations = ConvertFrom-Json $packageInfoJson + ${{ parameters.ScriptDirectory }}/Update-DocsMsMetadata.ps1 ` + -PackageInfoJsonLocations $packageInfoLocations ` + -DocRepoLocation "$(DocRepoLocation)" ` + -Language '${{parameters.Language}}' ` + -RepoId '${{ parameters.RepoId }}' + displayName: Apply Documentation Updates + +- template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(TargetBranchName) + BaseRepoOwner: ${{ parameters.TargetDocRepoOwner }} + CommitMsg: "Update docs metadata" + TargetRepoName: ${{ parameters.TargetDocRepoName }} + TargetRepoOwner: ${{ parameters.TargetDocRepoOwner }} + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} diff --git a/eng/common/scripts/Invoke-DevOpsAPI.ps1 b/eng/common/scripts/Invoke-DevOpsAPI.ps1 index 32c3569a3cd2..72d61edd081f 100644 --- a/eng/common/scripts/Invoke-DevOpsAPI.ps1 +++ b/eng/common/scripts/Invoke-DevOpsAPI.ps1 @@ -19,7 +19,9 @@ function Start-DevOpsBuild { $DefinitionId, [ValidateNotNullOrEmpty()] [Parameter(Mandatory = $true)] - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + [Parameter(Mandatory = $false)] + [string]$BuildParametersJson ) $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", "" @@ -27,6 +29,7 @@ function Start-DevOpsBuild { $parameters = @{ sourceBranch = $SourceBranch definition = @{ id = $DefinitionId } + parameters = $BuildParametersJson } return Invoke-RestMethod ` @@ -157,4 +160,4 @@ function Add-RetentionLease { -MaximumRetryCount 3 ` -ContentType "application/json" -} \ No newline at end of file +} diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index e00ea7bb9eaf..622c5fd925aa 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -5,6 +5,7 @@ class PackageProps { [string]$Name [string]$Version + [string]$DevVersion [string]$DirectoryPath [string]$ServiceDirectory [string]$ReadMePath diff --git a/eng/common/scripts/Queue-Pipeline.ps1 b/eng/common/scripts/Queue-Pipeline.ps1 index a8c147e2ef38..30a4282c8cbb 100644 --- a/eng/common/scripts/Queue-Pipeline.ps1 +++ b/eng/common/scripts/Queue-Pipeline.ps1 @@ -18,7 +18,10 @@ param( [string]$VsoQueuedPipelines, [Parameter(Mandatory = $true)] - [string]$Base64EncodedAuthToken + [string]$Base64EncodedAuthToken, + + [Parameter(Mandatory = $false)] + [string]$BuildParametersJson ) . (Join-Path $PSScriptRoot common.ps1) @@ -46,7 +49,13 @@ if ($CancelPreviousBuilds) } try { - $resp = Start-DevOpsBuild -SourceBranch $SourceBranch -DefinitionId $DefinitionId -Base64EncodedAuthToken $Base64EncodedAuthToken + $resp = Start-DevOpsBuild ` + -Organization $Organization ` + -Project $Project ` + -SourceBranch $SourceBranch ` + -DefinitionId $DefinitionId ` + -Base64EncodedAuthToken $Base64EncodedAuthToken ` + -BuildParametersJson $BuildParametersJson } catch { LogError "Start-DevOpsBuild failed with exception:`n$_" @@ -64,4 +73,4 @@ if ($VsoQueuedPipelines) { } $QueuedPipelineLinks Write-Host "##vso[task.setvariable variable=$VsoQueuedPipelines]$QueuedPipelineLinks" -} \ No newline at end of file +} diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 26f4f17c8807..7e51813830dd 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -1,12 +1,79 @@ +<# +.SYNOPSIS +Saves package properties from source into JSON files + +.DESCRIPTION +Saves package properties in source of a given service directory to JSON files. +JSON files are named in the form .json or .json if +an artifact name property is available in the package properties. + +Can optionally add a dev version property which can be used logic for daily +builds. + +.PARAMETER serviceDirectory +Service directory in which to search for packages + +.PARAMETER outDirectory +Output location (generally a package artifact directory in DevOps) for JSON +files + +.PARAMETER addDevVersion +Reads the version out of the source and adds a DevVersion property to the +package properties JSON file. If the package properties JSON file already +exists, read the Version property from the existing package properties JSON file +and set that as the Version property for the new output. This has the effect of +"adding" a DevVersion property to the file which could be different from the +Verison property in that file. +#> + [CmdletBinding()] Param ( [Parameter(Mandatory=$True)] [string] $serviceDirectory, [Parameter(Mandatory=$True)] - [string] $outDirectory + [string] $outDirectory, + [switch] $addDevVersion ) . (Join-Path $PSScriptRoot common.ps1) + +function SetOutput($outputPath, $incomingPackageSpec) { + $outputObject = $incomingPackageSpec + + if ($addDevVersion) { + # Use the "Version" property which was provided by the incoming package spec + # as the DevVersion. This may be overridden later. + $outputObject.DevVersion = $incomingPackageSpec.Version + + # If there is an exsiting package info json file read that and set the + # Version property from that JSON file. + if (Test-Path $outputPath) { + $originalObject = ConvertFrom-Json (Get-Content $outputPath -Raw) + $outputObject.Version = $originalObject.Version + } + } + + # Set file paths to relative paths + $outputObject.DirectoryPath = GetRelativePath $outputObject.DirectoryPath + $outputObject.ReadMePath = GetRelativePath $outputObject.ReadMePath + $outputObject.ChangeLogPath = GetRelativePath $outputObject.ChangeLogPath + + Set-Content ` + -Path $outputPath ` + -Value (ConvertTo-Json -InputObject $outputObject -Depth 100) +} + +function GetRelativePath($path) { + # If the path is empty return an empty string + if (!$path) { + return '' + } + $relativeTo = Resolve-Path $PSScriptRoot/../../../ + # Replace "\" with "/" so the path is valid across other platforms and tools + $relativePath = [IO.Path]::GetRelativePath($relativeTo, $path) -replace "\\", '/' + return $relativePath +} + $allPackageProperties = Get-AllPkgProperties $serviceDirectory if ($allPackageProperties) { @@ -29,9 +96,8 @@ if ($allPackageProperties) $configFilePrefix = $pkg.ArtifactName } $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json" - $outputObject = $pkg | ConvertTo-Json - Set-Content -Path $outputPath -Value $outputObject - } + SetOutput $outputPath $pkg + } } Get-ChildItem -Path $outDirectory diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 new file mode 100644 index 000000000000..2a9d4f26a999 --- /dev/null +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -0,0 +1,155 @@ +<# +.SYNOPSIS +Updates package README.md for publishing to docs.microsoft.com + +.DESCRIPTION +Given a PackageInfo .json file, format the package README.md file with metadata +and other information needed to release reference docs: + +* Adjust README.md content to include metadata +* Insert the package verison number in the README.md title +* Copy file to the appropriate location in the documentation repository +* Copy PackageInfo .json file to the metadata location in the reference docs + repository. This enables the Docs CI build to onboard packages which have not + shipped and for which there are no entries in the metadata CSV files. + +.PARAMETER PackageInfoJsonLocations +List of locations of the artifact information .json file. This is usually stored +in build artifacts under packages/PackageInfo/.json. Can also be +a single item. + +.PARAMETER DocRepoLocation +Location of the root of the docs.microsoft.com reference doc location. Further +path information is provided by $GetDocsMsMetadataForPackageFn + +.PARAMETER Language +Programming language to supply to metadata + +.PARAMETER RepoId +GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' + +#> + +param( + [Parameter(Mandatory = $true)] + [array]$PackageInfoJsonLocations, + + [Parameter(Mandatory = $true)] + [string]$DocRepoLocation, + + [Parameter(Mandatory = $true)] + [string]$Language, + + [Parameter(Mandatory = $true)] + [string]$RepoId +) + +. (Join-Path $PSScriptRoot common.ps1) + +$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)(?:master|main)" +$TITLE_REGEX = "(\#\s+(?Azure .+? (?:client|plugin|shared) library for (?:JavaScript|Java|Python|\.NET|C)))" + +function GetAdjustedReadmeContent($ReadmeContent, $PackageInfo, $PackageMetadata) { + # Normalize service name "Key Vault" -> "keyvault" + # TODO: Use taxonomy for service name -- https://github.com/Azure/azure-sdk-tools/issues/1442 + # probably from metadata + $service = $PackageMetadata.ServiceName.ToLower().Replace(" ", "") + # Generate the release tag for use in link substitution + $tag = "$($PackageInfo.Name)_$($PackageInfo.Version)" + $date = Get-Date -Format "MM/dd/yyyy" + + + $foundTitle = "" + if ($ReadmeContent -match $TITLE_REGEX) { + $ReadmeContent = $ReadmeContent -replace $TITLE_REGEX, "`${0} - Version $($PackageInfo.Version) `n" + $foundTitle = $matches["filetitle"] + } + + # If this is not a daily dev package, perform link replacement + if (!$packageInfo.DevVersion) { + $replacementPattern = "`${1}$tag" + $ReadmeContent = $ReadmeContent -replace $releaseReplaceRegex, $replacementPattern + } + + $header = @" +--- +title: $foundTitle +keywords: Azure, $Language, SDK, API, $($PackageInfo.Name), $service +author: maggiepint +ms.author: magpint +ms.date: $date +ms.topic: article +ms.prod: azure +ms.technology: azure +ms.devlang: $Language +ms.service: $service +--- + +"@ + + return "$header`n$ReadmeContent" +} + +function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { + $packageInfoJson = Get-Content $packageInfoJsonLocation -Raw + $packageInfo = ConvertFrom-Json $packageInfoJson + + $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version) + if ($packageInfo.DevVersion) { + # If the package is of a dev version there may be language-specific needs to + # specify the appropriate version. For example, in the case of JS, the dev + # version is always 'dev' when interacting with NPM. + if ($GetDocsMsDevLanguageSpecificPackageInfoFn -and (Test-Path "Function:$GetDocsMsDevLanguageSpecificPackageInfoFn")) { + $packageInfo = &$GetDocsMsDevLanguageSpecificPackageInfoFn $packageInfo + } else { + # Default: use the dev version from package info as the version for + # downstream processes + $packageInfo.Version = $packageInfo.DevVersion + } + } + + $packageMetadataArray = (Get-CSVMetadata).Where({ $_.Package -eq $packageInfo.Name -and $_.GroupId -eq $packageInfo.Group -and $_.Hide -ne 'true' -and $_.New -eq 'true' }) + if ($packageMetadataArray.Count -eq 0) { + LogError "Could not retrieve metadata for $($packageInfo.Name) from metadata CSV" + } elseif ($packageMetadataArray.Count -gt 1) { + LogWarning "Multiple metadata entries for $($packageInfo.Name) in metadata CSV. Using first entry." + } + $packageMetadata = $packageMetadataArray[0] + + $readmeContent = Get-Content $packageInfo.ReadMePath -Raw + $outputReadmeContent = "" + if ($readmeContent) { + $outputReadmeContent = GetAdjustedReadmeContent $readmeContent $packageInfo $packageMetadata + } + + $docsMsMetadata = &$GetDocsMsMetadataForPackageFn $packageInfo + $readMePath = $docsMsMetadata.LatestReadMeLocation + if ($originalVersion.IsPrerelease) { + $readMePath = $docsMsMetadata.PreviewReadMeLocation + } + + $suffix = $docsMsMetadata.Suffix + $readMeName = "$($docsMsMetadata.DocsMsReadMeName.ToLower())-readme${suffix}.md" + + $readmeLocation = Join-Path $DocRepoLocation $readMePath $readMeName + + Set-Content -Path $readmeLocation -Value $outputReadmeContent + + # Copy package info file to the docs repo + $metadataMoniker = 'latest' + if ($originalVersion.IsPrerelease) { + $metadataMoniker = 'preview' + } + $packageMetadataName = Split-Path $packageInfoJsonLocation -Leaf + $packageInfoLocation = Join-Path $DocRepoLocation "metadata/$metadataMoniker" + $packageInfoJson = ConvertTo-Json $packageInfo + New-Item -ItemType Directory -Path $packageInfoLocation -Force + Set-Content ` + -Path $packageInfoLocation/$packageMetadataName ` + -Value $packageInfoJson +} + +foreach ($packageInfo in $PackageInfoJsonLocations) { + Write-Host "Updating metadata for package: $packageInfo" + UpdateDocsMsMetadataForPackage $packageInfo +} diff --git a/eng/common/scripts/Update-DocsMsPackages.ps1 b/eng/common/scripts/Update-DocsMsPackages.ps1 index 4292200f3b6f..576159e7785d 100644 --- a/eng/common/scripts/Update-DocsMsPackages.ps1 +++ b/eng/common/scripts/Update-DocsMsPackages.ps1 @@ -1,10 +1,24 @@ -# This script is intended to update docs.ms CI configuration (currently supports Java, Python, C#, JS) in nightly build -# For details on calling, check `docindex.yml`. +<# +.SYNOPSIS +Update docs.microsoft.com CI configuration with provided metadata -# In this script, we will do the following business logic. -# 1. Filter out the packages from release csv file by `New=true`, `Hide!=true` -# 2. Compare current package list with the csv packages, and keep them in sync. Leave other packages as they are. -# 3. Update the tarage packages back to CI config files. +.DESCRIPTION +Update docs.microsoft.com CI configuration with metadata in the Azure/azure-sdk +metadata CSV file and information in the docs.microsoft.com repo's own /metadata +folder. The docs.microsoft.com repo's /metadata folder allows onboarding of +packages which have not released to a central package manager. + +* Use packages in the Azure/azure-sdk metadata CSV where New == true and + Hide != true +* Add metadata from files in the metadata/ folder to the CSV metadata +* Onboard new packages, update existing tracked packages, leave other packages + in place. (This is implemented on a per-language basis by + $UpdateDocsMsPackagesFn) + +.PARAMETER DocRepoLocation +Location of the docs.microsoft.com reference docs repo. + +#> param ( [Parameter(Mandatory = $true)] $DocRepoLocation # the location of the cloned doc repo @@ -12,8 +26,93 @@ param ( . (Join-Path $PSScriptRoot common.ps1) -function GetDocsMetadata() { - (Get-CSVMetadata).Where({ $_.New -eq 'true' -and $_.Hide -ne 'true' }) +function GetDocsMetadataForMoniker($moniker) { + $searchPath = Join-Path $DocRepoLocation 'metadata' $moniker + if (!(Test-Path $searchPath)) { + return @() + } + $paths = Get-ChildItem -Path $searchPath -Filter *.json + + $metadata = @() + foreach ($path in $paths) { + $fileContents = Get-Content $path -Raw + $fileObject = ConvertFrom-Json -InputObject $fileContents + $versionGa = '' + $versionPreview = '' + if ($moniker -eq 'latest') { + $versionGa = $fileObject.Version + } else { + $versionPreview = $fileObject.Version + } + + $metadata += @{ + Package = $fileObject.Name; + VersionGA = $versionGa; + VersionPreview = $versionPreview; + RepoPath = $fileObject.ServiceDirectory; + Type = $fileObject.SdkType; + New = $fileObject.IsNewSdk; + } + } + + return $metadata +} +function GetDocsMetadata() { + # Read metadata from CSV + $csvMetadata = (Get-CSVMetadata).Where({ $_.New -eq 'true' -and $_.Hide -ne 'true' }) + + # Read metadata from docs repo + $metadataByPackage = @{} + foreach ($package in GetDocsMetadataForMoniker 'latest') { + if ($metadataByPackage.ContainsKey($package.Package)) { + LogWarning "Duplicate package in latest metadata: $($package.Package)" + } + Write-Host "Adding latest package: $($package.Package)" + $metadataByPackage[$package.Package] = $package + } + + foreach ($package in GetDocsMetadataForMoniker 'preview') { + if ($metadataByPackage.ContainsKey($package.Package)) { + # Merge VersionPreview of each object + Write-Host "Merging preview package version for $($package.Package))" + $metadataByPackage[$package.Package].VersionPreview = $package.VersionPreview + } else { + Write-Host "Adding preview package: $($package.Package)" + $metadataByPackage[$package.Package] = $package + } + } + + # Override CSV metadata version information before returning + $outputMetadata = @() + foreach ($item in $csvMetadata) { + if ($metadataByPackage.ContainsKey($item.Package)) { + Write-Host "Overriding CSV metadata from docs repo for $($item.Package)" + $matchingPackage = $metadataByPackage[$item.Package] + + # Only update the version from metadata present in the docs repo IF there + # is a specified version. The absence of package metadata in the docs repo + # (e.g. no GA version) does not imply that the CSV metadata is incorrect. + if ($matchingPackage.VersionGA) { + $item.VersionGA = $matchingPackage.VersionGA + } + if ($matchingPackage.VersionPreview) { + $item.VersionPreview = $matchingPackage.VersionPreview + } + } + $outputMetadata += $item + } + + # Add entries present in the docs repo which are not present in CSV. These are + # usually packages which have not yet published a preview or GA version. + foreach ($item in $metadataByPackage.Values) { + $matchingPackagesInCsvMetadata = $csvMetadata.Where({ $_.Package -eq $item.Package }) + if (!$matchingPackagesInCsvMetadata) { + Write-Host "Adding package from docs metadata that is not found in CSV metadata: $($item.Package)" + $outputMetadata += $item + } + } + + return $outputMetadata } if ($UpdateDocsMsPackagesFn -and (Test-Path "Function:$UpdateDocsMsPackagesFn")) { diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index b92235c208f9..4e0b0847cdbf 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -40,5 +40,7 @@ $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" $UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages" +$GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage" +$GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo" $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"