From 150f176fe4ceb69a1f7389bc55880e4158354dbd Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 13:39:39 -0800 Subject: [PATCH 01/13] Make JS validation work for docindex and release pipeline --- .../templates/stages/archetype-js-release.yml | 2 + eng/scripts/Language-Settings.ps1 | 50 +++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-js-release.yml b/eng/pipelines/templates/stages/archetype-js-release.yml index 919d4bd884c5..0485c075ca0d 100644 --- a/eng/pipelines/templates/stages/archetype-js-release.yml +++ b/eng/pipelines/templates/stages/archetype-js-release.yml @@ -126,6 +126,7 @@ stages: SkipDefaultCheckout: true Paths: - sdk/**/*.md + - .github/CODEOWNERS - download: current - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: @@ -286,6 +287,7 @@ stages: SkipDefaultCheckout: true Paths: - sdk/**/*.md + - .github/CODEOWNERS - download: current - pwsh: | Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 37afc321d487..4d913ee477bb 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -233,7 +233,7 @@ function Get-DocsMsPackageName($packageName, $packageVersion) { # registry = ""; # ... # } -function ValidatePackagesForDocs($packages) { +function ValidatePackagesForDocs($packages, $DocValidationImageId) { # Using GetTempPath because it works on linux and windows $tempDirectory = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) New-Item -ItemType Directory -Force -Path $tempDirectory | Out-Null @@ -244,7 +244,7 @@ function ValidatePackagesForDocs($packages) { # Get value for variables outside of the Foreach-Object scope $scriptRoot = "$using:scriptRoot" $workingDirectory = "$using:tempDirectory" - return ."$scriptRoot\validate-docs-package.ps1" -Package $_ -DocValidationImageId "$using:ImageId" -WorkingDirectory $workingDirectory + return ."$scriptRoot\validate-docs-package.ps1" -Package $_ -DocValidationImageId "$using:DocValidationImageId" -WorkingDirectory $workingDirectory } # Clean up temp folder @@ -260,8 +260,7 @@ $PackageExclusions = @{ '@azure/core-asynciterator-polyfill' = 'Docs CI fails https://github.com/Azure/azure-sdk-for-js/issues/16675'; } -function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata) { - +function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId) { Write-Host "Excluded packages:" foreach ($excludedPackage in $PackageExclusions.Keys) { Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" @@ -273,16 +272,17 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata) { (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') ` 'preview' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') # Log file for package validation - + (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') `# Log file for package validation + $DocValidationImageId UpdateDocsMsPackages ` (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` 'latest' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') # Log file for package validation + (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') `# Log file for package validation + $DocValidationImageId } -function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageHistoryLogFile) { +function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageHistoryLogFile, $DocValidationImageId) { Write-Host "Updating configuration: $DocConfigFile with mode: $Mode" $packageConfig = Get-Content $DocConfigFile -Raw | ConvertFrom-Json @@ -373,7 +373,7 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageHist $outputPackages += @{ name = $packageName } } - $packageValidation = ValidatePackagesForDocs $outputPackages + $packageValidation = ValidatePackagesForDocs $outputPackages $DocValidationImageId $validationHash = @{} foreach ($result in $packageValidation) { $validationHash[$result.Package.name] = $result @@ -465,3 +465,35 @@ function GetExistingPackageVersions ($PackageName, $GroupId = $null) return $null } } + +function Validate-javascript-DocMsPackages { + Param( + [Parameter(Mandatory = $true)] + [PSCustomObject]$PackageInfo, + [Parameter(Mandatory = $false)] + [string]$PackageSourceOverride, + [Parameter(Mandatory = $false)] + [string]$DocValidationImageId + ) + $fileLocation = "" + if ($PackageInfo.DevVersion -or $PackageInfo.Version -contains "beta") { + $fileLocation = (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') + if ($PackageInfo.DevVersion) { + $PackageInfo.Version = $PackageInfo.DevVersion + } + } + else { + $fileLocation = (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') + } + + $packageConfig = Get-Content $fileLocation -Raw | ConvertFrom-Json + $outputPackage = $PackageInfo + foreach ($package in $packageConfig.npm_package_sources) { + if ($package.name -eq $PackageInfo.Name) { + $outputPackage = $package + $outputPackage.name = Get-DocsMsPackageName $package.name $PackageInfo.Version + break + } + } + ValidatePackagesForDocs -packages $outputPackage -DocValidationImageId $DocValidationImageId +} From 8f526364d5a22094dd9a5f4f8ce7f80c115613b4 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 13:44:28 -0800 Subject: [PATCH 02/13] Adding default --- eng/scripts/Language-Settings.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 4d913ee477bb..18d5a3c5ec70 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -260,7 +260,7 @@ $PackageExclusions = @{ '@azure/core-asynciterator-polyfill' = 'Docs CI fails https://github.com/Azure/azure-sdk-for-js/issues/16675'; } -function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId) { +function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId=$ImageId) { Write-Host "Excluded packages:" foreach ($excludedPackage in $PackageExclusions.Keys) { Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" From 644db7c1be234b13970e9dcc03b8c28bc45425e3 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 14:09:24 -0800 Subject: [PATCH 03/13] print the command line --- eng/scripts/validate-docs-package.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/scripts/validate-docs-package.ps1 b/eng/scripts/validate-docs-package.ps1 index 840eb11e2bfa..0283684e9bc1 100644 --- a/eng/scripts/validate-docs-package.ps1 +++ b/eng/scripts/validate-docs-package.ps1 @@ -63,6 +63,7 @@ function DockerValidation() { $commandLine = "$commandLine$folder" } $commandLine = "$commandLine $DocValidationImageId 2>&1" + Write-Host $commandLine $installOutput = Invoke-Expression $commandLine # The docker exit codes: https://docs.docker.com/engine/reference/run/#exit-status From b6ba669e06bfdd510dfc9150ae6e207c53285065 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 14:13:40 -0800 Subject: [PATCH 04/13] remove image id --- eng/pipelines/docindex.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 972e9ea63ae8..348cd2649a3e 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -23,13 +23,12 @@ jobs: Repositories: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) - # Pull and build the docker image. + # Pull and build the docker image. - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml parameters: ContainerRegistryClientId: $(azuresdkimages-cr-clientid) ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret) - ImageId: '$(DocValidationImageId)' - + ImageId: "$(DocValidationImageId)" # Call update docs ci script to onboard packages - task: Powershell@2 @@ -39,7 +38,7 @@ jobs: arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)' displayName: Update Docs Onboarding condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - + # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/set-default-branch.yml parameters: @@ -69,7 +68,7 @@ jobs: inputs: pwsh: true filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)' + arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for Daily branch - template: /eng/common/pipelines/templates/steps/git-push-changes.yml From a38586ffc08a6cbe785316c667b28b84fd557030 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 14:15:46 -0800 Subject: [PATCH 05/13] remove some comment --- eng/scripts/Language-Settings.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 18d5a3c5ec70..0449c1ce67e7 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -272,13 +272,12 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $Pac (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') ` 'preview' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') `# Log file for package validation - $DocValidationImageId + (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') ` UpdateDocsMsPackages ` (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` 'latest' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') `# Log file for package validation + (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') ` $DocValidationImageId } From b5d32c3df5e1da4c96e9b49f2d79c30e870bcd46 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Tue, 7 Dec 2021 14:22:51 -0800 Subject: [PATCH 06/13] add back the parameters --- eng/scripts/Language-Settings.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 0449c1ce67e7..20cdf1979de3 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -273,6 +273,8 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $Pac 'preview' ` $FilteredMetadata ` (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') ` + $DocValidationImageId + UpdateDocsMsPackages ` (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` 'latest' ` From d8584f53a52b0f3649b9eb06d35eba240a9d9a95 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Tue, 7 Dec 2021 21:49:52 -0800 Subject: [PATCH 07/13] Update Language-Settings.ps1 --- eng/scripts/Language-Settings.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 0449c1ce67e7..84080d1f3c89 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -260,7 +260,7 @@ $PackageExclusions = @{ '@azure/core-asynciterator-polyfill' = 'Docs CI fails https://github.com/Azure/azure-sdk-for-js/issues/16675'; } -function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId=$ImageId) { +function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $DocValidationImageId=$ImageId) { Write-Host "Excluded packages:" foreach ($excludedPackage in $PackageExclusions.Keys) { Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" @@ -273,6 +273,8 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $Pac 'preview' ` $FilteredMetadata ` (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') ` + $DocValidationImageId + UpdateDocsMsPackages ` (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` 'latest' ` @@ -470,8 +472,6 @@ function Validate-javascript-DocMsPackages { [Parameter(Mandatory = $true)] [PSCustomObject]$PackageInfo, [Parameter(Mandatory = $false)] - [string]$PackageSourceOverride, - [Parameter(Mandatory = $false)] [string]$DocValidationImageId ) $fileLocation = "" From fc3d42c1d76a945a7be344abba1783d5a392adfa Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Wed, 8 Dec 2021 15:32:48 -0800 Subject: [PATCH 08/13] Update eng/scripts/Language-Settings.ps1 Co-authored-by: Wes Haggard --- eng/scripts/Language-Settings.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index a105ebf27559..f6b2b551cf14 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -272,8 +272,8 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $Doc (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') ` 'preview' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') ` - $DocValidationImageId + (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json.log') ` + $DocValidationImageId UpdateDocsMsPackages ` (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` From 89ef5f8dfde159fb5115dec1d988456b424ee315 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Wed, 8 Dec 2021 15:32:56 -0800 Subject: [PATCH 09/13] Update eng/scripts/Language-Settings.ps1 Co-authored-by: Wes Haggard --- eng/scripts/Language-Settings.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index f6b2b551cf14..53a1ceee82b0 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -279,8 +279,8 @@ function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $Doc (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` 'latest' ` $FilteredMetadata ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') ` - $DocValidationImageId + (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json.log') ` + $DocValidationImageId } function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageHistoryLogFile, $DocValidationImageId) { From fc4a0939279f91bf61adf16af4954f9a155a3fd6 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Wed, 8 Dec 2021 15:56:07 -0800 Subject: [PATCH 10/13] remove the extra paramter definition --- eng/scripts/Language-Settings.ps1 | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index a105ebf27559..c0c2c50b3388 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -470,13 +470,8 @@ function GetExistingPackageVersions ($PackageName, $GroupId = $null) } } -function Validate-javascript-DocMsPackages { - Param( - [Parameter(Mandatory = $true)] - [PSCustomObject]$PackageInfo, - [Parameter(Mandatory = $false)] - [string]$DocValidationImageId - ) +function Validate-javascript-DocMsPackages ($PackageInfo, $DocValidationImageId) +{ $fileLocation = "" if ($PackageInfo.DevVersion -or $PackageInfo.Version -contains "beta") { $fileLocation = (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') From 0b718657710b5f90538ffca51285f3253d0d69ac Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:57:28 -0800 Subject: [PATCH 11/13] Update Language-Settings.ps1 --- eng/scripts/Language-Settings.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index c266fde11c0e..30f9e8a7a51e 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -470,17 +470,17 @@ function GetExistingPackageVersions ($PackageName, $GroupId = $null) } } -function Validate-javascript-DocMsPackages ($PackageInfo, $DocValidationImageId) +function Validate-javascript-DocMsPackages ($PackageInfo, $DocRepoLocation, $DocValidationImageId) { $fileLocation = "" if ($PackageInfo.DevVersion -or $PackageInfo.Version -contains "beta") { - $fileLocation = (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') + $fileLocation = (Join-Path $DocRepoLocation 'ci-configs/packages-preview.json') if ($PackageInfo.DevVersion) { $PackageInfo.Version = $PackageInfo.DevVersion } } else { - $fileLocation = (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') + $fileLocation = (Join-Path $DocRepoLocation 'ci-configs/packages-latest.json') } $packageConfig = Get-Content $fileLocation -Raw | ConvertFrom-Json From 210e1f972b66ff138240f0a33b9d21361a6f8611 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:58:04 -0800 Subject: [PATCH 12/13] Update Language-Settings.ps1 --- eng/scripts/Language-Settings.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 30f9e8a7a51e..bfe63a63b07a 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -260,7 +260,7 @@ $PackageExclusions = @{ '@azure/core-asynciterator-polyfill' = 'Docs CI fails https://github.com/Azure/azure-sdk-for-js/issues/16675'; } -function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $DocValidationImageId=$ImageId) { +function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $DocValidationImageId) { Write-Host "Excluded packages:" foreach ($excludedPackage in $PackageExclusions.Keys) { Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" From c129f9cafcd89ff4cc56ed407f05de682f402af8 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Thu, 9 Dec 2021 16:49:14 -0800 Subject: [PATCH 13/13] checkout more files --- eng/pipelines/templates/stages/archetype-js-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-js-release.yml b/eng/pipelines/templates/stages/archetype-js-release.yml index 0485c075ca0d..dc968559e951 100644 --- a/eng/pipelines/templates/stages/archetype-js-release.yml +++ b/eng/pipelines/templates/stages/archetype-js-release.yml @@ -140,6 +140,8 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ + - ci-configs/packages-latest.json + - ci-configs/packages-preview.json - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocsGitHubIO @@ -307,3 +309,5 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ + - ci-configs/packages-latest.json + - ci-configs/packages-preview.json