From b2954758f3187aabecc4c4e6944a3d06e6e5cd67 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 9 Apr 2024 08:14:01 -0700 Subject: [PATCH] Put the namespaces(packages) into the json files (#39593) * Put the namespaces(packages) into the json files * Add missing function end curly * Updates for testing * remove last fetch template usage * Actually return the packageInfo object from GetDocsMsDevLanguageSpecificPackageInfoFn * revert the docindex changes * remove the testing changes from template's ci.yml --- .../stages/archetype-java-release-batch.yml | 22 -- .../stages/archetype-java-release-patch.yml | 11 - .../stages/archetype-java-release.yml | 19 -- .../templates/steps/fetch-package-list.yml | 29 --- eng/scripts/Fetch-PackageList-Javadoc.ps1 | 49 ----- eng/scripts/Language-Settings.ps1 | 29 ++- eng/scripts/docs/Docs-ToC.ps1 | 193 ++++++++++-------- 7 files changed, 141 insertions(+), 211 deletions(-) delete mode 100644 eng/pipelines/templates/steps/fetch-package-list.yml delete mode 100644 eng/scripts/Fetch-PackageList-Javadoc.ps1 diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index de255797bf329..f51c9e3b7d7fc 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -353,17 +353,6 @@ stages: - metadata/ DocValidationImageId: "$(DocValidationImageId)" - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - template: /eng/pipelines/templates/steps/fetch-package-list.yml - parameters: - JavaDocJarLocation: "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.groupId}}/${{artifact.name}}" - ArtifactName: ${{artifact.name}} - TargetBranch: $(TargetBranchName) - DocRepoLocation: $(DocRepoLocation) - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - TargetDocRepoOwner: "Azure" - - deployment: PublishDocs displayName: Publish Docs to GitHubIO Blob Storage condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) @@ -518,15 +507,4 @@ stages: - metadata/ DocValidationImageId: "$(DocValidationImageId)" - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - template: /eng/pipelines/templates/steps/fetch-package-list.yml - parameters: - JavaDocJarLocation: "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.groupId}}/${{artifact.name}}" - ArtifactName: ${{artifact.name}} - TargetBranch: $(TargetBranchName) - DocRepoLocation: $(DocRepoLocation) - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - TargetDocRepoOwner: "azure-sdk" - - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index 79755180ea23a..1fce6895798d4 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -273,17 +273,6 @@ stages: - metadata/ DocValidationImageId: "$(DocValidationImageId)" - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - template: /eng/pipelines/templates/steps/fetch-package-list.yml - parameters: - JavaDocJarLocation: "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.groupId}}/${{artifact.name}}" - ArtifactName: ${{artifact.name}} - TargetBranch: $(TargetBranchName) - DocRepoLocation: $(DocRepoLocation) - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - TargetDocRepoOwner: "Azure" - - deployment: PublishDocs displayName: Publish Docs to GitHubIO Blob Storage condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index 996fcba1bca75..d65fac1774064 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -287,14 +287,6 @@ stages: - docs-ref-services/ - metadata/ DocValidationImageId: "$(DocValidationImageId)" - - template: /eng/pipelines/templates/steps/fetch-package-list.yml - parameters: - JavaDocJarLocation: "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.groupId}}/${{artifact.name}}" - ArtifactName: ${{artifact.name}} - TargetBranch: $(TargetBranchName) - DocRepoLocation: $(DocRepoLocation) - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - TargetDocRepoOwner: "Azure" - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocs @@ -495,15 +487,4 @@ stages: - metadata/ DocValidationImageId: "$(DocValidationImageId)" - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - template: /eng/pipelines/templates/steps/fetch-package-list.yml - parameters: - JavaDocJarLocation: "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.groupId}}/${{artifact.name}}" - ArtifactName: ${{artifact.name}} - TargetBranch: $(TargetBranchName) - DocRepoLocation: $(DocRepoLocation) - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - TargetDocRepoOwner: "azure-sdk" - - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/steps/fetch-package-list.yml b/eng/pipelines/templates/steps/fetch-package-list.yml deleted file mode 100644 index db814352bed98..0000000000000 --- a/eng/pipelines/templates/steps/fetch-package-list.yml +++ /dev/null @@ -1,29 +0,0 @@ -parameters: - JavaDocJarLocation: '' - ArtifactName: '' - TargetBranch: '' - DocRepoLocation: '' - TargetDocRepoName: '' - TargetDocRepoOwner: '' - WorkingDirectory: $(System.DefaultWorkingDirectory) -steps: - - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - - task: PowerShell@2 - displayName: Fetch package list from JavaDoc jar - inputs: - pwsh: true - filePath: ${{ parameters.WorkingDirectory }}/eng/scripts/Fetch-PackageList-Javadoc.ps1 - arguments: > - -JavaDocJarLocation ${{ parameters.JavaDocJarLocation }} - -ArtifactName ${{ parameters.ArtifactName }} - -DocRepoLocation ${{ parameters.DocRepoLocation }} - continueOnError: true - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: ${{ parameters.TargetBranch }} - BaseRepoOwner: ${{ parameters.TargetDocRepoOwner }} - CommitMsg: "Update docs metadata for java package namespaces" - TargetRepoName: ${{ parameters.TargetDocRepoName }} - TargetRepoOwner: ${{ parameters.TargetDocRepoOwner }} - WorkingDirectory: ${{ parameters.DocRepoLocation }} diff --git a/eng/scripts/Fetch-PackageList-Javadoc.ps1 b/eng/scripts/Fetch-PackageList-Javadoc.ps1 deleted file mode 100644 index c02718241ab67..0000000000000 --- a/eng/scripts/Fetch-PackageList-Javadoc.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -<# - .SYNOPSIS - Fetch the package list from javadoc jar and checkin to docs/metadata - - .DESCRIPTION - The scripts is used for docs.ms to fetch namespace list for particular artifact. - - .PARAMETER JavaDocJarLocation - Specifies the JavaDoc jar location. - - .PARAMETER DocRepoLocation - Specifies Location of the root of the docs.microsoft.com reference doc location. - - .PARAMETER ArtifactName - The artifact name. E.g. azure-sdk-template -#> -[CmdletBinding()] -param ( - [string] $JavaDocJarLocation = "", - [string] $DocRepoLocation = "", - [string] $ArtifactName = "" -) -. (Join-Path $PSScriptRoot ../common/scripts/common.ps1) - -Write-Host "JavaDocJarLocation: $JavaDocJarLocation" -Write-Host "DocRepoLocation: $DocRepoLocation" -Write-Host "ArtifactName: $ArtifactName" - -$jarFile = Get-ChildItem $JavaDocJarLocation -Recurse -Include "$ArtifactName*-javadoc.jar" -Write-Host "The javadoc jar file is $jarFile." - -$version = $jarFile.Name -replace "$ArtifactName-(.*)-javadoc.jar", '$1' -Write-Host "The full version: $version" -$originalVersion = [AzureEngSemanticVersion]::ParseVersionString($version) -Write-Host "The origin version: $originalVersion" -$metadataMoniker = 'latest' -if ($originalVersion -and $originalVersion.IsPrerelease) { - $metadataMoniker = 'preview' -} -$packageNameLocation = "$DocRepoLocation/metadata/$metadataMoniker" -New-Item -ItemType Directory -Path $packageNameLocation -Force -Write-Host "The moniker $packageNameLocation" - -if (!(Test-Path $jarFile.FullName)) { - LogWarning "Skipping the doc publishing for $ArtifactName because we cannot find the javadoc jar." - return -} - -Fetch-Namespaces-From-Javadoc $jarFile.FullName "$packageNameLocation/$ArtifactName.txt" diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index e0c61c81db134..1a55c0e60f0a4 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -110,7 +110,7 @@ function Get-java-PackageInfoFromPackageFile ($pkg, $workingDirectory) $pkgId = $contentXML.project.artifactId $docsReadMeName = $pkgId -replace "^azure-" , "" $pkgVersion = $contentXML.project.version - $groupId = if ($contentXML.project.groupId -eq $null) { $contentXML.project.parent.groupId } else { $contentXML.project.groupId } + $groupId = if ($null -eq $contentXML.project.groupId) { $contentXML.project.parent.groupId } else { $contentXML.project.groupId } $releaseNotes = "" $readmeContent = "" @@ -141,6 +141,33 @@ function Get-java-PackageInfoFromPackageFile ($pkg, $workingDirectory) } } +# Defined in common.ps1 as: +# $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo" +function Get-java-DocsMsDevLanguageSpecificPackageInfo($packageInfo, $packageSourceOverride) { + # If the default namespace isn't in the package info then it needs to be added + # Can't check if (!$packageInfo.Namespaces) in strict mode because Namespaces won't exist + # at all. + if (!($packageInfo | Get-Member Namespaces)) { + $version = $packageInfo.Version + # If the dev version is set, use that + if ($packageInfo.DevVersion) { + $version = $packageInfo.DevVersion + } + $namespaces = Fetch-Namespaces-From-Javadoc $packageInfo.Name $packageInfo.Group $version + if ($namespaces.Count -gt 0) { + $packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $namespaces + } else { + # If nothing else, the package name will have "-" instead of "." which should make it evident + # that we couldn't find namespaces. + LogWarning "Unable to find namespaces for $($packageInfo.Name):$version, using the package name." + $tempNamespaces = @() + $tempNamespaces += $packageInfo.Name + $packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $tempNamespaces + } + } + return $packageInfo +} + # Stage and Upload Docs to blob Storage function Publish-java-GithubIODocs ($DocLocation, $PublicArtifactLocation) { diff --git a/eng/scripts/docs/Docs-ToC.ps1 b/eng/scripts/docs/Docs-ToC.ps1 index 03c68676ed28a..1bd41b0f08523 100644 --- a/eng/scripts/docs/Docs-ToC.ps1 +++ b/eng/scripts/docs/Docs-ToC.ps1 @@ -1,6 +1,6 @@ function Get-java-OnboardedDocsMsPackages($DocRepoLocation) { $packageOnboardingFiles = "$DocRepoLocation/package.json" - + $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFiles -Raw) $allPackages = @{} foreach ($spec in $onboardingSpec) { @@ -11,13 +11,13 @@ function Get-java-OnboardedDocsMsPackages($DocRepoLocation) { function Get-java-OnboardedDocsMsPackagesForMoniker ($DocRepoLocation, $moniker) { $packageOnboardingFiles = "$DocRepoLocation/package.json" - + $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFiles -Raw) if ("preview" -eq $moniker) { $onboardingSpec = $onboardingSpec | Where-Object { $_.output_path -eq "preview/docs-ref-autogen" } } elseif("latest" -eq $moniker) { $onboardingSpec = $onboardingSpec | Where-Object { $_.output_path -eq "docs-ref-autogen" } - } elseif ("legacy" -eq $moniker) { + } elseif ("legacy" -eq $moniker) { $onboardingSpec = $onboardingSpec | Where-Object { $_.output_path -eq "legacy/docs-ref-autogen" } } @@ -49,10 +49,12 @@ function GetPackageReadmeName ($packageMetadata) { return $packageLevelReadmeName } -function Get-java-PackageLevelReadme($packageMetadata) { +function Get-java-PackageLevelReadme($packageMetadata) { return GetPackageReadmeName -packageMetadata $packageMetadata } +# Defined in common.ps1 +# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData" function Get-java-DocsMsTocData($packageMetadata, $docRepoLocation) { $packageLevelReadmeName = GetPackageReadmeName -packageMetadata $packageMetadata $packageTocHeader = GetDocsTocDisplayName -pkg $packageMetadata @@ -99,97 +101,128 @@ function Get-java-DocsMsTocChildrenForManagementPackages($packageMetadata, $docR return ($children | Sort-Object | Get-Unique) } -# This is a helper function which fetch the java package namespaces from javadoc jar. -# Here are the major workflow: -# 1. Read the ${package}.txt under /metadata folder -# 2. If file not found, then download javadoc jar from maven repository. -# 3. If there is 'element-list' in javadoc jar, then copy to destination ${package}.txt. -# 4. If no 'element-list', then parse the 'overview-frame.html' for the namespaces and copy to destination. -# 5. If no 'overview-frame.html', then read folder 'com/azure/...' for namespaces. E.g some mgmt packages use this structure. -# 6. Otherwise, return empty children. -function Get-Toc-Children($package, $groupId, $version, $docRepoLocation, $folderName) { - # Looking for the txt - $filePath = Join-Path "$docRepoLocation/metadata/$folderName" "$package.txt" - if (!(Test-Path $filePath)) { - # Download from maven - # javadoc jar url. e.g.: https://repo1.maven.org/maven2/com/azure/azure-core/1.25.0/azure-core-1.25.0-javadoc.jar - $artifact = "${groupId}:${package}:${version}:javadoc" - # A temp folder - $tempDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "javadoc" - if (!(Test-Path $tempDirectory)) { - New-Item $tempDirectory -ItemType Directory | Out-Null - } - try { - Write-Host "mvn dependency:copy -Dartifact=$artifact -DoutputDirectory=$tempDirectory" - $javadocLocation = "$tempDirectory/$package-$version-javadoc.jar" - & 'mvn' dependency:copy -Dartifact="$artifact" -DoutputDirectory="$tempDirectory" | Out-Null - Write-Host "Download complete." - } - catch { - Write-Error "Not able to download javadoc jar from $artifact." - return @() - } - Fetch-Namespaces-From-Javadoc -jarFilePath $javadocLocation -destination $filePath +# This function is called within a loop. To prevent multiple reads of the same +# file data, this uses a script-scoped cache variable. +$script:PackageMetadataJsonLookup = $null +function GetPackageMetadataJsonLookup($docRepoLocation) { + if ($script:PackageMetadataJsonLookup) { + return $script:PackageMetadataJsonLookup } - if (!(Test-Path $filePath)) { - # Log and warn - Write-Host "Not able to find namespaces from javadoc jar $package-$version-javadoc.jar" + $script:PackageMetadataJsonLookup = @{} + $packageJsonFiles = Get-ChildItem $docRepoLocation/metadata/ -Filter *.json -Recurse + foreach ($packageJsonFile in $packageJsonFiles) { + $packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json -AsHashtable + + if (!$script:PackageMetadataJsonLookup.ContainsKey($packageJson.Name)) { + $script:PackageMetadataJsonLookup[$packageJson.Name] = @($packageJson) + } else { + $script:PackageMetadataJsonLookup[$packageJson.Name] += $packageJson + } } - return (Get-Content $filePath | ForEach-Object {$_.Trim()}) + + return $script:PackageMetadataJsonLookup } - -function Fetch-Namespaces-From-Javadoc ($jarFilePath, $destination) { - $tempLocation = (Join-Path ([System.IO.Path]::GetTempPath()) "jarFiles") - if (Test-Path $tempLocation) { - Remove-Item $tempLocation/* -Recurse -Force - } - else { - New-Item -ItemType Directory -Path $tempLocation -Force | Out-Null - } - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($jarFilePath, $tempLocation) - if (Test-Path "$tempLocation/element-list") { - # Rename and move to location - Write-Host "Copying the element-list to $destination..." - Copy-Item "$tempLocation/element-list" -Destination $destination - } - elseif (Test-Path "$tempLocation/overview-frame.html") { - Parse-Overview-Frame -filePath "$tempLocation/overview-frame.html" -destination $destination + +# Grab the namespaces from the json file +function Get-Toc-Children($package, $docRepoLocation) { + $packageTable = GetPackageMetadataJsonLookup $docRepoLocation + + $namespaces = @() + if ($packageTable.ContainsKey($package)) { + foreach ($entry in $packageTable[$package]) { + if ($entry.ContainsKey('Namespaces')) { + $namespaces += $entry['Namespaces'] + } + } } - elseif (Test-Path "$tempLocation/com") { - $originLocation = Get-Location - try { - Set-Location $tempLocation - $allFolders = Get-ChildItem "$tempLocation/com" -Recurse -Directory | - Where-Object {$_.GetFiles().Count -gt 0 -and $_.name -notmatch "class-use"} - foreach ($path in $allFolders) { - $path = (Resolve-Path $path -Relative) -replace "\./|\.\\" - $path = $path -replace "\\|\/", "." - Add-Content $destination -Value $path.Trim() + # Sort the array and clean out any dupes (there shouldn't be any but better safe than sorry) + $namespaces = $namespaces | Sort-Object -Unique + # Ensure that this always returns an array, even if there's one item or 0 items + Write-Output -NoEnumerate $namespaces +} + +# Given a library, groupId and version, return the list of namespaces +function Fetch-Namespaces-From-Javadoc($package, $groupId, $version) { + + $namespaces = @() + # Create a temporary directory to drop the jar into + $tempDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "${groupId}-${package}-${version}" + New-Item $tempDirectory -ItemType Directory | Out-Null + try { + # Download the Jar file + $artifact = "${groupId}:${package}:${version}:jar:javadoc" + Write-Host "mvn dependency:copy -Dartifact=""$artifact"" -DoutputDirectory=""$tempDirectory""" + $mvnResults = mvn dependency:copy -Dartifact="$artifact" -DoutputDirectory="$tempDirectory" + if ($LASTEXITCODE) { + LogWarning "Could not download javadoc artifact: $artifact" + $mvnResults | Write-Host + return $namespaces + } + + # Unpack the Jar file + $javadocLocation = "$tempDirectory/$package-$version-javadoc.jar" + $unpackDirectory = Join-Path $tempDirectory "unpackedJavadoc" + New-Item $unpackDirectory -ItemType Directory | Out-Null + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($javadocLocation, $unpackDirectory) + if (Test-Path "$unpackDirectory/element-list") { + # Grab the namespaces from the element-list. + Write-Host "processing element-list" + foreach($line in [System.IO.File]::ReadLines("$unpackDirectory/element-list")) { + if (-not [string]::IsNullOrWhiteSpace($line)) { + $namespaces += $line + } + } + } + elseif (Test-Path "$unpackDirectory/overview-frame.html") { + # Grab the namespaces from the overview-frame.html's package elements + $htmlBody = Get-Content "$unpackDirectory/overview-frame.html" + $packages = [RegEx]::Matches($htmlBody, "
  • (?.*?)<\/a><\/li>") + $namespaces = $packages | ForEach-Object { $_.Groups["package"].Value } + } + elseif (Test-Path "$unpackDirectory/com") { + # If all else fails, scrape the namespaces from the directories + $originLocation = Get-Location + try { + Set-Location $unpackDirectory + $allFolders = Get-ChildItem "$unpackDirectory/com" -Recurse -Directory | + Where-Object {$_.GetFiles().Count -gt 0 -and $_.name -notmatch "class-use"} + foreach ($path in $allFolders) { + $path = (Resolve-Path $path -Relative) -replace "\./|\.\\" + $path = $path -replace "\\|\/", "." + # add the namespace to the list + $namespaces += $path.Trim() + } + } + finally { + Set-Location $originLocation } } - finally { - Set-Location $originLocation + else { + Write-Error "Can't find namespaces from javadoc jar $javadocLocation." } } - else { - Write-Error "Can't find namespaces from javadoc jar jarFilePath." + catch { + Write-Error "Exception=$($_.Exception.Message)" } + finally { + # everything is contained within the temp directory, clean it up every time + if (Test-Path $tempDirectory) { + Remove-Item $tempDirectory -Recurse -Force + } + } + + $namespaces = $namespaces | Sort-Object -Unique + # Make sure this always returns an array + Write-Output -NoEnumerate $namespaces } function Get-java-RepositoryLink ($packageInfo) { $groupIdPath = $packageInfo.GroupId -replace "\.", "/" return "$PackageRepositoryUri/$groupIdPath/$($packageInfo.Package)" } - -function Parse-Overview-Frame ($filePath, $destination) { - $htmlBody = Get-Content $filePath - $packages = [RegEx]::Matches($htmlBody, "
  • (?.*?)<\/a><\/li>") - $namespaces = $packages | ForEach-Object { $_.Groups["package"].Value } - Add-Content -Path $destination -Value $namespaces -} function Get-java-UpdatedDocsMsToc($toc) { $services = $toc[0].items @@ -258,7 +291,7 @@ function Get-java-UpdatedDocsMsToc($toc) { name = "Resource Management" href = "~/docs-ref-services/{moniker}/resourcemanager-msi-readme.md" children = @("com.azure.resourcemanager.msi*") - }, + }, [PSCustomObject]@{ name = "Client" children = @( @@ -329,7 +362,7 @@ function Get-java-UpdatedDocsMsToc($toc) { [PSCustomObject]@{ name = "Management" children = @( - "com.microsoft.azure.cognitiveservices.language.luis*", + "com.microsoft.azure.cognitiveservices.language.luis*", "com.microsoft.azure.cognitiveservices.language.luis.authoring*") }) }