Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Python regenerate docindex #14023

Closed
wants to merge 1 commit into from
Closed

Python regenerate docindex #14023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 61 additions & 34 deletions eng/common/scripts/Package-Properties.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class PackageProps

PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
{
$this.Initialize($name, $version, $directoryPath, $serviceDirectory)
$this.Initialize($name, $version, $directoryPath, $serviceDirectory, "")
}

PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory, [string]$group = "")
Expand All @@ -23,14 +23,15 @@ class PackageProps
[string]$name,
[string]$version,
[string]$directoryPath,
[string]$serviceDirectory
[string]$serviceDirectory,
[string]$group
)
{
$this.Name = $name
$this.Version = $version
$this.DirectoryPath = $directoryPath
$this.ServiceDirectory = $serviceDirectory

$this.Group = $group
if (Test-Path (Join-Path $directoryPath "README.md"))
{
$this.ReadMePath = Join-Path $directoryPath "README.md"
Expand All @@ -49,20 +50,9 @@ class PackageProps
$this.ChangeLogPath = $null
}
}

hidden [void]Initialize(
[string]$name,
[string]$version,
[string]$directoryPath,
[string]$serviceDirectory,
[string]$group
)
{
$this.Initialize($name, $version, $directoryPath, $serviceDirectory)
$this.Group = $group
}
}

Import-Module powershell-yaml
# Takes package name and service Name
# Returns important properties of the package as related to the language repo
# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
Expand All @@ -77,7 +67,6 @@ function Get-PkgProperties
[string]$ServiceDirectory
)

$pkgDirectoryPath = $null
$serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceDirectory
if (!(Test-Path $serviceDirectoryPath))
{
Expand Down Expand Up @@ -110,7 +99,7 @@ function Get-PkgProperties
# Takes ServiceName and Repo Root Directory
# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified
# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
function Get-AllPkgProperties ([string]$ServiceDirectory = $null)
function Get-AllPkgProperties ([string]$ServiceDirectory = $null, [bool]$includeMgmt = $true)
{
$pkgPropsResult = @()

Expand All @@ -120,38 +109,76 @@ function Get-AllPkgProperties ([string]$ServiceDirectory = $null)
foreach ($dir in (Get-ChildItem $searchDir -Directory))
{
$serviceDir = Join-Path $searchDir $dir.Name
$ciFiles = fetchCiFiles -serviceDirectory $serviceDir -includeincludeMgmtPkg $includeMgmt

if (Test-Path (Join-Path $serviceDir "ci.yml"))
foreach ($ciFile in $ciFiles)
{
$activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
if ($activePkgList -ne $null)
{
$pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -ServiceDirectory $dir.Name -pkgPropsResult $pkgPropsResult
}
$pkgPropsResult = Operate-OnPackages -ciFilePath $ciFile -ServiceDirectory $dir.Name -pkgPropsResult $pkgPropsResult
}
}
}
else
{
$serviceDir = Join-Path $RepoRoot "sdk" $ServiceDirectory
if (Test-Path (Join-Path $serviceDir "ci.yml"))
{
$activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
if ($activePkgList -ne $null)
{
$pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -ServiceDirectory $ServiceDirectory -pkgPropsResult $pkgPropsResult
}
$ciFiles = fetchCiFiles -serviceDirectory $serviceDir -includeincludeMgmtPkg $includeMgmt
foreach ($ciFile in $ciFiles) {
$pkgPropsResult = Operate-OnPackages -ciFilePath $ciFile -ServiceDirectory $ServiceDirectory -pkgPropsResult $pkgPropsResult
}
}

return $pkgPropsResult
}

function Operate-OnPackages ($activePkgList, $ServiceDirectory, [Array]$pkgPropsResult)
function GetMetaData($lang){
switch ($lang) {
"java" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
break
}
".net" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv"
break
}
"python" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv"
break
}
"javascript" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv"
break
}
default {
Write-Host "Unrecognized Language: $language"
exit(1)
}
}

$metadataResponse = Invoke-RestMethod -Uri $metadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv

return $metadataResponse
}

function fetchCiFiles([string]$serviceDirectory, [bool]$includeMgmtPkg) {
# We'd better to display artifacts based on the order of ci.yml, ci.data.yml and ci.mgmt.yml.
$ciFiles = @()
if (Test-Path (Join-Path $serviceDirectory ci.yml)) {
$ciFiles += (Join-Path $serviceDirectory ci.yml)
}
if (Test-Path (Join-Path $serviceDirectory ci.data.yml)) {
$ciFiles += (Join-Path $serviceDirectory ci.data.yml)
}
if ($includeMgmt -and (Test-Path (Join-Path $serviceDirectory ci.mgmt.yml))) {
$includeMgmtPkg += (Join-Path $serviceDirectory ci.mgmt.yml)
}
return $ciFiles
}

function Operate-OnPackages ($ciFilePath, $ServiceDirectory, [Array]$pkgPropsResult)
{
foreach ($pkg in $activePkgList)
$activePkgList = Get-PkgListFromYml -ciYmlPath $ciFilePath
foreach ($pkg in $activePkgList.name)
{
$pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceDirectory $ServiceDirectory
$pkgProps = Get-PkgProperties -PackageName $pkg -ServiceDirectory $ServiceDirectory
$pkgPropsResult += $pkgProps
}
return $pkgPropsResult
Expand All @@ -161,7 +188,7 @@ function Get-PkgListFromYml ($ciYmlPath)
{
$ProgressPreference = "SilentlyContinue"
Register-PSRepository -Default -ErrorAction:SilentlyContinue
Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
# Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
$ciYmlContent = Get-Content $ciYmlPath -Raw
$ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered
if ($ciYmlObj.Contains("stages"))
Expand Down
112 changes: 112 additions & 0 deletions eng/docgeneration/Generate-DocIndex.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
[CmdletBinding()]
Param (
$RepoRoot,
$DocGenDir,
$ExcludeDocIndexFile = "$PSScriptRoot/exclude-doc-index.json",
$lang = "python"
)

# Import required scripts
. (Join-Path $PSScriptRoot ../common/scripts/common.ps1)

# There are some artifact that show up, due to the way we do discovery, that are never shipped.
# Keep a list of those here and anything we don't want to ship can be added to here which will
# cause them to get skipped when generating the DocIndex
$ExcludeData = Get-Content -Raw -Path $ExcludeDocIndexFile | ConvertFrom-Json

Write-Verbose "Name Reccuring paths with variable names"
$DocFxTool = "${RepoRoot}/docfx/docfx.exe"
$DocOutDir = "${RepoRoot}/docfx_project"

Write-Verbose "Initializing Default DocFx Site..."
& "${DocFxTool}" init -q -o "${DocOutDir}"

Write-Verbose "Copying template and configuration..."
New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force
Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force

Write-Verbose "Creating Index using service directory and package names from repo..."

# The list of services is being constructed from the directory list under the sdk folder
# which, right now, only contains client/data directories. When management is moved to
# the under sdk it'll automatically get picked up.
$ServiceListData = Get-ChildItem "${RepoRoot}/sdk" -Directory | Where-Object {$_.PSIsContainer}
$YmlPath = "${DocOutDir}/api"
New-Item -Path $YmlPath -Name "toc.yml" -Force

$metadata = GetMetaData -lang $lang
foreach ($serviceDir in $ServiceListData)
{
$dirName = $serviceDir.name
if ($ExcludeData -and ($ExcludeData.services -contains $dirName)) {
continue
}
$serviceName = ""
# Store the list of artifacts into the arrays and write them into the .md file
# after processing the list of subdirectories. This will allow the correct
# division of the artifacts under the Client or Management headings
$clientArr = @()
$artifacts = Get-AllPkgProperties -ServiceDirectory $dirName
foreach ($pkgInfo in $artifacts) {
if (!$serviceName) {
$serviceInfo = $metadata | ? { $_.Package -eq $pkgInfo.Name -and $_.GroupId -eq $pkgInfo.Group}
if ($serviceInfo -and $serviceInfo.ServiceName) {
$serviceName = $serviceInfo.ServiceName
}
else {
$serviceName = (Get-Culture).TextInfo.ToTitleCase($dirName.ToLower())
}
}
if ($ExcludeData -and ($ExcludeData.artifacts -contains $pkgInfo.Name)) {
continue
}
$clientArr += $pkgInfo.Name
}

# Only create this if there's something to create
#if (($clientArr.Count -gt 0) -or ($mgmtArr.Count -gt 0))
if ($clientArr.Count -gt 0)
{
New-Item -Path $YmlPath -Name "${dirName}.md" -Force
Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n href: ${dirName}.md"
# loop through the arrays and add the appropriate artifacts under the appropriate headings
if ($clientArr.Count -gt 0)
{
Add-Content -Path "$($YmlPath)/${dirName}.md" -Value "# Client Libraries"
foreach($lib in $clientArr)
{
Write-Host "Write $($lib) to ${dirName}.md"
Add-Content -Path "$($YmlPath)/${dirName}.md" -Value "#### $lib"
}
}
# For the moment there are no management docs and with the way some of the libraries
# in management are versioned is a bit wonky. They aren't versioned by releasing a new
# version with the same groupId/artifactId, they're versioned with the same artifactId
# and version with a different groupId and the groupId happens to include the date. For
# example, the artifact/version of azure-mgmt-storage:1.0.0-beta has several different
# groupIds. com.microsoft.azure.storage.v2016_01_01, com.microsoft.azure.storage.v2017_10_01,
# com.microsoft.azure.storage.v2018_11_01 etc.
#if ($mgmtArr.Count -gt 0)
#{
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Management Libraries"
# foreach($lib in $mgmtArr)
# {
# Write-Output "Write $($lib) to $($Dir.Name).md"
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
# }
#}
}
}

Write-Verbose "Creating Site Title and Navigation..."
New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for Python APIs`r`n href: api/`r`n homepage: api/index.md"

Write-Verbose "Copying root markdowns"
Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force

Write-Verbose "Building site..."
& "${DocFxTool}" build "${DocOutDir}/docfx.json"

Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
76 changes: 76 additions & 0 deletions eng/docgeneration/assets/logo.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading