-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Test the common Generate_docindex scripts in each lang repo.
- Loading branch information
Showing
3 changed files
with
163 additions
and
158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,164 +1,150 @@ | ||
| # Generates an index page for cataloging different versions of the Docs | ||
| [CmdletBinding()] | ||
| Param ( | ||
| $DocFx, | ||
| $RepoRoot, | ||
| $DocGenDir | ||
| ) | ||
|
|
||
| Write-Verbose "Name Reccuring paths with variable names" | ||
| $DocFxTool = "${RepoRoot}/docfx/docfx.exe" | ||
| $DocOutDir = "${RepoRoot}/docfx_project" | ||
|
|
||
| Write-Verbose "Initializing Default DocFx Site..." | ||
| & "${DocFxTool}" init -q -o "${DocOutDir}" | ||
|
|
||
| Write-Verbose "Copying template and configuration..." | ||
| New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" | ||
| Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse | ||
| Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force | ||
|
|
||
| Write-Verbose "Creating Index using service directory and package names from repo..." | ||
| # The service mapper is used to map the directory names to the service names to produce | ||
| # a more friendly index. If something isn't in the mapper then the default will just be | ||
| # the service name in all caps | ||
| $serviceMapHash = Get-Content -Path "${DocGenDir}/service-mapper.json" | ConvertFrom-Json -AsHashtable | ||
|
|
||
| # There are some artifact that show up, due to the way we do discovery, that are never shipped. | ||
| # Keep a list of those here and anything we don't want to ship can be added to here which will | ||
| # cause them to get skipped when generating the DocIndex | ||
| $ArtifactsToSkip = ( | ||
| 'azure-cosmos-benchmark', | ||
| 'azure-sdk-template' | ||
| ) | ||
|
|
||
| # The list of services is being constructed from the directory list under the sdk folder | ||
| # which, right now, only contains client/data directories. When management is moved to | ||
| # the under sdk it'll automatically get picked up. | ||
| $ServiceListData = Get-ChildItem "${RepoRoot}/sdk" -Directory | ||
| $YmlPath = "${DocOutDir}/api" | ||
| New-Item -Path $YmlPath -Name "toc.yml" -Force | ||
| foreach ($Dir in $ServiceListData) | ||
| { | ||
| $mappedDir = "" | ||
| if ($serviceMapHash.ContainsKey($Dir.Name)) | ||
| { | ||
| $mappedDir = $serviceMapHash[$Dir.Name] | ||
| } | ||
| else | ||
| { | ||
| $mappedDir = $Dir.Name.ToUpper() | ||
| . (Join-Path $PSScriptRoot ../eng/common/scripts/common.ps1) | ||
| $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" | ||
|
|
||
| # Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, | ||
| # the function will return the csv metadata back as part of response. | ||
| function Get-CSVMetadata ([string]$MetadataUri) { | ||
| $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv | ||
| return $metadataResponse | ||
| } | ||
|
|
||
| # Given the github io blob storage url and language regex, | ||
| # the helper function will return a list of artifact names. | ||
| function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) { | ||
| LogDebug "Reading artifact from storage blob ..." | ||
| $returnedArtifacts = @() | ||
| $pageToken = "" | ||
| Do { | ||
| $resp = "" | ||
| if (!$pageToken) { | ||
| # First page call. | ||
| $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl | ||
| } | ||
| else { | ||
| # Next page call | ||
| $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken" | ||
| $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken | ||
| } | ||
| # Convert to xml documents. | ||
| $xmlDoc = [xml](removeBomFromString $resp) | ||
| foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) { | ||
| # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector" | ||
| $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement | ||
| $returnedArtifacts += $artifact | ||
| } | ||
| # Fetch page token | ||
| $pageToken = $xmlDoc.EnumerationResults.NextMarker | ||
| } while ($pageToken) | ||
| return $returnedArtifacts | ||
| } | ||
|
|
||
| # The sequence of Bom bytes differs by different encoding. | ||
| # The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api. | ||
| # Return the original string if not in BOM utf-8 sequence. | ||
| function RemoveBomFromString([string]$bomAwareString) { | ||
| if ($bomAwareString.length -le 3) { | ||
| return $bomAwareString | ||
| } | ||
|
|
||
| # Store the list of artifacts into the arrays and write them into the .md file | ||
| # after processing the list of subdirectories. This will allow the correct | ||
| # division of the artifacts under the Client or Management headings | ||
| $clientArr = @() | ||
| $mgmtArr = @() | ||
|
|
||
| $PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude changelog, faq, .github, build | ||
| if (($PkgList | Measure-Object).count -eq 0) | ||
| { | ||
| continue | ||
| $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf) | ||
| # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding. | ||
| $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3)) | ||
| if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) { | ||
| return $bomAwareString.Substring(3) | ||
| } | ||
| foreach ($Pkg in $PkgList) | ||
| { | ||
| # Load the pom file to pull the artifact name and grab the | ||
| # parent's relative path to see which parent pom is being | ||
| # used to determine whether or not the artifact is client | ||
| # or management. | ||
| $PomPath = Join-Path -Path $Pkg -ChildPath "pom.xml" | ||
|
|
||
| # no pom file = nothing to process | ||
| if (Test-Path -path $PomPath) | ||
| { | ||
| $xml = New-Object xml | ||
| $xml.Load($PomPath) | ||
|
|
||
| # Get the artifactId from the POM | ||
| $artifactId = $xml.project.artifactId | ||
|
|
||
| $parent = $xml.project.parent.relativePath | ||
|
|
||
| # If this is an artifact that isn't shipping then just | ||
| # move on to the next one | ||
| if ($ArtifactsToSkip -contains $artifactId) | ||
| { | ||
| Write-Output "skipping $artifactId" | ||
| continue | ||
| } | ||
|
|
||
| # If the parent is null or empty then the pom isn't directly including | ||
| # one of the pom.[client|data|management].xml and needs to be specially | ||
| # handled | ||
| if (("" -eq $parent) -or ($null -eq $parent)) | ||
| { | ||
| # Cosmos has a root pom which includes pom.client.xml that won't | ||
| # be detected by this logic. It's easier to deal with specially | ||
| # than it is to try and climb the pom chain here. | ||
| if ($Dir.BaseName -eq 'cosmos') | ||
| { | ||
| $clientArr += $artifactId | ||
| } | ||
| else | ||
| { | ||
| Write-Host "*snowflake* Pom $PomPath, has a null or empty relative path." | ||
| } | ||
| } | ||
| else | ||
| { | ||
| if (($parent.IndexOf('azure-client-sdk-parent') -ne -1) -or ($parent.IndexOf('azure-data-sdk-parent') -ne -1)) | ||
| { | ||
| $clientArr += $artifactId | ||
| } | ||
| else | ||
| { | ||
| $mgmtArr += $artifactId | ||
| } | ||
| return $bomAwareString | ||
| } | ||
|
|
||
| function Get-TocMapping { | ||
| Param ( | ||
| [Parameter(Mandatory = $true)] [Object[]] $metadata, | ||
| [Parameter(Mandatory = $true)] [String[]] $artifacts | ||
| ) | ||
| # Used for sorting the toc display order | ||
| $orderServiceMapping = @{} | ||
|
|
||
| foreach ($artifact in $artifacts) { | ||
| $packageInfo = $metadata | ? {$_.Package -eq $artifact} | ||
|
|
||
| if ($packageInfo -and $packageInfo[0].Hide -eq 'true') { | ||
| LogDebug "The artifact $artifact set 'Hide' to 'true'." | ||
| continue | ||
| } | ||
| $serviceName = "" | ||
| if (!$packageInfo -or !$packageInfo[0].ServiceName) { | ||
| LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " | ||
| # If no service name retrieved, print out warning message, and put it into Other page. | ||
| $serviceName = "Other" | ||
| } | ||
| else { | ||
| if ($packageInfo.Length -gt 1) { | ||
| LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " | ||
| } | ||
| $serviceName = $packageInfo[0].ServiceName.Trim() | ||
| } | ||
| $orderServiceMapping[$artifact] = $serviceName | ||
| } | ||
| # Only create this if there's something to create | ||
| #if (($clientArr.Count -gt 0) -or ($mgmtArr.Count -gt 0)) | ||
| if ($clientArr.Count -gt 0) | ||
| { | ||
| New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force | ||
| Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($mappedDir)`r`n href: $($Dir.Name).md" | ||
| # loop through the arrays and add the appropriate artifacts under the appropriate headings | ||
| if ($clientArr.Count -gt 0) | ||
| { | ||
| Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client Libraries" | ||
| foreach($lib in $clientArr) | ||
| { | ||
| Write-Host "Write $($lib) to $($Dir.Name).md" | ||
| Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib" | ||
| return $orderServiceMapping | ||
| } | ||
|
|
||
| function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) { | ||
| LogDebug "Start generating the docfx toc and build docfx site..." | ||
| $DocOutDir = "${RepoRoot}/docfx_project" | ||
|
|
||
| LogDebug "Initializing Default DocFx Site..." | ||
| & $($DocFx) init -q -o "${DocOutDir}" | ||
| # The line below is used for testing in local | ||
| # docfx init -q -o "${DocOutDir}" | ||
| LogDebug "Copying template and configuration..." | ||
| New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force | ||
| Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse | ||
| Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force | ||
| $YmlPath = "${DocOutDir}/api" | ||
| New-Item -Path $YmlPath -Name "toc.yml" -Force | ||
| $visitedService = @{} | ||
| # Sort and display toc service name by alphabetical order. | ||
| foreach ($serviceMapping in $tocContent.getEnumerator() | Sort Value) { | ||
| $artifact = $serviceMapping.Key | ||
| $serviceName = $serviceMapping.Value | ||
| $fileName = ($serviceName -replace '\s', '').ToLower().Trim() | ||
| if ($visitedService.ContainsKey($serviceName)) { | ||
| Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" | ||
| } | ||
| else { | ||
| Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n href: ${fileName}.md" | ||
| New-Item -Path $YmlPath -Name "${fileName}.md" -Force | ||
| Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" | ||
| $visitedService[$serviceName] = $true | ||
| } | ||
| } | ||
| # For the moment there are no management docs and with the way some of the libraries | ||
| # in management are versioned is a bit wonky. They aren't versioned by releasing a new | ||
| # version with the same groupId/artifactId, they're versioned with the same artifactId | ||
| # and version with a different groupId and the groupId happens to include the date. For | ||
| # example, the artifact/version of azure-mgmt-storage:1.0.0-beta has several different | ||
| # groupIds. com.microsoft.azure.storage.v2016_01_01, com.microsoft.azure.storage.v2017_10_01, | ||
| # com.microsoft.azure.storage.v2018_11_01 etc. | ||
| #if ($mgmtArr.Count -gt 0) | ||
| #{ | ||
| # Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Management Libraries" | ||
| # foreach($lib in $mgmtArr) | ||
| # { | ||
| # Write-Output "Write $($lib) to $($Dir.Name).md" | ||
| # Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib" | ||
| # } | ||
| #} | ||
| } | ||
| } | ||
|
|
||
| Write-Verbose "Creating Site Title and Navigation..." | ||
| New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force | ||
| Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for Java APIs`r`n href: api/`r`n homepage: api/index.md" | ||
| # Generate toc homepage. | ||
| LogDebug "Creating Site Title and Navigation..." | ||
| New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force | ||
| Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n href: api/`r`n homepage: api/index.md" | ||
|
|
||
| Write-Verbose "Copying root markdowns" | ||
| Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force | ||
| LogDebug "Copying root markdowns" | ||
| Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force | ||
| Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force | ||
|
|
||
| Write-Verbose "Building site..." | ||
| & "${DocFxTool}" build "${DocOutDir}/docfx.json" | ||
| LogDebug "Building site..." | ||
| & $($DocFx) build "${DocOutDir}/docfx.json" | ||
| # The line below is used for testing in local | ||
| # docfx build "${DocOutDir}/docfx.json" | ||
| Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force | ||
| } | ||
|
|
||
| Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force | ||
| if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetGithubIoDocIndexFn }).Count -gt 0) | ||
| { | ||
| &$GetGithubIoDocIndexFn | ||
| } | ||
| else | ||
| { | ||
| LogWarning "The function '$GetGithubIoDocIndexFn' was not found." | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters