Skip to content

Commit 4fc3fb6

Browse files
azure-sdkdanieljurek
azure-sdk
and
danieljurek
authored
Add logic to generate docs.ms ToC (Azure#23086)
Co-authored-by: Daniel Jurek <[email protected]>
1 parent d21976b commit 4fc3fb6

File tree

2 files changed

+318
-1
lines changed

2 files changed

+318
-1
lines changed

‎eng/common/scripts/Update-DocsMsToc.ps1‎

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
<#
2+
.SYNOPSIS
3+
Update unified ToC file for publishing reference docs on docs.microsoft.com
4+
5+
.DESCRIPTION
6+
Given a doc repo location and a location to output the ToC generate a Unified
7+
Table of Contents:
8+
9+
* Get list of packages onboarded to docs.microsoft.com (domain specific)
10+
* Get metadata for onboarded packages from metadata CSV
11+
* Build a sorted list of services
12+
* Add ToC nodes for the service
13+
* Add "Core" packages to the bottom of the ToC under "Other"
14+
15+
ToC node layout:
16+
* Service (service level overview page)
17+
* Client Package 1 (package level overview page)
18+
* Client Package 2 (package level overview page)
19+
...
20+
* Management
21+
* Management Package 1
22+
* Management Package 2
23+
...
24+
25+
.PARAMETER DocRepoLocation
26+
Location of the documentation repo. This repo may be sparsely checked out
27+
depending on the requirements for the domain
28+
29+
.PARAMETER OutputLocation
30+
Output location for unified reference yml file
31+
32+
#>
33+
34+
param(
35+
[Parameter(Mandatory = $true)]
36+
[string] $DocRepoLocation,
37+
38+
[Parameter(Mandatory = $true)]
39+
[string] $OutputLocation
40+
)
41+
Set-StrictMode -Version 3
42+
. $PSScriptRoot/common.ps1
43+
. $PSScriptRoot/Helpers/PSModule-Helpers.ps1
44+
45+
Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
46+
47+
function GetClientPackageNode($clientPackage) {
48+
$packageInfo = &$GetDocsMsTocDataFn `
49+
-packageMetadata $clientPackage `
50+
-docRepoLocation $DocRepoLocation
51+
52+
return [PSCustomObject]@{
53+
name = $packageInfo.PackageTocHeader
54+
href = $packageInfo.PackageLevelReadmeHref
55+
# This is always one package and it must be an array
56+
children = $packageInfo.TocChildren
57+
};
58+
}
59+
60+
function GetPackageKey($pkg) {
61+
$pkgKey = $pkg.Package
62+
$groupId = $null
63+
64+
if ($pkg.PSObject.Members.Name -contains "GroupId") {
65+
$groupId = $pkg.GroupId
66+
}
67+
68+
if ($groupId) {
69+
$pkgKey = "${groupId}:${pkgKey}"
70+
}
71+
72+
return $pkgKey
73+
}
74+
75+
function GetPackageLookup($packageList) {
76+
$packageLookup = @{}
77+
78+
foreach ($pkg in $packageList) {
79+
$pkgKey = GetPackageKey $pkg
80+
81+
# We want to prefer updating non-hidden packages but if there is only
82+
# a hidden entry then we will return that
83+
if (!$packageLookup.ContainsKey($pkgKey) -or $packageLookup[$pkgKey].Hide -eq "true") {
84+
$packageLookup[$pkgKey] = $pkg
85+
}
86+
else {
87+
# Warn if there are more then one non-hidden package
88+
if ($pkg.Hide -ne "true") {
89+
Write-Host "Found more than one package entry for $($pkg.Package) selecting the first non-hidden one."
90+
}
91+
}
92+
93+
if ($pkg.PSObject.Members.Name -contains "GroupId" -and ($pkg.New -eq "true") -and $pkg.Package) {
94+
$pkgKey = $pkg.Package
95+
if (!$packageLookup.ContainsKey($pkgKey)) {
96+
$packageLookup[$pkgKey] = $pkg
97+
}
98+
else {
99+
$packageValue = $packageLookup[$pkgKey]
100+
Write-Host "Found more than one package entry for $($packageValue.Package) selecting the first one with groupId $($packageValue.GroupId), skipping $($pkg.GroupId)"
101+
}
102+
}
103+
}
104+
105+
return $packageLookup
106+
}
107+
108+
$onboardedPackages = &$GetOnboardedDocsMsPackagesFn `
109+
-DocRepoLocation $DocRepoLocation
110+
111+
# This criteria is different from criteria used in `Update-DocsMsPackages.ps1`
112+
# because we need to generate ToCs for packages which are not necessarily "New"
113+
# in the metadata AND onboard legacy packages (which `Update-DocsMsPackages.ps1`
114+
# does not do)
115+
$metadata = (Get-CSVMetadata).Where({
116+
$_.Package `
117+
-and $onboardedPackages.ContainsKey($_.Package) `
118+
-and $_.Hide -ne 'true'
119+
})
120+
121+
$fileMetadata = @()
122+
foreach ($metadataFile in Get-ChildItem "$DocRepoLocation/metadata/*/*.json" -Recurse) {
123+
$fileContent = Get-Content $metadataFile -Raw
124+
$metadataEntry = ConvertFrom-Json $fileContent
125+
126+
if ($metadataEntry) {
127+
$fileMetadata += $metadataEntry
128+
}
129+
}
130+
131+
# Add file metadata information to package metadata from metadata CSV. Because
132+
# metadata can exist for packages in both preview and GA there may be more than
133+
# one file metadata entry. If that is the case keep the first entry found. We
134+
# only use the `DirectoryPath` property from the json file metadata at this time
135+
for ($i = 0; $i -lt $metadata.Count; $i++) {
136+
foreach ($fileEntry in $fileMetadata) {
137+
if ($fileEntry.Name -eq $metadata[$i].Package) {
138+
if ($metadata[$i].PSObject.Members.Name -contains "FileMetadata") {
139+
Write-Host "File metadata already added for $($metadata[$i].Package). Keeping the first entry found."
140+
continue
141+
}
142+
143+
Add-Member `
144+
-InputObject $metadata[$i] `
145+
-MemberType NoteProperty `
146+
-Name FileMetadata `
147+
-Value $fileEntry
148+
}
149+
}
150+
}
151+
152+
$packagesForToc = @{}
153+
foreach ($metadataEntry in (GetPackageLookup $metadata).Values) {
154+
if (!$metadataEntry.ServiceName) {
155+
LogWarning "Empty ServiceName for package `"$($metadataEntry.Package)`". Skipping."
156+
continue
157+
}
158+
$packagesForToc[$metadataEntry.Package] = $metadataEntry
159+
}
160+
161+
# Get unique service names and sort alphabetically to act as the service nodes
162+
# in the ToC
163+
$services = @{}
164+
foreach ($package in $packagesForToc.Values) {
165+
if ($package.ServiceName -eq 'Other') {
166+
# Skip packages under the service category "Other". Those will be handled
167+
# later
168+
continue
169+
}
170+
if (!$services.ContainsKey($package.ServiceName)) {
171+
$services[$package.ServiceName] = $true
172+
}
173+
}
174+
$serviceNameList = $services.Keys | Sort-Object
175+
176+
177+
$toc = @()
178+
foreach ($service in $serviceNameList) {
179+
Write-Host "Building service: $service"
180+
181+
$packageItems = @()
182+
183+
# Client packages get individual entries
184+
$clientPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('client' -eq $_.Type) })
185+
$clientPackages = $clientPackages | Sort-Object -Property Package
186+
if ($clientPackages) {
187+
foreach ($clientPackage in $clientPackages) {
188+
$packageItems += GetClientPackageNode -clientPackage $clientPackage
189+
}
190+
}
191+
192+
193+
# All management packages go under a single `Management` header in the ToC
194+
$mgmtPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('mgmt' -eq $_.Type) })
195+
$mgmtPackages = $mgmtPackages | Sort-Object -Property Package
196+
if ($mgmtPackages) {
197+
$children = &$GetDocsMsTocChildrenForManagementPackagesFn `
198+
-packageMetadata $mgmtPackages `
199+
-docRepoLocation $DocRepoLocation
200+
201+
$packageItems += [PSCustomObject]@{
202+
name = 'Management'
203+
# There could be multiple packages, ensure this is treated as an array
204+
# even if it is a single package
205+
children = @($children)
206+
};
207+
}
208+
209+
$uncategorizedPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and !(@('client', 'mgmt') -contains $_.Type) })
210+
if ($uncategorizedPackages) {
211+
foreach ($package in $uncategorizedPackages) {
212+
LogWarning "Uncategorized package for service: $service - $($package.Package). Package not onboarded."
213+
}
214+
}
215+
216+
$serviceReadmeBaseName = $service.ToLower().Replace(' ', '-')
217+
$serviceTocEntry = [PSCustomObject]@{
218+
name = $service;
219+
href = "~/docs-ref-services/{moniker}/$serviceReadmeBaseName.md"
220+
landingPageType = 'Service'
221+
items = @($packageItems)
222+
}
223+
224+
$toc += $serviceTocEntry
225+
}
226+
227+
# Core packages belong under the "Other" node in the ToC
228+
$otherPackageItems = New-Object -TypeName System.Collections.Generic.List[PSCustomObject]
229+
$otherPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq 'Other' })
230+
$otherPackages = $otherPackages | Sort-Object -Property DisplayName
231+
232+
if ($otherPackages) {
233+
foreach ($otherPackage in $otherPackages) {
234+
$segments = $otherPackage.DisplayName.Split('-').ForEach({ $_.Trim() })
235+
236+
237+
if ($segments.Count -gt 1) {
238+
$currentNode = $otherPackageItems
239+
240+
# Iterate up to the penultimate item in the array so that the final item
241+
# in the array can be added as a leaf node. Since the array always has at
242+
# least two elements this iteration will cover at least the first element.
243+
# e.g. @(0, 1)[0..0] => 0
244+
foreach ($segment in $segments[0..($segments.Count - 2)]) {
245+
$matchingNode = $currentNode.Where({ $_.name -eq $segment })
246+
247+
# ToC nodes can be "branches" which contain 0 or more branch
248+
# or leaf nodes in an "items" field OR they can be leaf nodes which have
249+
# a "children" field which can only contain package names or namespaces.
250+
# A node cannot contain both "items" and "children". If a node already
251+
# has a "children" field then it is a leaf node and cannot take
252+
# additional branch nodes.
253+
# Children are added using the `GetClientPackageNode` function
254+
if ($matchingNode -and $matchingNode.PSObject.Members.Name -contains "children") {
255+
LogWarning "Cannot create nested entry for package $($otherPackage.Package) because Segment `"$segment`" in the DisplayName $($otherPackage.DisplayName) is already a leaf node. Excluding package: $($otherPackage.Package)"
256+
$currentNode = $null
257+
break
258+
}
259+
260+
if ($matchingNode) {
261+
$currentNode = $matchingNode[0].items
262+
}
263+
else {
264+
$newNode = [PSCustomObject]@{
265+
name = $segment
266+
landingPageType = 'Service'
267+
items = New-Object -TypeName System.Collections.Generic.List[PSCustomObject]
268+
}
269+
$currentNode.Add($newNode)
270+
$currentNode = $newNode.items
271+
}
272+
}
273+
274+
if ($null -ne $currentNode) {
275+
$otherPackage.DisplayName = $segments[$segments.Count - 1]
276+
$currentNode.Add((GetClientPackageNode $otherPackage))
277+
}
278+
279+
}
280+
else {
281+
$otherPackageItems.Add((GetClientPackageNode $otherPackage))
282+
}
283+
}
284+
}
285+
286+
$toc += [PSCustomObject]@{
287+
name = 'Other';
288+
landingPageType = 'Service';
289+
items = $otherPackageItems + @(
290+
[PSCustomObject]@{
291+
name = "Uncategorized Packages";
292+
landingPageType = 'Service';
293+
# All onboarded packages which have not been placed in the ToC will be
294+
# handled by the docs system here. In this case the list would consist of
295+
# packages whose ServiceName field is empty in the metadata.
296+
children = @('**');
297+
}
298+
)
299+
}
300+
301+
$output = @([PSCustomObject]@{
302+
name = 'Reference';
303+
landingPageType = 'Root';
304+
expanded = $false;
305+
items = $toc
306+
})
307+
308+
if (Test-Path "Function:$UpdateDocsMsTocFn") {
309+
$output = &$UpdateDocsMsTocFn -toc $output
310+
}
311+
312+
$outputYaml = ConvertTo-Yaml $output
313+
Set-Content -Path $OutputLocation -Value $outputYaml

‎eng/common/scripts/common.ps1‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,8 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS
4545
$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
4646
$FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
4747
$TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"
48-
$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
48+
$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
49+
$GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages"
50+
$GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
51+
$GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages"
52+
$UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"

0 commit comments

Comments
 (0)