Add docs-generator tool to generate random docsets

.github/check-license-headers.sh

@@ -42,4 +42,4 @@ if [[ $nErrors -eq 0 ]]; then
     exit 0
 else
     exit 1
-fi
+fi

build/Program.cs

@@ -25,6 +25,12 @@
 		dotnet publish {source} -c Release -o .artifacts/publish \
 			--self-contained true /p:PublishTrimmed=true /p:PublishSingleFile=false /p:PublishAot=true
 		""";
+
+	var generatorSource = "src/docs-generator/docs-generator.csproj";
+	await $"""
+		dotnet publish {generatorSource} -c Release -o .artifacts/publish \
+			--self-contained true /p:PublishTrimmed=true /p:PublishSingleFile=false /p:PublishAot=true
+		""";
 });
 
 // this is manual for now and quite hacky.

docs-builder.sln

@@ -30,6 +30,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{67B576EE
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Markdown.Tests", "tests\Elastic.Markdown.Tests\Elastic.Markdown.Tests.csproj", "{B27C5107-128B-465A-B8F8-8985399E4CFB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-generator", "src\docs-generator\docs-generator.csproj", "{61904527-9753-4379-B546-56B6A29073AC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -60,10 +62,15 @@ Global
 		{B27C5107-128B-465A-B8F8-8985399E4CFB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B27C5107-128B-465A-B8F8-8985399E4CFB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B27C5107-128B-465A-B8F8-8985399E4CFB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{61904527-9753-4379-B546-56B6A29073AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{61904527-9753-4379-B546-56B6A29073AC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{61904527-9753-4379-B546-56B6A29073AC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{61904527-9753-4379-B546-56B6A29073AC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{01F05AD0-E0E0-401F-A7EC-905928E1E9F0} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{B27C5107-128B-465A-B8F8-8985399E4CFB} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
+		{61904527-9753-4379-B546-56B6A29073AC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 	EndGlobalSection
 EndGlobal

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -85,7 +85,7 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 		}
 
 		var contents = document
-			.Where(block => block is HeadingBlock { Level: 2 })
+			.Where(block => block is HeadingBlock { Level: >= 2 })
 			.Cast<HeadingBlock>()
 			.Select(h => h.Inline?.FirstChild?.ToString())
 			.Where(title => !string.IsNullOrWhiteSpace(title))

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -68,7 +68,7 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 			includeFrom = context.Parser.SourcePath.FullName;
 
 		var anchors = url.Split('#');
-		var anchor = anchors.Length > 1 ? anchors[1] : null;
+		var anchor = anchors.Length > 1 ? anchors[1].Trim() : null;
 		url = anchors[0];
 
 		if (!string.IsNullOrWhiteSpace(url))
@@ -87,7 +87,7 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 		if (link.FirstChild == null || !string.IsNullOrEmpty(anchor))
 		{
 			var file = string.IsNullOrWhiteSpace(url) ? context.Path
-				: context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url));
+				: context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url.TrimStart('/')));
 			var markdown = context.GetMarkdownFile?.Invoke(file);
 			var title = markdown?.Title;
 

src/docs-builder/Http/DocumentationWebHost.cs

@@ -26,8 +26,7 @@ public class DocumentationWebHost
 	public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fileSystem)
 	{
 		var builder = WebApplication.CreateSlimBuilder();
-		var sourcePath = path != null ? fileSystem.DirectoryInfo.New(path) : null;
-		var context = new BuildContext(fileSystem)
+		var context = new BuildContext(fileSystem, fileSystem, path, null)
 		{
 			Collector = new ConsoleDiagnosticsCollector(logger)
 		};
@@ -36,7 +35,7 @@ public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fil
 			s.FolderToMonitor = context.SourcePath.FullName;
 			s.ClientFileExtensions = ".md,.yml";
 		});
-		builder.Services.AddSingleton<ReloadableGeneratorState>(_ => new ReloadableGeneratorState(sourcePath, null, context, logger));
+		builder.Services.AddSingleton<ReloadableGeneratorState>(_ => new ReloadableGeneratorState(context.SourcePath, null, context, logger));
 		builder.Services.AddHostedService<ReloadGeneratorService>();
 		builder.Services.AddSingleton(logger);
 		builder.Logging.SetMinimumLevel(LogLevel.Warning);

src/docs-generator/Domain/Determinism.cs

@@ -0,0 +1,34 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Bogus;
+
+namespace Documentation.Generator.Domain;
+
+public record Determinism
+{
+	public Determinism(int? seedFileSystem, int? seedContent)
+	{
+		var randomizer = new Randomizer();
+		SeedFileSystem = seedFileSystem ?? randomizer.Int(1, int.MaxValue);
+		SeedContent = seedContent ?? randomizer.Int(1, int.MaxValue);
+		FileSystem = new Randomizer(SeedFileSystem);
+		Contents = new Randomizer(SeedContent);
+
+		SectionProbability = Contents.Float(0.001f, Contents.Float(0.1f));
+		FileProbability = FileSystem.Float(0.001f, Contents.Float(0.1f));
+	}
+
+	public int SeedFileSystem { get; }
+	public int SeedContent { get; }
+
+
+	public Randomizer FileSystem { get; }
+	public Randomizer Contents { get; }
+
+	public float SectionProbability { get; }
+	public float FileProbability { get;  }
+
+	public static Determinism Random { get; set; } = new(null, null);
+}

src/docs-generator/Domain/Generators.cs

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Slugify;
+using Soenneker.Utils.AutoBogus;
+
+namespace Documentation.Generator.Domain;
+
+public static class Generators
+{
+	public static AutoFaker<FolderName> FolderName { get; } = new();
+	public static AutoFaker<Section> Section { get; } = new();
+	public static AutoFaker<MarkdownFile> File { get; } = new();
+	public static SlugHelper Slug { get; } = new();
+
+	static Generators()
+	{
+		FolderName
+			.RuleFor(p => p.Folder, f => f.Lorem.Slug(1));
+
+		Section
+			.RuleFor(p => p.Paragraphs, f => f.Lorem.Paragraphs(f.Random.Number(1, 10)))
+			.RuleFor(p => p.Level, f => f.Random.Number(2, 4));
+
+		File
+			.Ignore(p => p.Links)
+			.Ignore(p => p.Directory)
+			.RuleFor(p => p.FileName, f => f.System.FileName("md"))
+			.RuleFor(p => p.IncludeInUpdate, f => f.Random.Float() <= Determinism.Random.FileProbability)
+			.RuleFor(p => p.Sections, f => Section.Generate(Determinism.Random.Contents.Number(1, 12)).ToArray());
+	}
+
+	public static IEnumerable<string> CreateSubPaths(string parent, int maxDepth, int currentDepth)
+	{
+		yield return parent;
+		if (currentDepth == maxDepth) yield break;
+		var subFolders = FolderName.Generate(Determinism.Random.FileSystem.Number(0, 4));
+		foreach (var subFolder in subFolders)
+		{
+			var path = $"{parent}/{subFolder.Folder}";
+			yield return path;
+			var subPaths = CreateSubPaths(path, maxDepth, currentDepth + 1);
+			foreach (var p in subPaths)
+				yield return p;
+		}
+	}
+
+	public static string[] FolderNames { get; set; } = [];
+}

src/docs-generator/Domain/MarkdownFile.cs

@@ -0,0 +1,49 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Slugify;
+using Soenneker.Utils.AutoBogus;
+
+namespace Documentation.Generator.Domain;
+
+public record MarkdownFile
+{
+	public required string FileName { get; init; }
+	public required Section[] Sections { get; init; }
+	public required string Title { get; init; }
+
+	public string RelativePath => $"{Directory}/{FileName}";
+
+	public required bool IncludeInUpdate { get; init; } = true;
+
+	public string Directory { get; set; } = string.Empty;
+	public List<string> Links { get; set; } = [];
+
+	public void RewriteLinksIntoSections()
+	{
+		var linksLength = Links.Count;
+		var sectionsLength = Sections.Length;
+		for (var i = 0; i < linksLength; i++)
+		{
+			var link = Links[i];
+			var section = Sections[Determinism.Random.Contents.Number(0, sectionsLength - 1)];
+			var words = section.Paragraphs.Split(" ");
+			var w = Determinism.Random.Contents.Number(0, words.Length - 1);
+			var word = words[w];
+			words[w] = $"[{word}](/{link})";
+			section.Paragraphs = string.Join(" ", words);
+		}
+	}
+
+
+	public string GetRandomLink()
+	{
+		var sectionLink = Determinism.Random.Contents.Bool(0.8f);
+		if (!sectionLink) return RelativePath;
+		var section = Sections[Determinism.Random.Contents.Number(0, Sections.Length - 1)];
+		return $"{RelativePath}#{Generators.Slug.GenerateSlug(section.Header)}";
+
+	}
+}
+

src/docs-generator/Domain/Path.cs

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Documentation.Generator.Domain;
+
+public static class Paths
+{
+	private static DirectoryInfo RootDirectoryInfo()
+	{
+		var directory = new DirectoryInfo(Directory.GetCurrentDirectory());
+		while (directory != null &&
+		       (directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0))
+			directory = directory.Parent;
+		return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory());
+	}
+
+	public static readonly DirectoryInfo Root = RootDirectoryInfo();
+}
+
+public record FolderName
+{
+	public required string Folder { get; init; }
+}
+
+public record Folder
+{
+	public required string Path { get; init; }
+	public required MarkdownFile[] Files { get; init; }
+}

src/docs-generator/Domain/Section.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Documentation.Generator.Domain;
+
+public record Section
+{
+	public required string Header { get; init; }
+
+	public required int Level { get; init; }
+
+	public required string Paragraphs { get; set; }
+
+}