diff --git a/src/Elastic.Markdown/Slices/DescriptionGenerator.cs b/src/Elastic.Markdown/DescriptionGenerator.cs similarity index 98% rename from src/Elastic.Markdown/Slices/DescriptionGenerator.cs rename to src/Elastic.Markdown/DescriptionGenerator.cs index 701d618bf..30f74bee1 100644 --- a/src/Elastic.Markdown/Slices/DescriptionGenerator.cs +++ b/src/Elastic.Markdown/DescriptionGenerator.cs @@ -7,7 +7,7 @@ using Markdig.Syntax; using Markdig.Syntax.Inlines; -namespace Elastic.Markdown.Slices; +namespace Elastic.Markdown; public interface IDescriptionGenerator { diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index cfbe8354b..841bbb171 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -15,7 +15,6 @@ using Elastic.Markdown.Exporters; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; -using Elastic.Markdown.Slices; using Markdig.Syntax; using Microsoft.Extensions.Logging; @@ -174,6 +173,7 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => _logger.LogInformation("-> Processed {ProcessedFiles}/{TotalFileCount} files", processedFiles, totalFileCount); }); _logger.LogInformation("-> Processed {ProcessedFileCount}/{TotalFileCount} files", processedFileCount, totalFileCount); + } private void HintUnusedSubstitutionKeys() @@ -246,7 +246,14 @@ private async Task ProcessFile(HashSet offendingFiles, DocumentationFile foreach (var exporter in _markdownExporters) { var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx); - _ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx); + _ = await exporter.ExportAsync(new MarkdownExportFileContext + { + BuildContext = Context, + Resolvers = DocumentationSet.MarkdownParser.Resolvers, + Document = document, + SourceFile = markdown, + DefaultOutputFile = outputFile + }, ctx); } } } diff --git a/src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs b/src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs index e4cffec43..50c4815d1 100644 --- a/src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs +++ b/src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs @@ -5,7 +5,6 @@ using System.IO.Abstractions; using Elastic.Documentation.Configuration; using Elastic.Markdown.IO; -using Elastic.Markdown.Slices; using Markdig.Syntax; namespace Elastic.Markdown.Exporters; diff --git a/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs index b96704c64..6d004de91 100644 --- a/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs +++ b/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs @@ -2,15 +2,25 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Abstractions; +using Elastic.Documentation.Configuration; using Elastic.Markdown.IO; +using Elastic.Markdown.Myst; using Markdig.Syntax; namespace Elastic.Markdown.Exporters; -public class MarkdownExportContext + +public record MarkdownExportContext +{ +} +public record MarkdownExportFileContext { + public required BuildContext BuildContext { get; init; } + public required IParserResolvers Resolvers { get; init; } public required MarkdownDocument Document { get; init; } - public required MarkdownFile File { get; init; } + public required MarkdownFile SourceFile { get; init; } + public required IFileInfo DefaultOutputFile { get; init; } public string? LLMText { get; set; } } @@ -18,5 +28,6 @@ public interface IMarkdownExporter { ValueTask StartAsync(Cancel ctx = default); ValueTask StopAsync(Cancel ctx = default); - ValueTask ExportAsync(MarkdownExportContext context, Cancel ctx); + ValueTask ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx); + ValueTask FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx); } diff --git a/src/Elastic.Markdown/Exporters/LLMTextExporter.cs b/src/Elastic.Markdown/Exporters/LLMTextExporter.cs new file mode 100644 index 000000000..4a01cd3d0 --- /dev/null +++ b/src/Elastic.Markdown/Exporters/LLMTextExporter.cs @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Buffers; +using System.IO.Abstractions; +using System.IO.Compression; +using System.Text; +using Elastic.Documentation.Configuration; +using Elastic.Markdown.Helpers; +using Elastic.Markdown.Myst; +using Elastic.Markdown.Myst.FrontMatter; + +namespace Elastic.Markdown.Exporters; + +public class LLMTextExporter : IMarkdownExporter +{ + public ValueTask StartAsync(Cancel ctx = default) => ValueTask.CompletedTask; + + public ValueTask StopAsync(Cancel ctx = default) => ValueTask.CompletedTask; + + public async ValueTask ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx) + { + var source = fileContext.SourceFile.SourceFile; + var fs = source.FileSystem; + var llmText = fileContext.LLMText ??= ToLLMText(fileContext.BuildContext, fileContext.SourceFile.YamlFrontMatter, fileContext.Resolvers, source); + + // write to the output version of the Markdown file directly + var outputFile = fileContext.DefaultOutputFile; + if (outputFile.Name == "index.md") + { + var root = fileContext.BuildContext.DocumentationOutputDirectory; + // Write to a file named after the parent folder + if (outputFile.Directory!.FullName == root.FullName) + { + // TODO in FinishExportAsync find a way to generate llms.txt + // e.g should it embedd all the links? + outputFile = fs.FileInfo.New(Path.Combine(root.FullName, "llms.md")); + } + else + outputFile = fs.FileInfo.New(outputFile.Directory!.FullName + ".md"); + } + + if (outputFile.Directory is { Exists: false }) + outputFile.Directory.Create(); + + await fs.File.WriteAllTextAsync(outputFile.FullName, llmText, ctx); + return true; + } + + /// + public ValueTask FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx) + { + var outputDirectory = Path.Combine(outputFolder.FullName, "docs"); + var zipPath = Path.Combine(outputDirectory, "llm.zip"); + using (var zip = ZipFile.Open(zipPath, ZipArchiveMode.Create)) + { + var markdownFiles = Directory.GetFiles(outputDirectory, "*.md", SearchOption.AllDirectories); + + foreach (var file in markdownFiles) + { + var relativePath = Path.GetRelativePath(outputDirectory, file); + _ = zip.CreateEntryFromFile(file, relativePath); + } + } + return ValueTask.FromResult(true); + } + + public static string ToLLMText(BuildContext buildContext, YamlFrontMatter? frontMatter, IParserResolvers resolvers, IFileInfo source) + { + var fs = source.FileSystem; + var sb = DocumentationObjectPoolProvider.StringBuilderPool.Get(); + + Read(source, fs, sb, buildContext.DocumentationSourceDirectory); + var full = sb.ToString(); + var state = new ParserState(buildContext) + { + YamlFrontMatter = frontMatter, + MarkdownSourcePath = source, + CrossLinkResolver = resolvers.CrossLinkResolver, + DocumentationFileLookup = resolvers.DocumentationFileLookup + }; + DocumentationObjectPoolProvider.StringBuilderPool.Return(sb); + var replaced = full.ReplaceSubstitutions(new ParserContext(state)); + return replaced; + } + + private static void Read(IFileInfo source, IFileSystem fs, StringBuilder sb, IDirectoryInfo setDirectory) + { + var text = fs.File.ReadAllText(source.FullName).AsSpan(); + var spanStart = ":::{include}".AsSpan(); + var include = SearchValues.Create([spanStart.ToString(), $":::{Environment.NewLine}"], StringComparison.OrdinalIgnoreCase); + int i; + var startIndex = 0; + while ((i = text[startIndex..].IndexOfAny(include)) >= 0) + { + var cursor = startIndex + i; + var marker = text[cursor..]; + if (marker.StartsWith(spanStart)) + { + _ = sb.Append(text.Slice(startIndex, i).TrimEnd('\n')); + var relativeFileStart = marker.IndexOf('}') + 1; + var relativeFileEnd = marker.IndexOf('\n'); + var relativeFile = marker[relativeFileStart..relativeFileEnd].Trim(); + var includePath = Path.GetFullPath(Path.Combine(source.Directory!.FullName, relativeFile.ToString())); + var includeSource = fs.FileInfo.New(includePath); + if (relativeFile.StartsWith('/')) + { + includePath = Path.Combine(setDirectory.FullName, relativeFile.TrimStart('/').ToString()); + includeSource = fs.FileInfo.New(includePath); + } + + if (includeSource.Extension == "md" && includePath.Contains("_snippets")) + Read(includeSource, fs, sb, setDirectory); + startIndex = cursor + relativeFileEnd; + startIndex = Math.Min(text.Length, startIndex); + } + else + { + startIndex += i + 3 + Environment.NewLine.Length; + startIndex = Math.Min(text.Length, startIndex); + } + } + + _ = sb.Append(text[startIndex..]); + } +} diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs similarity index 98% rename from src/Elastic.Markdown/Slices/HtmlWriter.cs rename to src/Elastic.Markdown/HtmlWriter.cs index 5d4420d83..43eff502f 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -10,11 +10,12 @@ using Elastic.Documentation.Site.Navigation; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.IO; +using Elastic.Markdown.Page; using Markdig.Syntax; using RazorSlices; using IFileInfo = System.IO.Abstractions.IFileInfo; -namespace Elastic.Markdown.Slices; +namespace Elastic.Markdown; public class HtmlWriter( DocumentationSet documentationSet, @@ -99,7 +100,7 @@ private async Task RenderLayout(MarkdownFile markdown, MarkdownDocument if (PositionalNavigation.MarkdownNavigationLookup.TryGetValue("docs-content://versions.md", out var item)) allVersionsUrl = item.Url; - var slice = Index.Create(new IndexViewModel + var slice = Page.Index.Create(new IndexViewModel { SiteName = siteName, DocSetName = DocumentationSet.Name, diff --git a/src/Elastic.Markdown/IO/DocumentationFile.cs b/src/Elastic.Markdown/IO/DocumentationFile.cs index f1dcbaff1..e867af7d7 100644 --- a/src/Elastic.Markdown/IO/DocumentationFile.cs +++ b/src/Elastic.Markdown/IO/DocumentationFile.cs @@ -5,7 +5,6 @@ using Elastic.Documentation.Site; using Elastic.Markdown.Myst; using Elastic.Markdown.Myst.FrontMatter; -using Elastic.Markdown.Slices; namespace Elastic.Markdown.IO; diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs index e23f5d7d0..dae259a96 100644 --- a/src/Elastic.Markdown/IO/MarkdownFile.cs +++ b/src/Elastic.Markdown/IO/MarkdownFile.cs @@ -13,9 +13,9 @@ using Elastic.Markdown.Links.CrossLinks; using Elastic.Markdown.Myst; using Elastic.Markdown.Myst.Directives; +using Elastic.Markdown.Myst.Directives.Include; using Elastic.Markdown.Myst.FrontMatter; using Elastic.Markdown.Myst.InlineParsers; -using Elastic.Markdown.Slices; using Markdig; using Markdig.Extensions.Yaml; using Markdig.Renderers.Roundtrip; @@ -185,17 +185,6 @@ public async Task ParseFullAsync(Cancel ctx) return document; } - public static string ToLLMText(MarkdownDocument document) - { - using var sw = new StringWriter(); - var rr = new RoundtripRenderer(sw); - rr.Write(document); - var outputMarkdown = sw.ToString(); - - return outputMarkdown; - - } - private IReadOnlyDictionary GetSubstitutions() { var globalSubstitutions = _globalSubstitutions; diff --git a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs index c82d7eba7..31009a135 100644 --- a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs +++ b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using Elastic.Documentation; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.TableOfContents; @@ -12,84 +11,6 @@ namespace Elastic.Markdown.IO.Navigation; -[DebuggerDisplay("Current: {Model.RelativePath}")] -public record FileNavigationItem(MarkdownFile Model, DocumentationGroup Group, bool Hidden = false) : ILeafNavigationItem -{ - public INodeNavigationItem? Parent { get; set; } = Group; - public IRootNavigationItem NavigationRoot { get; } = Group.NavigationRoot; - public string Url => Model.Url; - public string NavigationTitle => Model.NavigationTitle; - public int NavigationIndex { get; set; } -} - -public class TableOfContentsTreeCollector -{ - private Dictionary NestedTableOfContentsTrees { get; } = []; - - public void Collect(Uri source, TableOfContentsTree tree) => - NestedTableOfContentsTrees[source] = tree; - - public void Collect(TocReference tocReference, TableOfContentsTree tree) => - NestedTableOfContentsTrees[tocReference.Source] = tree; - - public bool TryGetTableOfContentsTree(Uri source, [NotNullWhen(true)] out TableOfContentsTree? tree) => - NestedTableOfContentsTrees.TryGetValue(source, out tree); -} - - -[DebuggerDisplay("Toc >{Depth} {FolderName} {Source} ({NavigationItems.Count} items)")] -public class TableOfContentsTree : DocumentationGroup, IRootNavigationItem -{ - public Uri Source { get; } - - public TableOfContentsTreeCollector TreeCollector { get; } - - public TableOfContentsTree( - Uri source, - BuildContext context, - NavigationLookups lookups, - TableOfContentsTreeCollector treeCollector, - ref int fileIndex) - : base(".", treeCollector, context, lookups, source, ref fileIndex, 0, null, null) - { - TreeCollector = treeCollector; - NavigationRoot = this; - - Source = source; - TreeCollector.Collect(source, this); - - //edge case if a tree only holds a single group, ensure we collapse it down to the root (this) - if (NavigationItems.Count == 1 && NavigationItems.First() is DocumentationGroup { NavigationItems.Count: 0 }) - NavigationItems = []; - - - } - - internal TableOfContentsTree( - Uri source, - string folderName, - TableOfContentsTreeCollector treeCollector, - BuildContext context, - NavigationLookups lookups, - ref int fileIndex, - int depth, - IRootNavigationItem toplevelTree, - DocumentationGroup? parent - ) : base(folderName, treeCollector, context, lookups, source, ref fileIndex, depth, toplevelTree, parent) - { - Source = source; - TreeCollector = treeCollector; - NavigationRoot = this; - TreeCollector.Collect(source, this); - } - - protected override IRootNavigationItem DefaultNavigation => this; - - // We rely on IsPrimaryNavEnabled to determine if we should show the dropdown - /// - public bool IsUsingNavigationDropdown => false; -} - [DebuggerDisplay("Group >{Depth} {FolderName} ({NavigationItems.Count} items)")] public class DocumentationGroup : INodeNavigationItem { diff --git a/src/Elastic.Markdown/IO/Navigation/FileNavigationItem.cs b/src/Elastic.Markdown/IO/Navigation/FileNavigationItem.cs new file mode 100644 index 000000000..a37900b2e --- /dev/null +++ b/src/Elastic.Markdown/IO/Navigation/FileNavigationItem.cs @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics; +using Elastic.Documentation.Site.Navigation; + +namespace Elastic.Markdown.IO.Navigation; + +[DebuggerDisplay("Current: {Model.RelativePath}")] +public record FileNavigationItem(MarkdownFile Model, DocumentationGroup Group, bool Hidden = false) : ILeafNavigationItem +{ + public INodeNavigationItem? Parent { get; set; } = Group; + public IRootNavigationItem NavigationRoot { get; } = Group.NavigationRoot; + public string Url => Model.Url; + public string NavigationTitle => Model.NavigationTitle; + public int NavigationIndex { get; set; } +} diff --git a/src/Elastic.Markdown/IO/Navigation/TableOfContentsTree.cs b/src/Elastic.Markdown/IO/Navigation/TableOfContentsTree.cs new file mode 100644 index 000000000..f2e92e695 --- /dev/null +++ b/src/Elastic.Markdown/IO/Navigation/TableOfContentsTree.cs @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Site.Navigation; + +namespace Elastic.Markdown.IO.Navigation; + +[DebuggerDisplay("Toc >{Depth} {FolderName} ({NavigationItems.Count} items)")] +public class TableOfContentsTree : DocumentationGroup, IRootNavigationItem +{ + public Uri Source { get; } + + public TableOfContentsTreeCollector TreeCollector { get; } + + public TableOfContentsTree( + Uri source, + BuildContext context, + NavigationLookups lookups, + TableOfContentsTreeCollector treeCollector, + ref int fileIndex) + : base(".", treeCollector, context, lookups, source, ref fileIndex, 0, null, null) + { + TreeCollector = treeCollector; + NavigationRoot = this; + + Source = source; + TreeCollector.Collect(source, this); + + //edge case if a tree only holds a single group, ensure we collapse it down to the root (this) + if (NavigationItems.Count == 1 && NavigationItems.First() is DocumentationGroup { NavigationItems.Count: 0 }) + NavigationItems = []; + + + } + + internal TableOfContentsTree( + Uri source, + string folderName, + TableOfContentsTreeCollector treeCollector, + BuildContext context, + NavigationLookups lookups, + ref int fileIndex, + int depth, + IRootNavigationItem toplevelTree, + DocumentationGroup? parent + ) : base(folderName, treeCollector, context, lookups, source, ref fileIndex, depth, toplevelTree, parent) + { + Source = source; + TreeCollector = treeCollector; + NavigationRoot = this; + TreeCollector.Collect(source, this); + } + + protected override IRootNavigationItem DefaultNavigation => this; + + // We rely on IsPrimaryNavEnabled to determine if we should show the dropdown + /// + public bool IsUsingNavigationDropdown => false; +} diff --git a/src/Elastic.Markdown/IO/Navigation/TableOfContentsTreeCollector.cs b/src/Elastic.Markdown/IO/Navigation/TableOfContentsTreeCollector.cs new file mode 100644 index 000000000..47d8e8547 --- /dev/null +++ b/src/Elastic.Markdown/IO/Navigation/TableOfContentsTreeCollector.cs @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using Elastic.Documentation.Configuration.TableOfContents; + +namespace Elastic.Markdown.IO.Navigation; + +public class TableOfContentsTreeCollector +{ + private Dictionary NestedTableOfContentsTrees { get; } = []; + + public void Collect(Uri source, TableOfContentsTree tree) => + NestedTableOfContentsTrees[source] = tree; + + public void Collect(TocReference tocReference, TableOfContentsTree tree) => + NestedTableOfContentsTrees[tocReference.Source] = tree; + + public bool TryGetTableOfContentsTree(Uri source, [NotNullWhen(true)] out TableOfContentsTree? tree) => + NestedTableOfContentsTrees.TryGetValue(source, out tree); +} diff --git a/src/Elastic.Markdown/Slices/Layout/_Archive.cshtml b/src/Elastic.Markdown/Layout/_Archive.cshtml similarity index 99% rename from src/Elastic.Markdown/Slices/Layout/_Archive.cshtml rename to src/Elastic.Markdown/Layout/_Archive.cshtml index 06d26704c..789445fe3 100644 --- a/src/Elastic.Markdown/Slices/Layout/_Archive.cshtml +++ b/src/Elastic.Markdown/Layout/_Archive.cshtml @@ -1,4 +1,4 @@ -@inherits RazorSlice +@inherits RazorSlice

Archive

diff --git a/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml b/src/Elastic.Markdown/Layout/_Breadcrumbs.cshtml similarity index 96% rename from src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml rename to src/Elastic.Markdown/Layout/_Breadcrumbs.cshtml index e6c056ca8..9686dcc10 100644 --- a/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml +++ b/src/Elastic.Markdown/Layout/_Breadcrumbs.cshtml @@ -1,4 +1,4 @@ -@inherits RazorSlice +@inherits RazorSlice @{ var targets = Model.Features.PrimaryNavEnabled diff --git a/src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml b/src/Elastic.Markdown/Layout/_LandingPage.cshtml similarity index 99% rename from src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml rename to src/Elastic.Markdown/Layout/_LandingPage.cshtml index eaae5daa3..3dd5e1c07 100644 --- a/src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml +++ b/src/Elastic.Markdown/Layout/_LandingPage.cshtml @@ -1,4 +1,4 @@ -@inherits RazorSlice +@inherits RazorSlice
diff --git a/src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml b/src/Elastic.Markdown/Layout/_NotFound.cshtml similarity index 89% rename from src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml rename to src/Elastic.Markdown/Layout/_NotFound.cshtml index c8503bc9e..709005176 100644 --- a/src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml +++ b/src/Elastic.Markdown/Layout/_NotFound.cshtml @@ -1,4 +1,4 @@ -@inherits RazorSlice +@inherits RazorSlice
Page not found

The page you are looking for could not be found.

diff --git a/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml b/src/Elastic.Markdown/Layout/_PrevNextNav.cshtml similarity index 96% rename from src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml rename to src/Elastic.Markdown/Layout/_PrevNextNav.cshtml index 3be23cc97..0ee95f90a 100644 --- a/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml +++ b/src/Elastic.Markdown/Layout/_PrevNextNav.cshtml @@ -1,4 +1,4 @@ -@inherits RazorSlice +@inherits RazorSlice