From 79d2998fec87a7c649112510d6079179359660a1 Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Tue, 8 Sep 2026 13:33:48 +0000 Subject: [PATCH 01/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index f07ca867e..ae03c8325 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4:nightly-2026-09-06 +leanprover/lean4-pr-releases:pr-release-15064-95b0bf8 From 36682c1ce3b6a196dd679618c3b5b60d371c2500 Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Wed, 9 Sep 2026 10:35:46 +0000 Subject: [PATCH 02/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index ae03c8325..a8fcba0d6 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4-pr-releases:pr-release-15064-95b0bf8 +leanprover/lean4-pr-releases:pr-release-15064-62f3bbf From cae278bf16ccbc5a3cda193dd43a332881a6ce61 Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 12:57:58 +0200 Subject: [PATCH 03/15] fix: verso against the Verso docstring parser rewrite Verso borrowed four parsers from `Lean.Doc.Parser` that the rewrite replaces. Take `asTokenFn` in place of `asStringFn`, and Verso's own `inline` and `metadataBlock` in place of Lean's, so that Verso's parser produces its own syntax throughout. Lean's parser now reads a Verso docstring into `Lean.Doc.Parser.Block` syntax, so the language server's folding ranges accept those kinds alongside the ones Verso's parser produces. `textLine` moves from `Verso.SyntaxUtils` to `Verso.Parser`, where `inline` is in scope. --- verso/src/verso-manual/VersoManual/Linters.lean | 2 +- verso/src/verso/Verso/Doc/ArgParse.lean | 2 +- verso/src/verso/Verso/Doc/Lsp.lean | 3 +++ verso/src/verso/Verso/Output/Html/AttributeName.lean | 2 +- verso/src/verso/Verso/Parser.lean | 5 +++++ verso/src/verso/Verso/SyntaxUtils.lean | 4 +--- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/verso/src/verso-manual/VersoManual/Linters.lean b/verso/src/verso-manual/VersoManual/Linters.lean index 2d1a69c6d..2faa7538e 100644 --- a/verso/src/verso-manual/VersoManual/Linters.lean +++ b/verso/src/verso-manual/VersoManual/Linters.lean @@ -67,7 +67,7 @@ meta partial def headerTagLinter : Linter where } let toks := Parser.getTokenTable (← getEnv) let s := { cache := { tokenCache := {}, parserCache := {} }, pos := nextLine.i } - let s := Lean.Doc.Parser.metadataBlock.run ictx pmctx toks s + let s := Verso.Parser.metadataBlock.run ictx pmctx toks s let tagNote := MessageData.note <| "The tag is used as a permanent name for the section or chapter. Writers "++ diff --git a/verso/src/verso/Verso/Doc/ArgParse.lean b/verso/src/verso/Verso/Doc/ArgParse.lean index 650003a89..bcf9730a3 100644 --- a/verso/src/verso/Verso/Doc/ArgParse.lean +++ b/verso/src/verso/Verso/Doc/ArgParse.lean @@ -6,7 +6,7 @@ Author: David Thrane Christiansen module public import Verso.Doc public import Verso.Hover -import Verso.Parser +public import Verso.Parser public import Verso.SyntaxUtils import Lean.Parser import Lean.Elab.GuardMsgs diff --git a/verso/src/verso/Verso/Doc/Lsp.lean b/verso/src/verso/Verso/Doc/Lsp.lean index 42ed13daa..3d1a4bcfb 100644 --- a/verso/src/verso/Verso/Doc/Lsp.lean +++ b/verso/src/verso/Verso/Doc/Lsp.lean @@ -880,6 +880,9 @@ where children.flatMap (getFromSyntax text) ++ here | _ => #[] isFoldable : Name → Bool + | `Lean.Doc.Parser.Block.codeblock | `Lean.Doc.Parser.Block.directive + | `Lean.Doc.Parser.Block.metadata_block | `Lean.Doc.Parser.Block.blockquote + | `Lean.Doc.Parser.Block.ol | `Lean.Doc.Parser.Block.ul | `Lean.Doc.Parser.Block.dl => true | `Lean.Doc.Syntax.codeblock | `Lean.Doc.Syntax.directive | `Lean.Doc.Syntax.metadata_block | `Lean.Doc.Syntax.blockquote | `Lean.Doc.Syntax.ol | `Lean.Doc.Syntax.ul | `Lean.Doc.Syntax.dl => true | `Verso.Syntax.codeblock | `Verso.Syntax.directive | `Verso.Syntax.metadata_block | `Verso.Syntax.blockquote diff --git a/verso/src/verso/Verso/Output/Html/AttributeName.lean b/verso/src/verso/Verso/Output/Html/AttributeName.lean index b5dcd36ba..728fdd885 100644 --- a/verso/src/verso/Verso/Output/Html/AttributeName.lean +++ b/verso/src/verso/Verso/Output/Html/AttributeName.lean @@ -19,7 +19,7 @@ open Lean.Doc.Parser public def attributeNameFn : ParserFn := atomicFn <| nodeFn attributeNameKind <| - asStringFn <| andthenFn (satisfyFn versoAttributeNameChar) (manyFn attributeNameCharFn) + asTokenFn <| andthenFn (satisfyFn versoAttributeNameChar) (manyFn attributeNameCharFn) where -- A slight divergence from the spec for the sake of quasiquotation syntax: -- attribute names can't start with a few special characters that the spec allows but that diff --git a/verso/src/verso/Verso/Parser.lean b/verso/src/verso/Verso/Parser.lean index 8db99208b..a5676ca0d 100644 --- a/verso/src/verso/Verso/Parser.lean +++ b/verso/src/verso/Verso/Parser.lean @@ -667,6 +667,11 @@ mutual text <|> linebreak ctxt <|> delimitedInline ctxt end +/-- +One or more inline elements. With `allowNewlines`, they may continue onto the following lines. +-/ +def textLine (allowNewlines := true) : ParserFn := many1Fn (inline { allowNewlines }) + open Lean.Parser Term in def metadataContents : Parser := structInstFields (sepByIndent structInstField ", " (allowTrailingSep := true)) diff --git a/verso/src/verso/Verso/SyntaxUtils.lean b/verso/src/verso/Verso/SyntaxUtils.lean index 9e6305d47..e77da035e 100644 --- a/verso/src/verso/Verso/SyntaxUtils.lean +++ b/verso/src/verso/Verso/SyntaxUtils.lean @@ -16,15 +16,13 @@ namespace Verso.Parser open Lean Doc Parser -public def textLine (allowNewlines := true) : ParserFn := many1Fn (inline { allowNewlines }) - public def nl := satisfyFn (· == '\n') "newline" /-- Parses a line that contains only spaces. -/ public def blankLine : ParserFn := - nodeFn `blankLine <| atomicFn <| asStringFn <| takeWhileFn (· == ' ') >> nl + nodeFn `blankLine <| atomicFn <| asTokenFn <| takeWhileFn (· == ' ') >> nl private def skipToNewline : ParserFn := takeUntilFn (· == '\n') From b7f612fd0f2f33f5fa0acdcc7a886b64b6b9c707 Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 15:02:41 +0200 Subject: [PATCH 04/15] refactor: verso reads the parser's syntax through views Verso carried its own copy of the docstring parser, producing the same syntax under the same node kinds. It now calls `Lean.Doc.Parser`, which takes `Verso/Parser.lean` from 1003 lines to 343; what stays is the generic combinator prelude that Verso and the reference manual use for lexers of their own. The elaboration interfaces take views rather than syntax. `InlineExpander`, `BlockExpander` and `PartCommand` receive an `InlineView` or `BlockView`, and `elabInline`, `elabBlock` and `partCommand` build it once and pass it to each expander. The registry stays keyed on syntax kind, so a package can still replace an element's expander. Views carry a production's delimiters and arguments with their positions, which is what a code action needs to write a replacement. `VersoLiteral` abstracts the kinds whose tokens denote a string: `decode` reads the text and `encode` writes a token at a given position. String literals and Verso's seven content tokens instantiate it, so a parser or a diff takes one argument again rather than a location and a string. Verso's linters read the syntax a document is made of, so the typography and header-tag linters move to views along with everything else that matched on it. The parser reports two mistakes that its predecessor recovered from silently. A `#doc` in `Tests/VersoBlog.lean` ran to the end of the file and swallowed two `#docs`, four `#guard`s, and a test appended later; those documents get delimiters, the missing `#guard_msgs` returns, and the deprecation test uses the role it asserts a warning for. `#.` opened a section that rendered as a paragraph. Folding ranges now cover the lists inside a document body, so two expectations gain entries. --- verso/doc/UsersGuide/Markup.lean | 246 +++--- .../tests/Tests/DocElabExtensions/Define.lean | 2 +- .../src/tests/Tests/ExtensionResolution.lean | 4 +- verso/src/tests/Tests/VersoBlog.lean | 19 +- .../folding_verso.lean.expected.out | 4 + .../folding_verso_term_doc.lean.expected.out | 2 + verso/src/verso-blog/VersoBlog.lean | 38 +- .../src/verso-illuminate/VersoIlluminate.lean | 8 +- .../verso-manual/VersoManual/Diagrams.lean | 2 +- .../verso-manual/VersoManual/Docstring.lean | 14 +- .../src/verso-manual/VersoManual/Imports.lean | 2 +- .../verso-manual/VersoManual/InlineLean.lean | 53 +- .../VersoManual/InlineLean/IO.lean | 46 +- .../VersoManual/InlineLean/IO/Context.lean | 13 +- .../VersoManual/InlineLean/LongLines.lean | 6 +- .../VersoManual/InlineLean/Option.lean | 4 +- .../VersoManual/InlineLean/Signature.lean | 2 +- .../VersoManual/InlineLean/SyntaxError.lean | 2 +- .../src/verso-manual/VersoManual/Linters.lean | 29 +- .../verso-manual/VersoManual/Literate.lean | 7 +- verso/src/verso-manual/VersoManual/Table.lean | 15 +- verso/src/verso/Verso/Code/External.lean | 63 +- verso/src/verso/Verso/Doc/ArgParse.lean | 2 +- verso/src/verso/Verso/Doc/Elab.lean | 321 ++++---- verso/src/verso/Verso/Doc/Elab/Basic.lean | 2 +- verso/src/verso/Verso/Doc/Elab/Block.lean | 18 +- verso/src/verso/Verso/Doc/Elab/Inline.lean | 7 +- verso/src/verso/Verso/Doc/Elab/Monad.lean | 135 ++-- verso/src/verso/Verso/Doc/Helpers.lean | 24 +- verso/src/verso/Verso/Doc/Lsp.lean | 217 +++--- verso/src/verso/Verso/ExpectString.lean | 9 +- verso/src/verso/Verso/Linters.lean | 31 +- verso/src/verso/Verso/Literal.lean | 60 ++ verso/src/verso/Verso/Parser.lean | 717 +----------------- verso/src/verso/Verso/SyntaxUtils.lean | 16 +- .../textbook/DemoTextbook/Meta/Lean.lean | 6 +- .../tutorial-test/TutorialExample/Data.lean | 2 +- .../test-projects/website/DemoSite/About.lean | 31 +- 38 files changed, 828 insertions(+), 1351 deletions(-) create mode 100644 verso/src/verso/Verso/Literal.lean diff --git a/verso/doc/UsersGuide/Markup.lean b/verso/doc/UsersGuide/Markup.lean index af6ab13f6..46ff01a5f 100644 --- a/verso/doc/UsersGuide/Markup.lean +++ b/verso/doc/UsersGuide/Markup.lean @@ -16,16 +16,14 @@ open Lean.Doc.Syntax variable [Monad m] [MonadError m] [MonadQuotation m] -def newlinesToSpace (inls : Array (TSyntax `inline)) : m (Array (TSyntax `inline)) := do +def newlinesToSpace (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : + m (TSyntaxArray ``Lean.Doc.Parser.inline) := do let mut out := #[] for h : i in [:inls.size] do - match inls[i] with - | inl@`(inline|line!$s) => - if i < inls.size - 1 then - out := out.push (← `(inline| " ")) - else - out := out.push inl - | inl => out := out.push inl + let inl := inls[i] + if (Lean.Doc.LinebreakView.of inl).isSome && i < inls.size - 1 then + out := out.push (← `(Lean.Doc.Parser.inline| $(← Lean.Doc.mkVersoTextFromRef " "):versoText)) + else out := out.push inl return out def log10 (n : Nat) : Nat := @@ -40,94 +38,123 @@ def asCode (s : String) : String := (lw - s.length).fold (init := s) fun _ _ => (" " ++ ·) (lines.mapIdx fun i l => (s!"{toString (i + 1) |> pad}|{l}⏎\n")) |> String.join |>.trimAsciiEnd |>.copy -partial def preview (stx : Syntax) : m Std.Format := - match stx with - | `(inline| $s:str) => pure <| Format.joinSep (s.getString.splitOn " ") .line - | `(inline| line! $_s:str) => pure .line - | `(inline| _[ $s:inline* ]) => do - let s ← newlinesToSpace s - let contents ← s.toList.mapM (preview ∘ TSyntax.raw) - pure <| .group <| .nest 2 ("" ++ .line ++ .fill (.join contents)) ++ .line ++ "" - | `(inline| *[ $s:inline* ]) => do - let s ← newlinesToSpace s - let contents ← s.toList.mapM (preview ∘ TSyntax.raw) - pure <| .fill <| "" ++ .join contents ++ "" - | `(inline| role{$x $args*}[$inls*]) => do - let inls ← newlinesToSpace inls - let args ← args.toList.mapM (preview ·.raw) - let contents ← inls.toList.mapM (preview ∘ TSyntax.raw) - pure <| .fill <| .group (.nest 2 (s!"<{x.getId.toString} {Format.joinSep args " " |>.pretty}>" ++ .line ++ .fill (Format.joinSep contents .line)) ++ .line ++ s!"") - | `(inline| \math code($s)) => - pure <| s!"" - | `(inline| \displaymath code($s)) => - pure <| s!"" - | `(inline| link[ $txt:inline* ]( $url:str )) => do - let txt ← newlinesToSpace txt - let contents ← txt.toList.mapM (preview ∘ TSyntax.raw) - pure <| .group <| .nest 2 (s!"" ++ .line ++ .fill (.join contents)) ++ .line ++ "" - | `(inline| link[ $txt:inline* ][ $tgt:str ]) => do - let txt ← newlinesToSpace txt - let contents ← txt.toList.mapM (preview ∘ TSyntax.raw) - pure <| .fill <| s!"" ++ .join contents ++ "" - | `(inline| image($s)($tgt:str)) => do - pure <| .group <| .nest 2 <| "" - | `(inline| image($s)[$tgt:str]) => do - pure <| .group <| .nest 2 <| "" - | `(inline| code( $code:str )) => - pure s!"{code.getString.quote}" - | `(block| para[ $i:inline* ]) => do - let contents ← i.toList.mapM (preview ∘ TSyntax.raw) - pure <| .group <| .nest 2 ("

" ++ .line ++ .fill (.join contents)) ++ .line ++ "

" - | `(block| > $bs*) => do - let contents ← bs.toList.mapM (preview ∘ TSyntax.raw) - pure <| .group <| .nest 2 ("
" ++ .line ++ .fill (Format.joinSep contents .line)) ++ .line ++ "
" - | `(block| [ $ref:str ]: $url:str) => - pure <| .group <| .nest 2 <| "where" ++ .line ++ .group (.nest 2 (s!"«{ref.getString}» :=" ++ .line ++ url.getString)) - | `(block| header($n){$inls*}) => do - let title ← Format.join <$> inls.toList.mapM (preview ∘ TSyntax.raw) - pure <| s!"" ++ title.fill ++ s!"" ++ Format.line - | `(block| ``` | $s ```) => - pure <| .nest 2 (s!"" ++ .line ++ asCode s.getString ++ .text "") ++ .line ++ s!"" - | `(block| ```$x $args* | $s ```) => do - let args ← args.toList.mapM (preview ·.raw) - pure <| .nest 2 (s!"<{x.getId.toString} {Std.Format.prefixJoin " " args |>.pretty}>" ++ .line ++ asCode s.getString ++ .nil) ++ .line ++ s!"" - | `(block| ::: $x $args* {$body*}) => do - let args ← args.toList.mapM (preview ·.raw) - let body ← body.toList.mapM (preview ·.raw) - pure <| .group <| .nest 2 (s!"<{x.getId.toString} {Std.Format.prefixJoin " " args |>.pretty}>" ++ .line ++ Format.joinSep body .line) ++ .line ++ s!"" - | `(block| command{$x $args*}) => do - let args ← args.toList.mapM (preview ·.raw) - pure s!"<{x.getId.toString} {Std.Format.prefixJoin " " args |>.pretty}/>" - | `(doc_arg|($x:ident := $v)) | `(doc_arg|$x:ident := $v) => do - pure <| s!"{x.getId.toString}=\"{← preview v.raw}\"" - | `(doc_arg|$v:arg_val) => preview v.raw - | `(arg_val|$v:ident) => pure s!"{v.getId}" - | `(arg_val|$v:num) => pure s!"{v.getNat}" - | `(arg_val|$v:str) => pure s!"{v.getString.quote}" - | `(block| ul{$lis*}) => do - let items ← lis.toList.mapM (preview ·.raw) - pure <| .group <| .nest 2 ("
    " ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" - | `(block| ol($_n){$lis*}) => do - let items ← lis.toList.mapM (preview ·.raw) - pure <| .group <| .nest 2 ("
    " ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" - | `(block| dl{$descs*}) => do - let items ← descs.toList.mapM fun x => (preview x.raw) <&> (" " ++ · ++ "\n") - pure <| .group <| .nest 2 <| ("
" ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" - | `(desc| : $dt* => $dd* ) => do - let dt ← dt.toList.mapM (preview ·.raw) - let dd ← dd.toList.mapM (preview ·.raw) +open Lean.Doc + +mutual + partial def previewInline (v : InlineView) : m Std.Format := do + match v with + | .text t => pure <| Format.joinSep (t.getVersoText.splitOn " ") .line + | .linebreak _ => pure .line + | .emph e => do + let s ← newlinesToSpace e.content + let contents ← s.toList.mapM (preview ∘ TSyntax.raw) + pure <| .group <| .nest 2 ("" ++ .line ++ .fill (.join contents)) ++ .line ++ "" + | .bold b => do + let s ← newlinesToSpace b.content + let contents ← s.toList.mapM (preview ∘ TSyntax.raw) + pure <| .fill <| "" ++ .join contents ++ "" + | .role r => do + let inls ← newlinesToSpace r.content + let args ← r.args.toList.mapM (preview ·.raw) + let contents ← inls.toList.mapM (preview ∘ TSyntax.raw) + let name := r.name.getId.toString + pure <| .fill <| .group (.nest 2 (s!"<{name} {Format.joinSep args " " |>.pretty}>" ++ .line ++ .fill (Format.joinSep contents .line)) ++ .line ++ s!"") + | .math mv => + let tag := match mv.mode with | .inline => "math" | .display => "displaymath" + pure <| s!"<{tag} contents={mv.getVersoCode.quote}/>" + | .link l => do + let txt ← newlinesToSpace l.content + let contents ← txt.toList.mapM (preview ∘ TSyntax.raw) + match l.target with + | .url _ _ url _ => + pure <| .group <| .nest 2 (s!"" ++ .line ++ .fill (.join contents)) ++ .line ++ "" + | .ref _ _ name _ => + pure <| .fill <| s!"" ++ .join contents ++ "" + | .image i => + let src := match i.target with + | .url _ _ url _ => url.getVersoLinkUrl + | .ref _ _ name _ => s!"value of «{name.getVersoRefName}»" + pure <| .group <| .nest 2 <| "" + | .code c => pure s!"{c.getVersoCode.quote}" + | .footnote f => pure s!"" + + partial def previewBlock (v : BlockView) : m Std.Format := do + match v with + | .para p => do + let contents ← p.content.toList.mapM (preview ∘ TSyntax.raw) + pure <| .group <| .nest 2 ("

" ++ .line ++ .fill (.join contents)) ++ .line ++ "

" + | .blockquote b => do + let contents ← b.content.toList.mapM (preview ∘ TSyntax.raw) + pure <| .group <| .nest 2 ("
" ++ .line ++ .fill (Format.joinSep contents .line)) ++ .line ++ "
" + | .linkRef r => + pure <| .group <| .nest 2 <| "where" ++ .line ++ .group (.nest 2 (s!"«{r.getName}» :=" ++ .line ++ r.getUrl)) + | .footnoteRef r => do + let contents ← r.content.toList.mapM (preview ∘ TSyntax.raw) + pure <| .group <| .nest 2 (s!"" ++ .line ++ .fill (.join contents)) ++ .line ++ "" + | .header h => do + let title ← Format.join <$> h.content.toList.mapM (preview ∘ TSyntax.raw) + pure <| s!"" ++ title.fill ++ s!"" ++ Format.line + | .codeblock c => do + match c.name? with + | none => + pure <| .nest 2 (s!"" ++ .line ++ asCode c.getVersoCodeBlock ++ .text "") ++ .line ++ s!"" + | some x => + let args ← c.args.toList.mapM (preview ·.raw) + pure <| .nest 2 (s!"<{x.getId.toString} {Std.Format.prefixJoin " " args |>.pretty}>" ++ .line ++ asCode c.getVersoCodeBlock ++ .nil) ++ .line ++ s!"" + | .directive d => do + let args ← d.args.toList.mapM (preview ·.raw) + let body ← d.content.toList.mapM (preview ·.raw) + let name := d.name.getId.toString + pure <| .group <| .nest 2 (s!"<{name} {Std.Format.prefixJoin " " args |>.pretty}>" ++ .line ++ Format.joinSep body .line) ++ .line ++ s!"" + | .command c => do + let args ← c.args.toList.mapM (preview ·.raw) + pure s!"<{c.name.getId.toString} {Std.Format.prefixJoin " " args |>.pretty}/>" + | .ul l => do + let items ← l.items.toList.mapM (previewListItem ·.contents) + pure <| .group <| .nest 2 ("
    " ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" + | .ol l => do + let items ← l.items.toList.mapM (previewListItem ·.contents) + pure <| .group <| .nest 2 ("
    " ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" + | .dl l => do + let items ← l.items.toList.mapM fun d => previewDescItem d <&> (" " ++ · ++ "\n") + pure <| .group <| .nest 2 <| ("
" ++ .line ++ Format.joinSep items .line) ++ .line ++ "
" + | .metadata _ => pure "" + + partial def previewListItem (contents : TSyntaxArray ``Lean.Doc.Parser.block) : m Std.Format := do + let content ← contents.toList.mapM (preview ∘ TSyntax.raw) + pure <| .group <| .nest 2 ("
  • " ++ .line ++ .join content) ++ .line ++ "
  • " + + partial def previewDescItem (d : DescItemView) : m Std.Format := do + let dt ← d.term.toList.mapM (preview ·.raw) + let dd ← d.desc.toList.mapM (preview ·.raw) pure <| .group (.nest 2 ("
    " ++ .line ++ .join dt) ++ .line ++ "
    ") ++ .line ++ .group (.nest 2 ("
    " ++ .line ++ .join dd) ++ .line ++ "
    ") - | `(li| * $content*) => do - let content ← content.toList.mapM (preview ∘ TSyntax.raw) - pure <| .group <| .nest 2 ("
  • " ++ .line ++ .join content) ++ .line ++ "
  • " - | other => do - if other.getKind = nullKind then - pure <| .joinSep (← other.getArgs.toList.mapM preview) (.line ++ .line) + + partial def previewArg (a : ArgView) : m Std.Format := do + match a with + | .anon _ v => preview v.raw + | .named _ _ x _ v => pure <| s!"{x.getId.toString}=\"{← preview v.raw}\"" + | .flag _ _ x on => pure <| s!"{if on then "+" else "-"}{x.getId.toString}" + + partial def preview (stx : Syntax) : m Std.Format := do + if let some v := InlineView.of ⟨stx⟩ then previewInline v + else if let some v := BlockView.of ⟨stx⟩ then previewBlock v + else if let some v := ArgView.of ⟨stx⟩ then previewArg v + else if let some v := ArgValView.of ⟨stx⟩ then + match v with + | .name x => pure s!"{x.getId}" + | .num _ n => pure s!"{n}" + | .str _ s => pure s!"{s.quote}" + else if let some v := UnorderedListItemView.of ⟨stx⟩ then previewListItem v.contents + else if let some v := OrderedListItemView.of ⟨stx⟩ then previewListItem v.contents + else if let some v := DescItemView.of ⟨stx⟩ then previewDescItem v + else if stx.getKind = nullKind then + pure <| .joinSep (← stx.getArgs.toList.mapM preview) (.line ++ .line) else throwErrorAt stx "Didn't understand {Verso.SyntaxUtils.ppSyntax stx} for preview" end +end block_extension MarkupExample (title : String) where data := title @@ -286,26 +313,27 @@ def markupPreview : DirectiveExpanderOf MarkupPreviewConfig | {title}, contents => do let #[blk1, blk2] := contents.filter nonempty | throwError "Expected precisely two code blocks, got {contents.filter nonempty}" - let `(block|``` | $contents ```) := blk1 - | throwErrorAt blk1 "Expected anonymous code block" - let `(block|``` | $expected ```) := blk2 + let some cb1 := Lean.Doc.CodeBlockView.of blk1 | throwErrorAt blk1 "Expected anonymous code block" + let some cb2 := Lean.Doc.CodeBlockView.of blk2 + | throwErrorAt blk2 "Expected anonymous code block" + let (contents, expected) := (cb1.content, cb2.content) - let stx ← blocks {} |>.parseString contents.getString.trimAsciiEnd.copy + let stx ← blocks {} |>.parseString contents.getVersoCodeBlock.trimAsciiEnd.copy let p ← preview stx let p := p.pretty (width := 35) withOptions (verso.code.warnLineLength.set · 35) do - warnLongLines none contents - warnLongLines none expected + warnLongLines none contents.raw + warnLongLines none expected.raw - unless eq expected.getString p do + unless eq expected.getVersoCodeBlock p do let hint ← MessageData.hint m!"Replace with actual output" #[withNl p] (ref? := expected) - throwErrorAt expected m!"Expected {indentD expected.getString} but got {indentD p}\n{hint}" + throwErrorAt expected m!"Expected {indentD expected.getVersoCodeBlock} but got {indentD p}\n{hint}" Hover.addCustomHover contents s!"```\n{p}\n```" ``(Block.other (MarkupExample $(quote title.getString)) #[ - Block.code $(quote contents.getString), + Block.code $(quote contents.getVersoCodeBlock), Block.code $(quote <| toString <| p) ]) where @@ -314,12 +342,16 @@ where let lines2 := s2.trimAscii.split (· == '\n') |>.map (·.trimAsciiEnd) |>.toArray lines1 == lines2 - nonemptyI : TSyntax `inline → Bool - | `(inline|$s:str) | `(inline|line!$s) => !s.getString.isEmpty - | _ => true - nonempty : TSyntax `block → Bool - | `(block|para[$inls*]) => inls.any nonemptyI - | _ => true + -- A paragraph of only whitespace separates the two code blocks that the directive expects. + nonemptyI (inl : TSyntax ``Lean.Doc.Parser.inline) : Bool := + match Lean.Doc.InlineView.of inl with + | some (.text t) => !t.getVersoText.isEmpty + | some (.linebreak _) => false + | _ => true + nonempty (blk : TSyntax ``Lean.Doc.Parser.block) : Bool := + match Lean.Doc.ParaView.of blk with + | some p => p.content.any nonemptyI + | none => true open Lean Verso Doc Elab in open Verso.Parser in @@ -327,11 +359,11 @@ open Verso.Parser in def markupPreviewPre : CodeBlockExpanderOf MarkupPreviewConfig | {title}, contents => do - let stx ← blocks {} |>.parseString contents.getString + let stx ← blocks {} |>.parseString contents.getVersoCodeBlock let p ← preview stx let p := p.pretty (width := 35) - let directive := s!":::markupPreview {title.getString.quote}\n```\n{withNl contents.getString}```\n```\n{withNl p}```\n:::" + let directive := s!":::markupPreview {title.getString.quote}\n```\n{withNl contents.getVersoCodeBlock}```\n```\n{withNl p}```\n:::" let hint ← MessageData.hint m!"Replace with directive:" #[directive] throwError m!"Expected a directive.{hint}" diff --git a/verso/src/tests/Tests/DocElabExtensions/Define.lean b/verso/src/tests/Tests/DocElabExtensions/Define.lean index 689f9725c..da7f8fc1c 100644 --- a/verso/src/tests/Tests/DocElabExtensions/Define.lean +++ b/verso/src/tests/Tests/DocElabExtensions/Define.lean @@ -32,7 +32,7 @@ meta def inheritedRole : RoleExpanderOf Unit @[code_block] meta def inheritedCode : CodeBlockExpanderOf Unit - | (), str => ``(Doc.Block.code $(quote str.getString)) + | (), str => ``(Doc.Block.code $(quote str.getVersoCodeBlock)) @[directive] meta def inheritedDirective : DirectiveExpanderOf Unit diff --git a/verso/src/tests/Tests/ExtensionResolution.lean b/verso/src/tests/Tests/ExtensionResolution.lean index d3ed01b2a..a25c8d1b9 100644 --- a/verso/src/tests/Tests/ExtensionResolution.lean +++ b/verso/src/tests/Tests/ExtensionResolution.lean @@ -354,7 +354,7 @@ namespace CodeBlockCases @[code_block] meta def registeredBlock : CodeBlockExpanderOf Unit | (), str => do - `(Verso.Doc.Block.code $(quote str.getString)) + `(Verso.Doc.Block.code $(quote str.getVersoCodeBlock)) #docs (.none) codeBlockRegistered "Registered code block" := ::::::: @@ -365,7 +365,7 @@ content def unregisteredBlock : CodeBlockExpanderOf Unit | (), str => do - `(Verso.Doc.Block.code $(quote str.getString)) + `(Verso.Doc.Block.code $(quote str.getVersoCodeBlock)) /-- error: Declaration `unregisteredBlock` can be used as a code block expander but is not registered as a code block. Register it with `@[code_block]`. diff --git a/verso/src/tests/Tests/VersoBlog.lean b/verso/src/tests/Tests/VersoBlog.lean index c29e1a020..9d5c58e79 100644 --- a/verso/src/tests/Tests/VersoBlog.lean +++ b/verso/src/tests/Tests/VersoBlog.lean @@ -76,7 +76,8 @@ def runBlogTests : IO Nat := do end -- Regression test for hidden blog Lean blocks. -#doc (Post) "Hidden Lean Block Flags" => +#docs (Post) hiddenLeanBlockFlags "Hidden Lean Block Flags" := +::::::: ```leanInit post ``` @@ -95,27 +96,38 @@ example : base = 40 := rfl ```lean post +error #check scratch ``` +::::::: + +#guard hiddenLeanBlockFlags.toPart.content.size > 0 -- Regression test for inline Lean role naming in Blog: -- canonical `{lean}` works without warnings. #docs (Post) inlineLeanRoleNames "Inline Lean Role Names" := +::::::: ```leanInit post ``` Canonical role: {lean post}`Nat.succ 1`. +::::::: /-- warning: `{leanInline}` is deprecated; use `{lean}` instead. -/ +#guard_msgs in #docs (Post) inlineLeanRoleNamesDeprecated "Inline Lean Role Names (deprecated alias)" := +::::::: ```leanInit post2 ``` -Legacy role: {lean post2}`Nat.succ 1`. +Legacy role: {leanInline post2}`Nat.succ 1`. +::::::: #guard inlineLeanRoleNames.toPart.content.size > 0 #guard inlineLeanRoleNamesDeprecated.toPart.content.size > 0 +-- Regression test for a role that reads the environment of its Lean block. +#docs (Post) roleSeesEnvironment "Role Sees Environment" := +::::::: ```leanInit env ``` @@ -124,3 +136,6 @@ def x := 1 ``` Role is aware of the environment: {lean env}`x` +::::::: + +#guard roleSeesEnvironment.toPart.content.size > 0 diff --git a/verso/src/tests/interactive/test-cases/folding_verso.lean.expected.out b/verso/src/tests/interactive/test-cases/folding_verso.lean.expected.out index fec0018e9..bbd2cf8c6 100644 --- a/verso/src/tests/interactive/test-cases/folding_verso.lean.expected.out +++ b/verso/src/tests/interactive/test-cases/folding_verso.lean.expected.out @@ -11,7 +11,10 @@ {"startLine": 51, "endLine": 63}, {"startLine": 54, "endLine": 63}, {"startLine": 11, "endLine": 13}, + {"startLine": 25, "endLine": 26}, {"startLine": 28, "endLine": 30}, + {"startLine": 41, "endLine": 42}, + {"startLine": 57, "endLine": 58}, {"startLine": 61, "endLine": 63}, {"startLine": 25, "endLine": 26}, {"startLine": 25, "endLine": 26}, @@ -24,4 +27,5 @@ {"startLine": 14, "kind": "region", "endLine": 15}, {"startLine": 17, "kind": "region", "endLine": 31}, {"startLine": 33, "kind": "region", "endLine": 43}, + {"startLine": 57, "kind": "region", "endLine": 58}, {"startLine": 61, "kind": "region", "endLine": 63}] diff --git a/verso/src/tests/interactive/test-cases/folding_verso_term_doc.lean.expected.out b/verso/src/tests/interactive/test-cases/folding_verso_term_doc.lean.expected.out index 84b5381cf..0470e0338 100644 --- a/verso/src/tests/interactive/test-cases/folding_verso_term_doc.lean.expected.out +++ b/verso/src/tests/interactive/test-cases/folding_verso_term_doc.lean.expected.out @@ -9,6 +9,8 @@ {"startLine": 29, "endLine": 38}, {"startLine": 33, "endLine": 38}, {"startLine": 19, "endLine": 20}, + {"startLine": 37, "endLine": 39}, + {"startLine": 19, "endLine": 20}, {"startLine": 19, "endLine": 20}, {"startLine": 37, "endLine": 39}, {"startLine": 37, "endLine": 39}, diff --git a/verso/src/verso-blog/VersoBlog.lean b/verso/src/verso-blog/VersoBlog.lean index 3faebe463..06dd1b0e1 100644 --- a/verso/src/verso-blog/VersoBlog.lean +++ b/verso/src/verso-blog/VersoBlog.lean @@ -373,10 +373,10 @@ meta instance : FromArgs NoArgs m where @[role] meta def leanKw : RoleExpanderOf NoArgs | ⟨⟩, #[arg] => do - let `(inline|code( $kw:str )) := arg + let some ⟨_, _, kw, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the keyword" - let hl : SubVerso.Highlighting.Highlighted := .token ⟨.keyword none none none, kw.getString⟩ - ``(Inline.other (Blog.InlineExt.customHighlight $(quote hl)) #[Inline.code $(quote kw.getString)]) + let hl : SubVerso.Highlighting.Highlighted := .token ⟨.keyword none none none, kw.getVersoCode⟩ + ``(Inline.other (Blog.InlineExt.customHighlight $(quote hl)) #[Inline.code $(quote kw.getVersoCode)]) | _, more => if h : more.size > 0 then throwErrorAt more[0] "Unexpected contents" @@ -396,9 +396,9 @@ meta instance : FromArgs LeanTermArgs DocElabM where @[role] meta def leanTerm : RoleExpanderOf LeanTermArgs | {project, showProofStates}, #[arg] => withTraceNode `Elab.Verso.block.lean (fun _ => pure m!"leanTerm") <| do - let `(inline|code( $name:str )) := arg + let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" - let exampleName := name.getString.toName + let exampleName := name.getVersoCode.toName let projectExamples ← getSubproject project let (_, {highlighted := hls, original := str, ..}) ← projectExamples.getOrSuggest <| mkIdentFrom name exampleName Verso.Hover.addCustomHover arg s!"```lean\n{str}\n```" @@ -467,7 +467,7 @@ meta def leanInit : CodeBlockExpanderOf LeanInitBlockConfig let commandState := { commandState with scopes := [{ header := "", opts := pp.tagAppFns.set {} true }] } modifyEnv <| fun env => exampleContextExt.modifyState env fun s => {s with contexts := s.contexts.insert config.exampleContext.getId (.inline commandState state)} if config.show then - ``(Block.code $(quote str.getString)) -- TODO highlighting hack + ``(Block.code $(quote str.getVersoCodeBlock)) -- TODO highlighting hack else ``(Block.concat #[]) where @@ -531,7 +531,7 @@ meta def lean : CodeBlockExpanderOf LeanBlockConfig setInfoState infoSt setEnv env if config.show then - `(Block.other (Blog.BlockExt.highlightedCode { contextName := $(quote x.getId), showProofStates := $(quote config.showProofStates) } $(quote hls)) #[Block.code $(quote str.getString)]) + `(Block.other (Blog.BlockExt.highlightedCode { contextName := $(quote x.getId), showProofStates := $(quote config.showProofStates) } $(quote hls)) #[Block.code $(quote str.getVersoCodeBlock)]) else ``(Block.concat #[]) @@ -611,7 +611,7 @@ private meta def leanInlineImpl : RoleExpanderOf LeanInlineConfig | config, elts => withTraceNode `Elab.Verso.block.lean (fun _ => pure m!"lean block") <| do let #[code] := elts | throwError "Expected precisely one code element" - let `(inline|code( $str:str )) := code + let some ⟨_, _, str, _⟩ := Lean.Doc.CodeView.of code | throwErrorAt code "Expected an inline code element" let x := config.exampleContext let (commandState, _) ← match exampleContextExt.getState (← getEnv) |>.contexts.find? x.getId with @@ -686,7 +686,7 @@ private meta def leanInlineImpl : RoleExpanderOf LeanInlineConfig isSilent := msg.isSilent || msg.severity != .error } - `(Inline.other (Blog.InlineExt.highlightedCode { contextName := $(quote config.exampleContext.getId) } $(quote hls)) #[Inline.code $(quote str.getString)]) + `(Inline.other (Blog.InlineExt.highlightedCode { contextName := $(quote config.exampleContext.getId) } $(quote hls)) #[Inline.code $(quote str.getVersoCode)]) @[role lean] meta def leanCanonical : RoleExpanderOf LeanInlineConfig := @@ -747,12 +747,12 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig | .inl (env, log) => let messages ← liftM <| log.toArray.mapM contents for m in log.toArray do - if mostlyEqual config.whitespace str.getString (← contents m) then + if mostlyEqual config.whitespace str.getVersoCodeBlock (← contents m) then if let some s := config.severity then if s != m.severity then throwErrorAt str s!"Expected severity {sevStr s}, but got {sevStr m.severity}" let content ← if config.summarize then - let lines := str.getString.splitOn "\n" + let lines := str.getVersoCodeBlock.splitOn "\n" let pre := lines.take 3 let post := String.join (lines.drop 3 |>.intersperse "\n") let preHtml : Html := pre.map (fun (l : String) => {{{{l}}}}) @@ -765,30 +765,30 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig withOptions (·.set `pp.tagAppFns true) do SubVerso.Highlighting.highlightMessage m finally setEnv myEnv - ``(Block.other (Blog.BlockExt.message false $(quote m') ([] : List Lean.Name)) #[Block.code $(quote str.getString)]) + ``(Block.other (Blog.BlockExt.message false $(quote m') ([] : List Lean.Name)) #[Block.code $(quote str.getVersoCodeBlock)]) return content pure messages | .inr msgs => let messages := msgs.toArray.map Prod.snd for (sev, txt) in msgs do - if mostlyEqual config.whitespace str.getString txt then + if mostlyEqual config.whitespace str.getVersoCodeBlock txt then if let some s := config.severity then if s != sev then throwErrorAt str s!"Expected severity {sevStr s}, but got {sevStr sev}" let content ← if config.summarize then - let lines := str.getString.splitOn "\n" + let lines := str.getVersoCodeBlock.splitOn "\n" let pre := lines.take 3 let post := String.join (lines.drop 3 |>.intersperse "\n") let preHtml : Html := pre.map (fun (l : String) => {{{{l}}}}) ``(Block.other (Blog.BlockExt.htmlDetails $(quote (sevStr sev)) $(quote preHtml)) #[Block.code $(quote post)]) else - ``(Block.other (Blog.BlockExt.htmlDiv $(quote (sevStr sev))) #[Block.code $(quote str.getString)]) + ``(Block.other (Blog.BlockExt.htmlDiv $(quote (sevStr sev))) #[Block.code $(quote str.getVersoCodeBlock)]) return content pure messages for m in messages do Verso.Doc.Suggestion.saveSuggestion str ((m.take 30).copy ++ "…") m - throwErrorAt str "Didn't match - expected one of: {indentD (toMessageData messages)}\nbut got:{indentD (toMessageData str.getString)}" + throwErrorAt str "Didn't match - expected one of: {indentD (toMessageData messages)}\nbut got:{indentD (toMessageData str.getVersoCodeBlock)}" where withNewline (str : String) := if str == "" || str.back != '\n' then str ++ "\n" else str @@ -810,14 +810,14 @@ elab "define_lexed_text" blockName:ident " ← " lexerName:ident : command => do elabCommand <| ← `(@[code_block] def $blockName : Doc.Elab.CodeBlockExpanderOf NoArgs | ⟨⟩, str => do - let out ← Verso.Genre.Blog.LexedText.highlight $(mkIdentFrom lexerName lexer) str.getString + let out ← Verso.Genre.Blog.LexedText.highlight $(mkIdentFrom lexerName lexer) str.getVersoCodeBlock ``(Block.other (Blog.BlockExt.lexedText $$(quote out)) #[])) elabCommand <| ← `(@[role] def $(mkIdent <| blockName.getId ++ `role) : Doc.Elab.RoleExpanderOf NoArgs | ⟨⟩, #[inl] => do - let `(inline|code($$str)) := inl + let some ⟨_, _, str, _⟩ := Lean.Doc.CodeView.of inl | throwErrorAt inl "Expected code" - let out ← Verso.Genre.Blog.LexedText.highlight $(mkIdentFrom lexerName lexer) str.getString + let out ← Verso.Genre.Blog.LexedText.highlight $(mkIdentFrom lexerName lexer) str.getVersoCode ``(Inline.other (Blog.InlineExt.lexedText $$(quote out)) #[]) | _, str => throwError "Expected no arguments and a single code element") diff --git a/verso/src/verso-illuminate/VersoIlluminate.lean b/verso/src/verso-illuminate/VersoIlluminate.lean index 55e874814..f339dc4b8 100644 --- a/verso/src/verso-illuminate/VersoIlluminate.lean +++ b/verso/src/verso-illuminate/VersoIlluminate.lean @@ -86,7 +86,7 @@ section open Lean Widget Elab Term Meta Illuminate -private meta unsafe def evalDiagramUnsafe (str : StrLit) (stx : Syntax) : +private meta unsafe def evalDiagramUnsafe (str : Syntax) (stx : Syntax) : TermElabM (String × Float) := do let diaTy ← Meta.mkAppM ``Diagram #[.const ``SVG []] let e ← Elab.Term.elabTerm stx (some diaTy) @@ -129,7 +129,7 @@ private meta unsafe def evalDiagramUnsafe (str : StrLit) (stx : Syntax) : pure (svgStr, diagramWidth) @[implemented_by evalDiagramUnsafe] -private opaque evalDiagramImpl (str : StrLit) (stx : Syntax) : +private opaque evalDiagramImpl (str : Syntax) (stx : Syntax) : TermElabM (String × Float) end @@ -145,10 +145,10 @@ section variables for the Manual genre). It defaults to the identity. Genre-specific code-block expanders call this to do the shared evaluation work and then emit their own `GenreDiagram.diagramBlock` term. -/ -public def elabAndStoreDiagram (str : StrLit) +public def elabAndStoreDiagram [Verso.VersoLiteral k] (str : TSyntax k) (scope : {α : Type} → TermElabM α → TermElabM α := fun act => act) : DocElabM (String × Float) := do let stx ← parseStrLitAsCategory `term str if stx.isMissing then return ("", 0) - scope (evalDiagramImpl str stx) + scope (evalDiagramImpl str.raw stx) diff --git a/verso/src/verso-manual/VersoManual/Diagrams.lean b/verso/src/verso-manual/VersoManual/Diagrams.lean index 9b6cdaa19..b1c42672d 100644 --- a/verso/src/verso-manual/VersoManual/Diagrams.lean +++ b/verso/src/verso-manual/VersoManual/Diagrams.lean @@ -163,4 +163,4 @@ meta def diagram : CodeBlockExpanderOf DiagramConfig ``(Verso.Doc.Block.other (Block.diagram $(quote svg) $(quote cssWidth) $(quote <| cfg.texWidth.getD "\\textwidth") $(quote cfg.inline)) - #[Verso.Doc.Block.code $(quote str.getString)]) + #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) diff --git a/verso/src/verso-manual/VersoManual/Docstring.lean b/verso/src/verso-manual/VersoManual/Docstring.lean index 153f6ab6b..ab4883e9a 100644 --- a/verso/src/verso-manual/VersoManual/Docstring.lean +++ b/verso/src/verso-manual/VersoManual/Docstring.lean @@ -1833,10 +1833,10 @@ meta def tacticInline : RoleExpanderOf TacticInlineOptions | {«show»}, inlines => do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $tac:str )) := arg + let some ⟨_, _, tac, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the tactic name" - let tacTok := tac.getString - let tacName := tac.getString.toName + let tacTok := tac.getVersoCode + let tacName := tac.getVersoCode.toName let some tacticDoc := (← getTactic? (.inl tacTok)) <|> (← getTactic? (.inr tacName)) | throwErrorAt tac "Didn't find tactic named {tac}" @@ -1990,14 +1990,14 @@ meta def convInline : RoleExpander | _args, inlines => do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $convTac:str )) := arg + let some ⟨_, _, convTac, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the conv tactic name" - let convTacName := convTac.getString.toName + let convTacName := convTac.getVersoCode.toName let convTacDoc ← getConvTactic (.inr (mkIdent convTacName)) none - let hl : Highlighted := convToken convTacDoc convTac.getString + let hl : Highlighted := convToken convTacDoc convTac.getVersoCode - return #[← `(Verso.Doc.Inline.other (Inline.conv $(quote hl)) #[Verso.Doc.Inline.code $(quote convTac.getString)])] + return #[← `(Verso.Doc.Inline.other (Inline.conv $(quote hl)) #[Verso.Doc.Inline.code $(quote convTac.getVersoCode)])] where convToken (t : ConvTacticDoc) (showStr : String) : Highlighted := .token ⟨.keyword (some t.name) none t.docs?, showStr⟩ diff --git a/verso/src/verso-manual/VersoManual/Imports.lean b/verso/src/verso-manual/VersoManual/Imports.lean index 1b9b45cd7..fa7038fda 100644 --- a/verso/src/verso-manual/VersoManual/Imports.lean +++ b/verso/src/verso-manual/VersoManual/Imports.lean @@ -39,6 +39,6 @@ meta def imports : CodeBlockExpanderOf ImportsParams let headerStx ← parseStrLitWith p str let hl ← highlight headerStx #[] {} if «show» then - ``(Block.other (Block.lean $(quote hl) {}) #[Block.code $(quote str.getString)]) + ``(Block.other (Block.lean $(quote hl) {}) #[Block.code $(quote str.getVersoCodeBlock)]) else ``(Block.empty) diff --git a/verso/src/verso-manual/VersoManual/InlineLean.lean b/verso/src/verso-manual/VersoManual/InlineLean.lean index a93054b37..ac1a5de73 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean.lean @@ -200,7 +200,8 @@ private meta def quoteHighlightViaSerialization (hls : Highlighted) : DocElabM T De-indents and returns (syntax of) a Block representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanBlock (shouldShow : Bool) (hls : Highlighted) (str: StrLit) : DocElabM Term := do +private meta def toHighlightedLeanBlock [Verso.VersoLiteral k] (shouldShow : Bool) + (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Block.concat #[]) @@ -209,21 +210,22 @@ private meta def toHighlightedLeanBlock (shouldShow : Bool) (hls : Highlighted) | .none => hls | .some col => hls.deIndent col - let range := Syntax.getRange? str + let range := str.raw.getRange? let range := range.map (← getFileMap).utf8RangeToLspRange ``(Block.other (Block.lean $(← quoteHighlightViaSerialization hls) (some $(quote (← getFileName))) $(quote range)) - #[Block.code $(quote str.getString)]) + #[Block.code $(quote (Verso.decode str))]) /-- Returns (syntax of) an Inline representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanInline (shouldShow : Bool) (hls : Highlighted) (str : StrLit) : DocElabM Term := do +private meta def toHighlightedLeanInline [Verso.VersoLiteral k] (shouldShow : Bool) + (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Inline.concat #[]) - ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote str.getString)]) + ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote (Verso.decode str))]) /-- @@ -246,13 +248,14 @@ private meta partial def disableUnusedVarLinterInInfoTree : InfoTree → InfoTre .node info (children.map disableUnusedVarLinterInInfoTree) | .hole id => .hole id -meta def elabCommands (config : LeanBlockConfig) (str : StrLit) - (toHighlightedLeanContent : (shouldShow : Bool) → (hls : Highlighted) → (str: StrLit) → DocElabM Term) +meta def elabCommands [Verso.VersoLiteral k] (config : LeanBlockConfig) (str : TSyntax k) + (toHighlightedLeanContent : + (shouldShow : Bool) → (hls : Highlighted) → (str : TSyntax k) → DocElabM Term) (minCommands : Option Nat := none) (maxCommands : Option Nat := none) : DocElabM Term := withoutAsync <| do - PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 str.getString)) + PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (Verso.decode str))) (kind := Lsp.SymbolKind.file) (detail? := some ("Lean code" ++ config.outlineMeta)) @@ -350,7 +353,7 @@ meta def elabCommands (config : LeanBlockConfig) (str : StrLit) reportMessages config.error str cmdState.messages if config.show then - warnLongLines col? str + warnLongLines col? str.raw where runCommand (act : Command.CommandElabM Unit) (stx : Syntax) (cctx : Command.Context) (cmdState : Command.State) : @@ -466,7 +469,7 @@ meta def leanInline : RoleExpanderOf LeanInlineConfig | config, inlines => withoutAsync do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $term:str )) := arg + let some ⟨_, _, term, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let leveller := @@ -556,7 +559,7 @@ meta def inst : RoleExpanderOf LeanBlockConfig | config, inlines => withoutAsync <| do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $term:str )) := arg + let some ⟨_, _, term, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let stx ← parseStrLitAsCategory `term term @@ -714,8 +717,8 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig let expected := if config.normalizeMetas then - normalizeMetavars str.getString - else str.getString + normalizeMetavars str.getVersoCodeBlock + else str.getVersoCodeBlock let mut texts : Array (Highlighted.Span.Kind × String) := #[] @@ -748,7 +751,7 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig if s != msg.severity.toSeverity then throwErrorAt str s!"Expected severity {sevStr s}, but got {sevStr msg.severity.toSeverity}" if config.show then - let content ← `(Block.other {Block.leanOutput with data := ToJson.toJson ($(quote msg), $(quote config.summarize), ($(quote config.expandTraces) : List Name))} #[Block.code $(quote str.getString)]) + let content ← `(Block.other {Block.leanOutput with data := ToJson.toJson ($(quote msg), $(quote config.summarize), ($(quote config.expandTraces) : List Name))} #[Block.code $(quote str.getVersoCodeBlock)]) return content else return (← ``(Block.concat #[])) else @@ -773,12 +776,12 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig Log.logSilentInfo m!"Diff is {d} lines:\n{d'}" if config.show then - let content ← `(Block.other {Block.leanOutput with data := ToJson.toJson ($(quote msg), $(quote config.summarize), ($(quote config.expandTraces) : List Name))} #[Block.code $(quote str.getString)]) + let content ← `(Block.other {Block.leanOutput with data := ToJson.toJson ($(quote msg), $(quote config.summarize), ($(quote config.expandTraces) : List Name))} #[Block.code $(quote str.getVersoCodeBlock)]) return content else return (← ``(Block.concat #[])) let suggs : Array (Nat × Meta.Hint.Suggestion) := texts.map fun (sev, msg) => - ((diffSize config.whitespace msg str.getString).1, { + ((diffSize config.whitespace msg str.getVersoCodeBlock).1, { suggestion := withNl msg, preInfo? := some s!"{sevStr sev.toSeverity}: " }) @@ -787,7 +790,7 @@ meta def leanOutput : CodeBlockExpanderOf LeanOutputConfig let hintMsg := if suggs.size > 1 then m!"Replace with one of the actual messages:" else m!"Replace with the actual message:" let hint ← hintAt str hintMsg suggs - throwErrorAt str (m!"Didn't match{if config.allowDiff > 0 then s!" even with allowDiff := {config.allowDiff}" else ""} - got: {indentD (toMessageData <| texts.map (Std.Format.text ·.2))}\nbut expected:{indentD (toMessageData str.getString)}" ++ hint) + throwErrorAt str (m!"Didn't match{if config.allowDiff > 0 then s!" even with allowDiff := {config.allowDiff}" else ""} - got: {indentD (toMessageData <| texts.map (Std.Format.text ·.2))}\nbut expected:{indentD (toMessageData str.getVersoCodeBlock)}" ++ hint) where sevStr : MessageSeverity → String | .error => "error" @@ -858,9 +861,9 @@ meta def constTok [Monad m] [MonadEnv m] [MonadLiftT MetaM m] [MonadLiftT IO m] @[role] meta def name : RoleExpanderOf NameConfig | cfg, #[arg] => do - let `(inline|code( $name:str )) := arg + let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" - let exampleName := name.getString.toName + let exampleName := name.getVersoCode.toName let identStx := mkIdentFrom arg (cfg.full.getD exampleName) (canonical := true) try @@ -869,12 +872,12 @@ meta def name : RoleExpanderOf NameConfig withInfoTreeContext (mkInfoTree := pure ∘ InfoTree.node (.ofCommandInfo {elaborator := `Manual.Meta.name, stx := identStx})) do realizeGlobalConstNoOverloadWithInfo identStx - let hl : Highlighted ← constTok resolvedName name.getString + let hl : Highlighted ← constTok resolvedName name.getVersoCode - `(Inline.other {Inline.name with data := ToJson.toJson $(quote hl)} #[Inline.code $(quote name.getString)]) + `(Inline.other {Inline.name with data := ToJson.toJson $(quote hl)} #[Inline.code $(quote name.getVersoCode)]) catch e => logErrorAt identStx e.toMessageData - ``(Inline.code $(quote name.getString)) + ``(Inline.code $(quote name.getVersoCode)) | _, more => if h : more.size > 0 then throwErrorAt more[0] "Unexpected contents" @@ -886,11 +889,11 @@ meta def name : RoleExpanderOf NameConfig @[role] meta def module : RoleExpanderOf Unit | (), #[arg] => do - let `(inline|code( $name:str )) := arg + let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the module's name" - let exampleName := name.getString.toName + let exampleName := name.getVersoCode.toName let identStx := mkIdentFrom arg exampleName (canonical := true) - ``(Inline.code $(quote name.getString)) + ``(Inline.code $(quote name.getVersoCode)) | _, more => if h : more.size > 0 then throwErrorAt more[0] "Expected code literal with the module's name" diff --git a/verso/src/verso-manual/VersoManual/InlineLean/IO.lean b/verso/src/verso-manual/VersoManual/InlineLean/IO.lean index 01116340d..b8c2dbd58 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/IO.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/IO.lean @@ -97,7 +97,7 @@ end @[code_block] meta def exampleFile : CodeBlockExpanderOf ExampleFileConfig | config, str => do - let s := str.getString + let s := str.getVersoCodeBlock if config.show then IOExample.exampleFileSyntax config.type s else @@ -300,7 +300,7 @@ meta def startExample [Monad m] [MonadEnv m] [MonadError m] [MonadQuotation m] [ modifyEnv fun env => ioExampleCtx.setState env (some {leanCodeName}) -meta def saveLeanCode (src : StrLit) : DocElabM Ident := do +meta def saveLeanCode (src : Lean.Doc.VersoCodeBlock) : DocElabM Ident := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't set Lean code - not in an IO example" | some st => @@ -311,19 +311,19 @@ meta def saveLeanCode (src : StrLit) : DocElabM Ident := do else throwError "Code already specified" -meta def saveInputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : StrLit) : m Unit := do +meta def saveInputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : Lean.Doc.VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save file - not in an IO example" | some st => modifyEnv fun env => ioExampleCtx.setState env (some {st with inputFiles := st.inputFiles.push (name, contents)}) -meta def saveOutputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : StrLit) : m Unit := do +meta def saveOutputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : Lean.Doc.VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save file - not in an IO example" | some st => modifyEnv fun env => ioExampleCtx.setState env (some {st with outputFiles := st.outputFiles.push (name, contents)}) -meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : m Unit := do +meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stdin - not in an IO example" | some st => @@ -331,7 +331,7 @@ meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : m | none => modifyEnv fun env => ioExampleCtx.setState env (some {st with stdin := some contents}) | some _ => throwError "stdin already specified" -meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : m Unit := do +meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stdout - not in an IO example" | some st => @@ -339,7 +339,7 @@ meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : | none => modifyEnv fun env => ioExampleCtx.setState env (some {st with stdout := some contents}) | some _ => throwError "stdout already specified" -meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : m Unit := do +meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stderr - not in an IO example" | some st => @@ -349,9 +349,9 @@ meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : StrLit) : meta def check - (leanCode : StrLit) (leanCodeName : Name) - (inputFiles outputFiles : Array (System.FilePath × StrLit)) - (stdin stdout stderr : Option StrLit) : DocElabM Highlighted := + (leanCode : Lean.Doc.VersoCodeBlock) (leanCodeName : Name) + (inputFiles outputFiles : Array (System.FilePath × Lean.Doc.VersoCodeBlock)) + (stdin stdout stderr : Option Lean.Doc.VersoCodeBlock) : DocElabM Highlighted := IO.FS.withTempDir fun dirname => do let toolchain : String ← IO.FS.readFile "lean-toolchain" let leanCodeName : String := @@ -370,7 +370,7 @@ meta def check -- Avoid contention during parallel builds let leanFileName : System.FilePath := (leanCodeName : System.FilePath).addExtension "lean" IO.FS.writeFile (dirname / "lean-toolchain") toolchain - IO.FS.writeFile (dirname / leanFileName) leanCode.getString + IO.FS.writeFile (dirname / leanFileName) leanCode.getVersoCodeBlock IO.FS.writeFile (dirname / "lakefile.toml") s!"name = \"example\" defaultTargets = [\"{leanCodeName}\"] @@ -379,7 +379,7 @@ meta def check name = \"{leanCodeName}\" " for (f, i) in inputFiles do - IO.FS.writeFile (dirname / f) i.getString + IO.FS.writeFile (dirname / f) i.getVersoCodeBlock let out ← IO.Process.output {cmd := "lake", args := #["clean"], cwd := some dirname} if out.exitCode != 0 then @@ -394,7 +394,7 @@ meta def check m!"Stderr:\n{out.stderr}\n\nStdout:\n{out.stdout}\n\n" let proc ← IO.Process.spawn {cmd := "lake", args := #["--quiet", "exe", leanCodeName], cwd := some dirname, stdin := .piped, stdout := .piped, stderr := .piped} let (stdinH, proc) ← proc.takeStdin - stdinH.putStr (stdin.map (·.getString) |>.getD "") + stdinH.putStr (stdin.map (·.getVersoCodeBlock) |>.getD "") stdinH.flush let stdoutTask ← IO.asTask proc.stdout.readToEnd Task.Priority.dedicated let stderrOut ← proc.stderr.readToEnd @@ -407,23 +407,23 @@ meta def check Lean.logError s!"Running 'lake --quiet exe {leanCodeName}' failed with exit code {exitCode}." let stdoutOut ← IO.ofExcept stdoutTask.get - let expectedStdout := stdout.map (·.getString) |>.getD "" + let expectedStdout := stdout.map (·.getVersoCodeBlock) |>.getD "" if stdoutOut.trimAscii != expectedStdout.trimAscii then if let some stdoutLit := stdout then Verso.Doc.Suggestion.saveSuggestion stdoutLit (shorten stdoutOut) stdoutOut logErrorAt (loc stdout) s!"Mismatched stdout. Expected:\n{expectedStdout}\nGot:\n{stdoutOut}" - let expectedStderr := stderr.map (·.getString) |>.getD "" + let expectedStderr := stderr.map (·.getVersoCodeBlock) |>.getD "" if stderrOut.trimAscii != expectedStderr.trimAscii then if let some stderrLit := stderr then Verso.Doc.Suggestion.saveSuggestion stderrLit (shorten stderrOut) stderrOut - logErrorAt (loc stderr) s!"Mismatched stderr. Expected:\n{stderr.map (·.getString) |>.getD ""}\nGot:{stderrOut}\n" + logErrorAt (loc stderr) s!"Mismatched stderr. Expected:\n{stderr.map (·.getVersoCodeBlock) |>.getD ""}\nGot:{stderrOut}\n" for (f, o) in outputFiles do let f' := dirname / f if ← f'.pathExists then let contents ← IO.FS.readFile f' - if contents.trimAscii != o.getString.trimAscii then + if contents.trimAscii != o.getVersoCodeBlock.trimAscii then Verso.Doc.Suggestion.saveSuggestion o (shorten contents) contents logErrorAt (loc (some o)) s!"Output file {f} mismatch. Got:\n{contents}" else Lean.logError s!"Output file {f} not found" @@ -491,7 +491,7 @@ meta def inputFile : CodeBlockExpanderOf FileConfig -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - exampleFileSyntax (.input opts.name) str.getString + exampleFileSyntax (.input opts.name) str.getVersoCodeBlock else ``(Block.concat #[]) @@ -503,7 +503,7 @@ meta def outputFile : CodeBlockExpanderOf FileConfig -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - exampleFileSyntax (.output opts.name) str.getString + exampleFileSyntax (.output opts.name) str.getVersoCodeBlock else ``(Block.concat #[]) @@ -515,7 +515,7 @@ meta def stdin : CodeBlockExpanderOf Config -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - exampleFileSyntax .stdin str.getString + exampleFileSyntax .stdin str.getVersoCodeBlock else ``(Block.concat #[]) @@ -527,7 +527,7 @@ meta def stdout : CodeBlockExpanderOf Config -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - exampleFileSyntax .stdout str.getString + exampleFileSyntax .stdout str.getVersoCodeBlock else ``(Block.concat #[]) @@ -539,7 +539,7 @@ meta def stderr : CodeBlockExpanderOf Config -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - exampleFileSyntax .stderr str.getString + exampleFileSyntax .stderr str.getVersoCodeBlock else ``(Block.concat #[]) @@ -552,7 +552,7 @@ meta def ioLean : CodeBlockExpanderOf Config if opts.show then let range := Syntax.getRange? str let range := range.map (← getFileMap).utf8RangeToLspRange - ``(Block.other (Block.lean $x (some $(quote (← getFileName))) $(quote range)) #[Block.code $(quote str.getString)]) + ``(Block.other (Block.lean $x (some $(quote (← getFileName))) $(quote range)) #[Block.code $(quote str.getVersoCodeBlock)]) else ``(Block.concat #[]) diff --git a/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean b/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean index 1b60ea85d..dac61b293 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean @@ -5,6 +5,7 @@ Author: David Thrane Christiansen -/ module +public import Lean.DocString.View public import Lean.Environment public section @@ -15,12 +16,12 @@ open Lean structure IOExampleContext where leanCodeName : Ident - code : Option StrLit := none - inputFiles : Array (System.FilePath × StrLit) := #[] - outputFiles : Array (System.FilePath × StrLit) := #[] - stdin : Option StrLit := none - stdout : Option StrLit := none - stderr : Option StrLit := none + code : Option Doc.VersoCodeBlock := none + inputFiles : Array (System.FilePath × Doc.VersoCodeBlock) := #[] + outputFiles : Array (System.FilePath × Doc.VersoCodeBlock) := #[] + stdin : Option Doc.VersoCodeBlock := none + stdout : Option Doc.VersoCodeBlock := none + stderr : Option Doc.VersoCodeBlock := none deriving Repr initialize ioExampleCtx : EnvExtension (Option IOExampleContext) ← diff --git a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean index 4f5342087..f11580af5 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean @@ -25,13 +25,13 @@ def getWarnLineLength [Monad m] [MonadOptions m] : m (Option Nat) := do let val := (← getOptions).get verso.code.warnLineLength.name verso.code.warnLineLength.defValue if val = 0 then return none else return some val -def warnLongLines [Monad m] [MonadFileMap m] [MonadLog m] [AddMessageContext m] [MonadOptions m] (indent? : Option Nat) (str : StrLit) : m Unit := do +def warnLongLines [Monad m] [MonadFileMap m] [MonadLog m] [AddMessageContext m] [MonadOptions m] (indent? : Option Nat) (str : Syntax) : m Unit := do let some maxCodeColumns ← getWarnLineLength | pure () let fileMap ← getFileMap let maxCol := maxCodeColumns + indent?.getD 0 - if let some startPos := str.raw.getPos? then - if let some stopPos := str.raw.getTailPos? then + if let some startPos := str.getPos? then + if let some stopPos := str.getTailPos? then let ⟨startLine, _⟩ := fileMap.toPosition startPos let ⟨stopLine, _⟩ := fileMap.toPosition stopPos for l in [startLine:stopLine] do diff --git a/verso/src/verso-manual/VersoManual/InlineLean/Option.lean b/verso/src/verso-manual/VersoManual/InlineLean/Option.lean index 069cd4d7b..92e1b16fc 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/Option.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/Option.lean @@ -29,9 +29,9 @@ meta def option : RoleExpanderOf Unit | (), inlines => withoutAsync do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $optName:str )) := arg + let some ⟨_, _, optName, _⟩ := Lean.Doc.CodeView.of arg | throwErrorAt arg "Expected code literal with the option name" - let optName := optName.getString.toName + let optName := optName.getVersoCode.toName let optDecl ← getOptionDecl optName let hl : Highlighted := optTok optName optDecl.declName optDecl.descr diff --git a/verso/src/verso-manual/VersoManual/InlineLean/Signature.lean b/verso/src/verso-manual/VersoManual/InlineLean/Signature.lean index 25bb4df3e..359e949c7 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/Signature.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/Signature.lean @@ -99,6 +99,6 @@ meta def signature : CodeBlockExpanderOf SignatureConfig else hls if «show» then - `(Block.other {Block.signature with data := ToJson.toJson $(quote hls)} #[Block.code $(quote str.getString)]) + `(Block.other {Block.signature with data := ToJson.toJson $(quote hls)} #[Block.code $(quote str.getVersoCodeBlock)]) else ``(Block.concat #[]) diff --git a/verso/src/verso-manual/VersoManual/InlineLean/SyntaxError.lean b/verso/src/verso-manual/VersoManual/InlineLean/SyntaxError.lean index 2c925b86c..41bf09f96 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/SyntaxError.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/SyntaxError.lean @@ -145,7 +145,7 @@ meta def syntaxError : CodeBlockExpanderOf SyntaxErrorConfig (kind := Lsp.SymbolKind.file) (detail? := some "Syntax error") - let s := str.getString + let s := str.getVersoCodeBlock match runParserCategory' (← getEnv) (← getOptions) config.category s with | .ok stx => throwErrorAt str m!"Expected a syntax error for category {config.category}, but got {indentD stx}" diff --git a/verso/src/verso-manual/VersoManual/Linters.lean b/verso/src/verso-manual/VersoManual/Linters.lean index 2faa7538e..020561027 100644 --- a/verso/src/verso-manual/VersoManual/Linters.lean +++ b/verso/src/verso-manual/VersoManual/Linters.lean @@ -48,7 +48,8 @@ meta partial def headerTagLinter : Linter where let text ← getFileMap discard <| stx.replaceM fun block => do - if let `(block|header($n){$inls*}) := block then + if let some hdr := Lean.Doc.HeaderView.of ⟨block⟩ then + let inls := hdr.content let some ⟨start, stop⟩ := block.getRange? | return none let mut nextLine : String.Legacy.Iterator := {s := text.source, i := stop} @@ -89,8 +90,10 @@ meta partial def headerTagLinter : Linter where if s.stxStack.size = 1 then pure (s.stxStack.get! 0) else return none - if let`(block|%%%%$tk1 $fieldOrAbbrev* %%%%$tk2) := nextStx then - let metadataStx ← `(term| { $fieldOrAbbrev* }) + if let some metaView := Lean.Doc.MetadataView.of ⟨nextStx⟩ then + let fieldOrAbbrev := metaView.fields + let metadataStx : Term := + ⟨(← `(Lean.Parser.Term.structInst| { $[$fieldOrAbbrev],* })).raw⟩ let isMissing ← runTermElabM fun _ => do let type := .const `Verso.Genre.Manual.PartMetadata [] let metadataTerm ← Term.elabTerm metadataStx (some type) @@ -102,9 +105,9 @@ meta partial def headerTagLinter : Linter where if isMissing && noFieldIsTag fieldOrAbbrev then let name := suggestId inls -- Find the beginning of the line after the token - let some ⟨start1, stop1⟩ := tk1.getRange? + let some ⟨start1, stop1⟩ := metaView.opener.getRange? | return none - let some ⟨start2, stop2⟩ := tk2.getRange? + let some ⟨start2, stop2⟩ := metaView.closer.getRange? | return none let blockStr := start.extract text.source stop let suggestions : Array Meta.Hint.Suggestion := #[ @@ -125,17 +128,19 @@ where | `(Lean.Parser.Term.structInstField|$x:ident := $_ ) => x.getId ≠ `tag | _ => true - suggestId (name : TSyntaxArray `inline) : String := + suggestId (name : TSyntaxArray ``Lean.Doc.Parser.inline) : String := suggestId' name |>.sluggify |>.toString - suggestId' (name : TSyntaxArray `inline) : String := Id.run do + suggestId' (name : TSyntaxArray ``Lean.Doc.Parser.inline) : String := Id.run do let mut strTitle := "" for inl in name do - match inl with - | `(inline|$s:str) => strTitle := strTitle ++ s.getString.toLower - | `(inline|code($s)) => strTitle := strTitle ++ s.getString - | `(inline|_[$i*]) | `(inline|*[$i*]) | `(inline|link[$i*]$_) | `(inline|role{$_ $_*}[$i*]) => - strTitle := strTitle ++ suggestId' i + match Lean.Doc.InlineView.of inl with + | some (.text s) => strTitle := strTitle ++ s.getVersoText.toLower + | some (.code c) => strTitle := strTitle ++ c.getVersoCode + | some (.emph e) => strTitle := strTitle ++ suggestId' e.content + | some (.bold b) => strTitle := strTitle ++ suggestId' b.content + | some (.link l) => strTitle := strTitle ++ suggestId' l.content + | some (.role r) => strTitle := strTitle ++ suggestId' r.content | _ => pure () return strTitle diff --git a/verso/src/verso-manual/VersoManual/Literate.lean b/verso/src/verso-manual/VersoManual/Literate.lean index 9fd8c6e0a..adebad0e1 100644 --- a/verso/src/verso-manual/VersoManual/Literate.lean +++ b/verso/src/verso-manual/VersoManual/Literate.lean @@ -155,10 +155,11 @@ meta instance : FromArgs IncludeLiterateConfig m where IncludeLiterateConfig.mk <$> .positional' `path <*> .named' `level true <*> .positional' `name <*> .positional' `title -@[part_command Lean.Doc.Syntax.command] +@[part_command Lean.Doc.Parser.Block.command] meta def includeLiterateSection : PartCommand - | `(block|command{includeLiterate $args* }) => do - let {path, level, modName, title} ← parseThe IncludeLiterateConfig (← parseArgs args) + | .command v => do + unless v.name.getId == `includeLiterate do Lean.Elab.throwUnsupportedSyntax + let {path, level, modName, title} ← parseThe IncludeLiterateConfig (← parseArgs v.args) let ref ← getRef if let some lvl := level then let name ← getModuleWithDocs path modName title none diff --git a/verso/src/verso-manual/VersoManual/Table.lean b/verso/src/verso-manual/VersoManual/Table.lean index 18f779df3..a36fe8910 100644 --- a/verso/src/verso-manual/VersoManual/Table.lean +++ b/verso/src/verso-manual/VersoManual/Table.lean @@ -194,15 +194,15 @@ meta def table : DirectiveExpanderOf TableConfig -- The table should be a list of lists. Extract them! let #[oneBlock] := contents | throwError "Expected a single unordered list" - let `(block|ul{$items*}) := oneBlock + let some outer := Lean.Doc.UnorderedListView.of oneBlock | throwErrorAt oneBlock "Expected a single unordered list" - let preRows ← items.mapM getLi + let preRows := outer.items.map (·.contents) let rows ← preRows.mapM fun blks => do - let #[oneInRow] := blks.filter (·.raw.isOfKind ``Lean.Doc.Syntax.ul) + let #[oneInRow] := blks.filter (·.raw.isOfKind ``Lean.Doc.Parser.Block.ul) | throwError "Each row should have exactly one list in it" - let `(block|ul{ $cellItems*}) := oneInRow + let some inner := Lean.Doc.UnorderedListView.of oneInRow | throwErrorAt oneInRow "Each row should have exactly one list in it" - cellItems.mapM getLi + pure (inner.items.map (·.contents)) if h : rows.size = 0 then throwErrorAt oneBlock "Expected at least one row" else @@ -217,7 +217,4 @@ meta def table : DirectiveExpanderOf TableConfig let blocks : Array (Syntax.TSepArray `term ",") ← flattened.mapM (·.mapM elabBlock) ``(Block.other (Block.table $(quote columns) $(quote cfg.header) $(quote cfg.name) $(quote cfg.alignment)) #[Block.ul #[$[Verso.Doc.ListItem.mk #[$blocks,*]],*]]) -where - getLi : Syntax → DocElabM (TSyntaxArray `block) - | `(list_item| * $content* ) => pure content - | other => throwErrorAt other "Expected list item" + diff --git a/verso/src/verso/Verso/Code/External.lean b/verso/src/verso/Verso/Code/External.lean index 29046b264..37f0f7305 100644 --- a/verso/src/verso/Verso/Code/External.lean +++ b/verso/src/verso/Verso/Code/External.lean @@ -193,12 +193,13 @@ where if k > n then n := k n.fold (fun _ _ s => s.push '`') "" -meta def moduleContentBlock (args : Array Arg) (code : StrLit) : DocElabM (Array Term) := do +meta def moduleContentBlock (args : Array Arg) (code : Lean.Doc.VersoCodeBlock) : + DocElabM (Array Term) := do let cfg@{ module := moduleName, project, anchor?, showProofStates := _, defSite := _ } ← parseThe CodeContext args withAnchored project moduleName anchor? fun hl => do logInfos hl let hlString := hl.toString - if code.getString.trimAscii.isEmpty && !hlString.trimAscii.isEmpty then + if code.getVersoCodeBlock.trimAscii.isEmpty && !hlString.trimAscii.isEmpty then let ref ← getRef let h ← if let some s ← editCodeBlock ref hlString then @@ -238,7 +239,7 @@ public meta def anchor : CodeBlockExpander else throwError "Expected a positional argument first (the anchor name)" -meta def moduleInline (args : Array Arg) (inls : TSyntaxArray `inline) : DocElabM (Array Term) := do +meta def moduleInline (args : Array Arg) (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : DocElabM (Array Term) := do let cfg@{module := moduleName, project, anchor?, showProofStates := _, defSite := _} ← parseThe CodeContext args let code? ← oneCodeStr? inls @@ -278,10 +279,10 @@ where mkHover (sig : String) (doc? : Option String) : String := s!"\n\n----------\n\n{d}" else "" -public meta def moduleNameInline (args : Array Arg) (inls : TSyntaxArray `inline) : DocElabM (Array Term) := do +public meta def moduleNameInline (args : Array Arg) (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : DocElabM (Array Term) := do let cfg@{module := moduleName, project, anchor?, show?, showProofStates := _, defSite := _} ← parseThe NameContext args let name ← oneCodeStr inls - let nameStr := name.getString + let nameStr := name.getVersoCode withAnchored project moduleName anchor? fun hl => do if let some tok@⟨k, _txt⟩ := hl.matchingName? nameStr then @@ -346,13 +347,14 @@ private meta def suggestTerms (hl : Highlighted) (input : String) : Array String lines ++ (smartSuggestions out.toArray input (threshold := (max ·.length ·.length)) (count := 15)) -public meta def moduleTermInline (args : Array Arg) (inls : TSyntaxArray `inline) : DocElabM (Array Term) := do +public meta def moduleTermInline (args : Array Arg) (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : DocElabM (Array Term) := do let cfg@{module := moduleName, project, anchor?, showProofStates := _, defSite := _} ← parseThe CodeContext args let term ← oneCodeStr inls + let termStr := term.getVersoCode withAnchored project moduleName anchor? fun hl => do - if term.getString.trimAscii.isEmpty then - let suggs := suggestTerms hl term.getString + if termStr.trimAscii.isEmpty then + let suggs := suggestTerms hl termStr let h ← hintAt term "Use one of these" suggs let expectedString := ExpectString.abbreviateString (maxLength := 100) <| hl.toString let mut msg := m!"No expected term provided.\n" @@ -360,14 +362,14 @@ public meta def moduleTermInline (args : Array Arg) (inls : TSyntaxArray `inline msg := msg ++ h logErrorAt term msg return #[← ``(sorryAx _ true)] - else if let some e := hl.matchingExpr? term.getString then + else if let some e := hl.matchingExpr? termStr then logInfos e return #[← ``(leanInline $(quote e) $(quote cfg.toCodeConfig))] else - let suggs := suggestTerms hl term.getString + let suggs := suggestTerms hl termStr let h ← hintAt term "Use one of these" suggs let expectedString := ExpectString.abbreviateString (maxLength := 100) <| hl.toString - let mut msg := m!"Not found: `{term.getString}`\n" + let mut msg := m!"Not found: `{termStr}`\n" msg := msg ++ m!"in:{indentD <| m!"\n".joinSep <| (m!"{·}") <$> expectedString.splitOn "\n"}" msg := msg ++ h logErrorAt term msg @@ -397,11 +399,12 @@ public meta def anchorTerm : RoleExpander else throwError "Expected a positional argument first (the anchor name)" -public meta def moduleTermBlock (args : Array Arg) (term : StrLit) : DocElabM (Array Term) := do +public meta def moduleTermBlock (args : Array Arg) (term : Lean.Doc.VersoCodeBlock) : + DocElabM (Array Term) := do let cfg@{module := moduleName, project, anchor?, showProofStates := _, defSite := _} ← parseThe CodeContext args withAnchored project moduleName anchor? fun hl => do - let str := term.getString.trimAscii.copy + let str := term.getVersoCodeBlock.trimAscii.copy if str.isEmpty then let ref ← getRef let suggs := suggestTerms hl str @@ -468,8 +471,10 @@ private meta partial def findTrace? (header : String) : MessageContents Highligh if msg.toString == header then pure t else chs.findSome? (findTrace? header) -public meta def outputBlock (args : Array Arg) (str : StrLit) : DocElabM (Array Term) := do +public meta def outputBlock (args : Array Arg) (str : Lean.Doc.VersoCodeBlock) : + DocElabM (Array Term) := do let {module := moduleName, project, anchor?, severity, expandTraces, onlyTrace, showProofStates := _, defSite := _} ← parseThe MessageContext args + let strText := str.getVersoCodeBlock withAnchored project moduleName anchor? fun hl => do let infos : Array _ := allInfo hl @@ -484,7 +489,7 @@ public meta def outputBlock (args : Array Arg) (str : StrLit) : DocElabM (Array else continue else pure <| msg candidates := candidates.push msg - if SubVerso.Examples.Messages.messagesMatch (msg.toString (expandTraces := expandTraces)) str.getString then + if SubVerso.Examples.Messages.messagesMatch (msg.toString (expandTraces := expandTraces)) strText then if msg.severity == .ofSeverity severity.1 then return #[← ``(leanOutputBlock $(quote msg) (expandTraces := $(quote expandTraces)))] else @@ -502,10 +507,10 @@ public meta def outputBlock (args : Array Arg) (str : StrLit) : DocElabM (Array err := err ++ (m!"\nor".joinSep <| candidates.toList.map fun msg => indentD (msg.toString (expandTraces := expandTraces)) ++ "\n") - if str.getString.trimAscii.isEmpty then + if strText.trimAscii.isEmpty then err := err ++ "but nothing was provided." else - err := err ++ m!"but got:{indentD str.getString.trimAscii.copy}" + err := err ++ m!"but got:{indentD strText.trimAscii.copy}" if suggs.size = 1 then err := err ++ (← hintAt str "Use this:\n" suggs) else if suggs.size > 1 then @@ -594,7 +599,7 @@ public meta def anchorWarning : CodeBlockExpander throwError "Expected a positional argument first (the anchor name)" -public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray `inline) : DocElabM (Array Term) := do +public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : DocElabM (Array Term) := do let str? ← oneCodeStr? inls let {module := moduleName, project, anchor?, expandTraces, onlyTrace, severity, showProofStates := _, defSite := _} ← parseThe MessageContext args @@ -602,6 +607,7 @@ public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray `inline) withAnchored project moduleName anchor? fun hl => do let infos := allInfo hl if let some str := str? then + let strText := str.getVersoCode let mut candidates : Array Highlighted.Message := #[] for (msg, _) in infos do let msg ← @@ -612,15 +618,18 @@ public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray `inline) else pure <| msg candidates := candidates.push msg - if SubVerso.Examples.Messages.messagesMatch (msg.toString (expandTraces := expandTraces)) str.getString then + if SubVerso.Examples.Messages.messagesMatch (msg.toString (expandTraces := expandTraces)) strText then if msg.severity == .ofSeverity severity.1 then return #[← ``(leanOutputInline $(quote msg) true (expandTraces := $(quote expandTraces)))] else let wanted ← severityName msg.severity.toSeverity throwError "Mismatched severity. Expected '{repr severity.1}', got '{wanted}'.{← severityHint wanted severity.2}" + -- A role's content is the better place to report at than the role as a whole. let ref := - if let `(inline|role{ $_ $_* }[ $x ]) := (← getRef) then x.raw else str + match Lean.Doc.RoleView.of ⟨← getRef⟩ with + | some v => if h : v.content.size = 1 then v.content[0].raw else str.raw + | none => str.raw let suggs : Array Suggestion := candidates.map fun msg => { suggestion := quoteCode (msg.toString (expandTraces := expandTraces)).trimAscii.copy, @@ -632,7 +641,7 @@ public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray `inline) let err := m!"Expected one of:{indentD (m!"\n".joinSep <| candidates.toList.map (·.toString (expandTraces := expandTraces)))}" ++ - m!"\nbut got:{indentD str.getString}\n" ++ h + m!"\nbut got:{indentD strText}\n" ++ h logErrorAt str err else let candidates := infos.filterMap fun (msg, _) => @@ -692,7 +701,7 @@ public meta def moduleOutWarningRole : RoleExpander | args, inls => withTraceNode `Elab.Verso (fun _ => pure m!"moduleOutWarningRole") <| moduleOutInline (#[.anon <| .name <| mkIdent ``MessageSeverity.warning] ++ args) inls -public meta def anchorOutAsRole (severity : Name) (args : Array Arg) (inls : TSyntaxArray `inline) : DocElabM (Array Term) := +public meta def anchorOutAsRole (severity : Name) (args : Array Arg) (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : DocElabM (Array Term) := if let some (Arg.anon a) := args[0]? then moduleOutInline (#[.anon <| .name <| mkIdent severity, .named .missing (mkIdent `anchor) a] ++ args.drop 1) inls else @@ -722,7 +731,7 @@ public meta def lit : RoleExpander | args, inls => do ArgParse.done.run args let kw ← oneCodeStr inls - return #[← ``(Inline.code $(quote kw.getString))] + return #[← ``(Inline.code $(quote kw.getVersoCode))] private meta def hasSubstring (s pattern : String) : Bool := @@ -747,10 +756,10 @@ private meta def hasSubstring (s pattern : String) : Bool := /-- Internal detail of anchor suggestion mechanism. -/ -@[inline_expander Lean.Doc.Syntax.code] +@[inline_expander Lean.Doc.Parser.Inline.code] public meta def suggest : InlineExpander - | `(inline| code( $str )) => do - let str' := str.getString + | .code v => do + let str' := v.getVersoCode unless verso.examples.suggest.get (← getOptions) do -- Delegate to the next handler @@ -825,5 +834,5 @@ public meta def suggest : InlineExpander let h ← hint m!"Try one of these:" suggestions logWarning <| m!"Code element could be highlighted." ++ h - return (← ``(Inline.code $(quote str.getString))) + return (← ``(Inline.code $(quote str'))) | _ => Elab.throwUnsupportedSyntax diff --git a/verso/src/verso/Verso/Doc/ArgParse.lean b/verso/src/verso/Verso/Doc/ArgParse.lean index bcf9730a3..838220ebf 100644 --- a/verso/src/verso/Verso/Doc/ArgParse.lean +++ b/verso/src/verso/Verso/Doc/ArgParse.lean @@ -699,7 +699,7 @@ open Verso.Parser in Parses a sequence of Verso inline elements from a string literal. The resulting syntax is adjusted so that escapes in the string literal are accounted for in source positions. -/ -def ValDesc.inlinesString [MonadFileMap m] : ValDesc m (TSyntaxArray `inline) where +def ValDesc.inlinesString [MonadFileMap m] : ValDesc m (TSyntaxArray ``Lean.Doc.Parser.inline) where description := doc!"a string that contains a sequence of inline elements" signature := .String get diff --git a/verso/src/verso/Verso/Doc/Elab.lean b/verso/src/verso/Verso/Doc/Elab.lean index c204bfb23..87737f736 100644 --- a/verso/src/verso/Verso/Doc/Elab.lean +++ b/verso/src/verso/Verso/Doc/Elab.lean @@ -24,65 +24,56 @@ open Verso.ArgParse (SigDoc) set_option backward.privateInPublic false -@[inline_expander Lean.Doc.Syntax.text] -public meta partial def _root_.Lean.Doc.Syntax.text.expand : InlineExpander := fun x => - match x with - | `(inline| $s:str) => do - -- Erase the source locations from the string literal to prevent unwanted hover info - ``(Inline.text $(⟨deleteInfo s.raw⟩)) +@[inline_expander Lean.Doc.Parser.Inline.text] +public meta partial def _root_.Lean.Doc.Parser.Inline.text.expand : InlineExpander + -- `quote` builds the literal without source locations, preventing unwanted hover info + | .text v => ``(Inline.text $(quote v.getVersoText)) | _ => throwUnsupportedSyntax - where - deleteInfo : Syntax → Syntax - | .node _ k args => .node .none k (args.map deleteInfo) - | .atom _ val => .atom .none val - | .ident _ rawVal val preres => .ident .none rawVal val preres - | .missing => .missing - -@[inline_expander Lean.Doc.Syntax.linebreak] -public meta def _root_.linebreak.expand : InlineExpander - | `(inline|line! $s:str) => - ``(Inline.linebreak $(quote s.getString)) + +@[inline_expander Lean.Doc.Parser.Inline.linebreak] +public meta def _root_.Lean.Doc.Parser.Inline.linebreak.expand : InlineExpander + | .linebreak _ => ``(Inline.linebreak $(quote "\n")) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.emph] -public meta def _root_.Lean.Doc.Syntax.emph.expand : InlineExpander - | `(inline| _[ $args* ]) => do - ``(Inline.emph #[$[$(← args.mapM elabInline)],*]) +@[inline_expander Lean.Doc.Parser.Inline.emph] +public meta def _root_.Lean.Doc.Parser.Inline.emph.expand : InlineExpander + | .emph v => do + ``(Inline.emph #[$[$(← v.content.mapM elabInline)],*]) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.bold] -public meta def _root_.Lean.Doc.Syntax.bold.expand : InlineExpander - | `(inline| *[ $args* ]) => do - ``(Inline.bold #[$[$(← args.mapM elabInline)],*]) +@[inline_expander Lean.Doc.Parser.Inline.bold] +public meta def _root_.Lean.Doc.Parser.Inline.bold.expand : InlineExpander + | .bold v => do + ``(Inline.bold #[$[$(← v.content.mapM elabInline)],*]) | _ => throwUnsupportedSyntax -meta def parseArgVal (val : TSyntax `arg_val) : DocElabM ArgVal := do - match val with - | `(arg_val|$s:str) => pure <| .str s - | `(arg_val|$x:ident) => pure <| .name x - | `(arg_val|$n:num) => pure <| .num n - | other => throwErrorAt other "Can't decode argument value '{repr other}'" +meta def parseArgVal (val : TSyntax ``Lean.Doc.Parser.argVal) : DocElabM ArgVal := do + match Lean.Doc.ArgValView.of val with + | some (.str s _) => pure <| .str s + | some (.name x) => pure <| .name x + | some (.num n _) => pure <| .num n + | none => throwErrorAt val "Can't decode argument value '{repr val}'" -public meta def parseArgs (argStx : TSyntaxArray `doc_arg) : DocElabM (Array Arg) := do +public meta def parseArgs (argStx : TSyntaxArray ``Lean.Doc.Parser.arg) : + DocElabM (Array Arg) := do let mut argVals := #[] for arg in argStx do - match arg with - | `(doc_arg|$v:arg_val) => + match Lean.Doc.ArgView.of arg with + | some (.anon _ v) => argVals := argVals.push (.anon (← parseArgVal v)) - | `(doc_arg|$x:ident := $v) => do + | some (.named _ none x _ v) => do + -- A named argument without parentheses is the deprecated spelling. let src := (← getFileMap).source if let some ⟨s, e⟩ := x.raw.getRange? (canonicalOnly := true) then if let some ⟨s', e'⟩ := v.raw.getRange? (canonicalOnly := true) then let hint ← MessageData.hint m!"Replace with the updated syntax:" #[s!"({s.extract src e} := {s'.extract src e'})"] (ref? := some arg) logWarningAt arg m!"Deprecated named argument syntax for `{x}`{hint}" argVals := argVals.push (.named arg x (← parseArgVal v)) - | `(doc_arg|($x:ident := $v)) => + | some (.named _ (some _) x _ v) => argVals := argVals.push (.named arg x (← parseArgVal v)) - | `(doc_arg|+$x) => - argVals := argVals.push (.flag arg x true) - | `(doc_arg|-$x) => - argVals := argVals.push (.flag arg x false) - | other => throwErrorAt other "Can't decode argument '{repr other}'" + | some (.flag _ _ x isOn) => + argVals := argVals.push (.flag arg x isOn) + | none => throwErrorAt arg "Can't decode argument '{repr arg}'" pure argVals open Lean.Parser.Term in @@ -142,82 +133,77 @@ private meta def extensionResult {α : Type} return (← ``(Block.concat (genre := $(⟨genre⟩)) #[$[$termStxs],*])) open Lean.Parser.Term in -@[inline_expander Lean.Doc.Syntax.role] -public meta def _root_.Lean.Doc.Syntax.role.expand : InlineExpander - | inline@`(inline| role{$name $args*} [$subjects*]) => do - withRef inline <| withFreshMacroScope <| withIncRecDepth <| do +@[inline_expander Lean.Doc.Parser.Inline.role] +public meta def _root_.Lean.Doc.Parser.Inline.role.expand : InlineExpander + | .role v => do + let (name, args, subjects) := (v.name, v.args, v.content) + withRef v.stx <| withFreshMacroScope <| withIncRecDepth <| do let (resolvedName, exp) ← registeredExtensionExpanders "role" "@[role]" registeredRoleNames roleExpandersFor isRoleExpanderTargetType name let argVals ← parseArgs args extensionResult .inline "Role" name resolvedName exp fun e => e argVals subjects | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.link] -public meta def _root_.Lean.Doc.Syntax.link.expand : InlineExpander - | `(inline| link[ $txt* ] $dest:link_target) => do +@[inline_expander Lean.Doc.Parser.Inline.link] +public meta def _root_.Lean.Doc.Parser.Inline.link.expand : InlineExpander + | .link v => do let url : TSyntax `term ← - match dest with - | `(link_target| ( $url )) => - pure (↑ url) - | `(link_target| [ $ref ]) => do - -- Round-trip through quote to get rid of source locations, preventing unwanted IDE info - addLinkRef ref - | _ => throwErrorAt dest "Couldn't parse link destination" - ``(Inline.link #[$[$(← txt.mapM elabInline)],*] $url) + match v.target with + | .url _ _ u _ => pure (quote u.getVersoLinkUrl) + | .ref _ _ name _ => addLinkRef name + ``(Inline.link #[$[$(← v.content.mapM elabInline)],*] $url) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.footnote] -public meta def _root_.Lean.Doc.Syntax.link.footnote : InlineExpander - | `(inline| footnote( $name:str )) => do - ``(Inline.footnote $(quote name.getString) $(← addFootnoteRef name)) +@[inline_expander Lean.Doc.Parser.Inline.footnote] +public meta def _root_.Lean.Doc.Parser.Inline.footnote.expand : InlineExpander + | .footnote v => do + ``(Inline.footnote $(quote v.getName) $(← addFootnoteRef v.name)) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.image] -public meta def _root_.Lean.Doc.Syntax.image.expand : InlineExpander - | `(inline| image( $alt:str ) $dest:link_target) => do - let altText := alt.getString +@[inline_expander Lean.Doc.Parser.Inline.image] +public meta def _root_.Lean.Doc.Parser.Inline.image.expand : InlineExpander + | .image v => do let url : TSyntax `term ← - match dest with - | `(link_target| ( $url )) => - pure (↑ url) - | `(link_target| [ $ref ]) => do - -- Round-trip through quote to get rid of source locations, preventing unwanted IDE info - addLinkRef ref - | _ => throwErrorAt dest "Couldn't parse link destination" - ``(Inline.image $(quote altText) $url) + match v.target with + | .url _ _ u _ => pure (quote u.getVersoLinkUrl) + | .ref _ _ name _ => addLinkRef name + ``(Inline.image $(quote v.getAlt) $url) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.code] -public meta def _root_.Lean.Doc.Syntax.code.expand : InlineExpander - | `(inline| code( $s )) => - ``(Inline.code $(quote s.getString)) +@[inline_expander Lean.Doc.Parser.Inline.code] +public meta def _root_.Lean.Doc.Parser.Inline.code.expand : InlineExpander + | .code v => ``(Inline.code $(quote v.getVersoCode)) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.inline_math] -public meta def _root_.Lean.Doc.Syntax.inline_math.expand : InlineExpander - | `(inline| \math code( $s )) => - ``(Inline.math MathMode.inline $(quote s.getString)) +/-- Both math markers share a view, which records which of them was written. -/ +private meta def mathExpand : InlineExpander + | .math v => + match v.mode with + | .inline => ``(Inline.math MathMode.inline $(quote v.getVersoCode)) + | .display => ``(Inline.math MathMode.display $(quote v.getVersoCode)) | _ => throwUnsupportedSyntax -@[inline_expander Lean.Doc.Syntax.display_math] -public meta def _root_.Lean.Doc.Syntax.display_math.expand : InlineExpander - | `(inline| \displaymath code( $s )) => - ``(Inline.math MathMode.display $(quote s.getString)) - | _ => throwUnsupportedSyntax +@[inline_expander Lean.Doc.Parser.Inline.inline_math] +public meta def _root_.Lean.Doc.Parser.Inline.inline_math.expand : InlineExpander := mathExpand +@[inline_expander Lean.Doc.Parser.Inline.display_math] +public meta def _root_.Lean.Doc.Parser.Inline.display_math.expand : InlineExpander := mathExpand -public meta def partCommand (cmd : TSyntax `block) : PartElabM Unit := + +public meta def partCommand (cmd : TSyntax ``Lean.Doc.Parser.block) : PartElabM Unit := withTraceNode `Elab.Verso.part (fun _ => pure m!"Part modification {cmd}") <| withRef cmd <| withFreshMacroScope <| do match cmd.raw with | stx@(.node _ kind _) => + let some view := Lean.Doc.BlockView.of ⟨stx⟩ + | fallback let exp ← partCommandsFor kind for e in exp do try - withFreshMacroScope <| e stx + withFreshMacroScope <| e view return catch | ex@(.internal id) => @@ -242,16 +228,15 @@ where elabBlock cmd addBlock blk (blockInternalDocReconstructionPlaceholder := hygenicName) -@[part_command Lean.Doc.Syntax.footnote_ref] -public meta partial def _root_.Lean.Doc.Syntax.footnote_ref.command : PartCommand - | `(block| [^ $name:str ]: $contents* ) => - addFootnoteDef name =<< contents.mapM (withRefsAllowed .onlyIfDefined <| elabInline ·) +@[part_command Lean.Doc.Parser.Block.footnote_ref] +public meta partial def _root_.Lean.Doc.Parser.Block.footnote_ref.command : PartCommand + | .footnoteRef v => + addFootnoteDef v.name =<< v.content.mapM (withRefsAllowed .onlyIfDefined <| elabInline ·) | _ => throwUnsupportedSyntax -@[part_command Lean.Doc.Syntax.link_ref] -public meta partial def _root_.Lean.Doc.Syntax.link_ref.command : PartCommand - | `(block| [ $name:str ]: $url:str ) => - addLinkDef name url.getString +@[part_command Lean.Doc.Parser.Block.link_ref] +public meta partial def _root_.Lean.Doc.Parser.Block.link_ref.command : PartCommand + | .linkRef v => addLinkDef v.name v.getUrl | _ => throwUnsupportedSyntax partial def PartElabM.State.close (endPos : String.Pos.Raw) (state : PartElabM.State) : Option PartElabM.State := @@ -267,21 +252,22 @@ partial def PartElabM.State.closeAll (endPos : String.Pos.Raw) (state : PartElab -@[part_command Lean.Doc.Syntax.header] -public meta partial def _root_.Lean.Doc.Syntax.header.command : PartCommand - | stx@`(block|header($headerLevel){$inlines*}) => do - let titleBits ← liftDocElabM <| inlines.mapM elabInline - let titleString := headerStxToString (← getEnv) stx +@[part_command Lean.Doc.Parser.Block.header] +public meta partial def _root_.Lean.Doc.Parser.Block.header.command : PartCommand + | .header v => do + let stx := v.stx + let titleBits ← liftDocElabM <| v.content.mapM elabInline + let titleString := inlinesToString (← getEnv) (v.content.map (·.raw)) let ambientLevel ← currentLevel - let headerLevel := headerLevel.getNat + 1 + let headerLevel := v.level + 1 if headerLevel > ambientLevel + 1 then throwErrorAt stx "Wrong header nesting - got {"".pushn '#' headerLevel} but expected at most {"#".pushn '#' ambientLevel}" -- New subheader? if headerLevel == ambientLevel + 1 then -- Prelude is done! pure () else - if let none := stx.getPos? then dbg_trace "No start position for {stx}" - PartElabM.closePartsUntil headerLevel stx.getPos! + if let none := stx.raw.getPos? then dbg_trace "No start position for {stx}" + PartElabM.closePartsUntil headerLevel stx.raw.getPos! -- Start a new subpart push { @@ -295,21 +281,24 @@ public meta partial def _root_.Lean.Doc.Syntax.header.command : PartCommand | _ => throwUnsupportedSyntax -@[part_command Lean.Doc.Syntax.metadata_block] -public meta def _root_.Lean.Doc.Syntax.metadata_block.command : PartCommand - | `(block| %%%%$tk $fieldOrAbbrev* %%%) => do +@[part_command Lean.Doc.Parser.Block.metadata_block] +public meta def _root_.Lean.Doc.Parser.Block.metadata_block.command : PartCommand + | .metadata v => do let ctxt := (← getThe PartElabM.State).partContext if ctxt.blocks.size > 0 || ctxt.priorParts.size > 0 then - throwErrorAt tk "Metadata blocks must precede both content and subsections" + throwErrorAt v.opener "Metadata blocks must precede both content and subsections" if ctxt.metadata.isSome then - throwErrorAt tk "Metadata already provided for this section" - let stx ← `(term| { $fieldOrAbbrev* }) + throwErrorAt v.opener "Metadata already provided for this section" + let fields := v.fields + let stx : Term := ⟨(← `(Lean.Parser.Term.structInst| { $[$fields],* })).raw⟩ modifyThe PartElabM.State fun st => {st with partContext.metadata := some stx} | _ => throwUnsupportedSyntax -@[part_command Lean.Doc.Syntax.command] +@[part_command Lean.Doc.Parser.Block.command] public meta def includeSection : PartCommand - | `(block|command{include $args* }) => do + | .command v => do + unless v.name.getId == `include do Lean.Elab.throwUnsupportedSyntax + let args := v.args if h : args.size = 0 then throwError "Expected an argument" else if h : args.size > 2 then throwErrorAt args[2] "Expected one or two arguments" else @@ -333,10 +322,10 @@ public meta def includeSection : PartCommand where resolved id := mkIdentFrom id <$> realizeGlobalConstNoOverloadWithInfo (mkIdentFrom id (docName id.getId)) -@[block_expander Lean.Doc.Syntax.command] -public meta def _root_.Lean.Doc.Syntax.command.expand : BlockExpander := fun block => - match block with - | `(block|command{$name $args*}) => do +@[block_expander Lean.Doc.Parser.Block.command] +public meta def _root_.Lean.Doc.Parser.Block.command.expand : BlockExpander + | .command v => do + let (block, name, args) := (v.stx, v.name, v.args) withTraceNode `Elab.Verso.block (fun _ => pure m!"Block role {name}") <| withRef block <| withFreshMacroScope <| withIncRecDepth <| do let resolvedName ← resolveKnownExtensionName "block command" registeredBlockCommandNames name @@ -347,115 +336,107 @@ public meta def _root_.Lean.Doc.Syntax.command.expand : BlockExpander := fun blo extensionResult .block "Command" name resolvedName exp fun e => e argVals | _ => throwUnsupportedSyntax -@[block_expander Lean.Doc.Syntax.para] -public meta partial def _root_.Lean.Doc.Syntax.para.expand : BlockExpander - | `(block| para[ $args:inline* ]) => do +@[block_expander Lean.Doc.Parser.Block.para] +public meta partial def _root_.Lean.Doc.Parser.Block.para.expand : BlockExpander + | .para v => do let genre := (← readThe DocElabContext).genreSyntax - ``(Block.para (genre := $(⟨genre⟩)) #[$[$(← args.mapM elabInline)],*]) + ``(Block.para (genre := $(⟨genre⟩)) #[$[$(← v.content.mapM elabInline)],*]) | _ => throwUnsupportedSyntax -meta def elabLi (block : Syntax) : DocElabM (Syntax × TSyntax `term) := - withRef block <| - match block with - | `(list_item|*%$dot $contents:block*) => do +meta def elabLi (marker : Syntax) (contents : TSyntaxArray ``Lean.Doc.Parser.block) + (stx : Syntax) : DocElabM (Syntax × TSyntax `term) := + withRef stx <| do let genre := (← readThe DocElabContext).genreSyntax let item ← ``(ListItem.mk (α := Block $(⟨genre⟩)) #[$[$(← contents.mapM elabBlock)],*]) - pure (dot, item) - | _ => - throwUnsupportedSyntax + pure (marker, item) -@[block_expander Lean.Doc.Syntax.ul] -public meta def _root_.Lean.Doc.Syntax.ul.expand : BlockExpander - | `(block|ul{$itemStxs*}) => do +@[block_expander Lean.Doc.Parser.Block.ul] +public meta def _root_.Lean.Doc.Parser.Block.ul.expand : BlockExpander + | .ul v => do let genre := (← readThe DocElabContext).genreSyntax let mut bullets : Array Syntax := #[] let mut items : Array (TSyntax `term) := #[] - for i in itemStxs do - let (b, item) ← elabLi i + for i in v.items do + let (b, item) ← elabLi i.marker i.contents i.stx bullets := bullets.push b items := items.push item - let info := DocListInfo.mk bullets itemStxs + let info := DocListInfo.mk bullets (v.items.map (·.stx.raw)) for b in bullets do pushInfoLeaf <| .ofCustomInfo {stx := b, value := Dynamic.mk info} ``(Block.ul (genre := $(⟨genre⟩)) #[$items,*]) | _ => throwUnsupportedSyntax -@[block_expander Lean.Doc.Syntax.ol] -public meta def _root_.Lean.Doc.Syntax.ol.expand : BlockExpander - | `(block|ol($start:num){$itemStxs*}) => do +@[block_expander Lean.Doc.Parser.Block.ol] +public meta def _root_.Lean.Doc.Parser.Block.ol.expand : BlockExpander + | .ol v => do let genre := (← readThe DocElabContext).genreSyntax let mut bullets : Array Syntax := #[] let mut items : Array (TSyntax `term) := #[] - for i in itemStxs do - let (b, item) ← elabLi i + for i in v.items do + let (b, item) ← elabLi i.marker i.contents i.stx bullets := bullets.push b items := items.push item - let info := DocListInfo.mk bullets itemStxs + let info := DocListInfo.mk bullets (v.items.map (·.stx.raw)) for b in bullets do pushInfoLeaf <| .ofCustomInfo {stx := b, value := Dynamic.mk info} - ``(Block.ol (genre := $(⟨genre⟩)) $start #[$items,*]) + ``(Block.ol (genre := $(⟨genre⟩)) $(quote v.start) #[$items,*]) | _ => throwUnsupportedSyntax -meta def elabDesc (block : Syntax) : DocElabM (Syntax × TSyntax `term) := - withRef block <| - match block with - | `(desc_item|:%$colon $dts* => $dds*) => do +meta def elabDesc (item : Lean.Doc.DescItemView) : DocElabM (Syntax × TSyntax `term) := + withRef item.stx <| do let genre := (← readThe DocElabContext).genreSyntax - let item ← ``(DescItem.mk (α := Inline $(⟨genre⟩)) (β := Block $(⟨genre⟩)) #[$[$(← dts.mapM elabInline)],*] #[$[$(← dds.mapM elabBlock)],*]) - pure (colon, item) - | _ => - throwUnsupportedSyntax + let item' ← ``(DescItem.mk (α := Inline $(⟨genre⟩)) (β := Block $(⟨genre⟩)) #[$[$(← item.term.mapM elabInline)],*] #[$[$(← item.desc.mapM elabBlock)],*]) + pure (item.marker, item') -@[block_expander Lean.Doc.Syntax.dl] -public meta def _root_.Lean.Doc.Syntax.dl.expand : BlockExpander - | `(block|dl{$itemStxs*}) => do +@[block_expander Lean.Doc.Parser.Block.dl] +public meta def _root_.Lean.Doc.Parser.Block.dl.expand : BlockExpander + | .dl v => do let genre := (← readThe DocElabContext).genreSyntax let mut colons : Array Syntax := #[] let mut items : Array (TSyntax `term) := #[] - for i in itemStxs do + for i in v.items do let (b, item) ← elabDesc i colons := colons.push b items := items.push item - let info := DocListInfo.mk colons itemStxs + let info := DocListInfo.mk colons (v.items.map (·.stx.raw)) for b in colons do pushInfoLeaf <| .ofCustomInfo {stx := b, value := Dynamic.mk info} ``(Block.dl (genre := $(⟨genre⟩)) #[$[$items],*]) | _ => throwUnsupportedSyntax -@[block_expander Lean.Doc.Syntax.blockquote] -public meta def _root_.Lean.Doc.Syntax.blockquote.expand : BlockExpander - | `(block|> $innerBlocks*) => do - ``(Block.blockquote #[$[$(← innerBlocks.mapM elabBlock)],*]) +@[block_expander Lean.Doc.Parser.Block.blockquote] +public meta def _root_.Lean.Doc.Parser.Block.blockquote.expand : BlockExpander + | .blockquote v => do + ``(Block.blockquote #[$[$(← v.content.mapM elabBlock)],*]) | _ => throwUnsupportedSyntax -@[block_expander Lean.Doc.Syntax.codeblock] -public meta def _root_.Lean.Doc.Syntax.codeblock.expand : BlockExpander - | `(block|``` $nameStx:ident $argsStx* | $contents:str ```) => do - -- TODO typed syntax here - let args ← parseArgs <| argsStx.map (⟨·⟩) +@[block_expander Lean.Doc.Parser.Block.codeblock] +public meta def _root_.Lean.Doc.Parser.Block.codeblock.expand : BlockExpander + | .codeblock v => do + let some nameStx := v.name? + | return ← ``(Block.code $(quote v.getVersoCodeBlock)) + let args ← parseArgs v.args let (resolvedName, exp) ← registeredExtensionExpanders "code block" "@[code_block]" registeredCodeBlockNames codeBlockExpandersFor isCodeBlockExpanderTargetType nameStx - extensionResult .block "Code block" nameStx resolvedName exp fun e => e args contents - | `(block|``` | $contents:str ```) => do - ``(Block.code $(quote contents.getString)) + extensionResult .block "Code block" nameStx resolvedName exp fun e => e args v.content | _ => throwUnsupportedSyntax -@[block_expander Lean.Doc.Syntax.directive] -public meta def _root_.Lean.Doc.Syntax.directive.expand : BlockExpander - | `(block| ::: $nameStx:ident $argsStx* { $contents:block* } ) => do - let args ← parseArgs argsStx +@[block_expander Lean.Doc.Parser.Block.directive] +public meta def _root_.Lean.Doc.Parser.Block.directive.expand : BlockExpander + | .directive v => do + let args ← parseArgs v.args let (resolvedName, exp) ← registeredExtensionExpanders "directive" "@[directive]" registeredDirectiveNames directiveExpandersFor - isDirectiveExpanderTargetType nameStx - extensionResult .block "Directive" nameStx resolvedName exp fun e => e args contents + isDirectiveExpanderTargetType v.name + extensionResult .block "Directive" v.name resolvedName exp fun e => e args v.content | _ => throwUnsupportedSyntax diff --git a/verso/src/verso/Verso/Doc/Elab/Basic.lean b/verso/src/verso/Verso/Doc/Elab/Basic.lean index 1816f7f7c..6ea4940d7 100644 --- a/verso/src/verso/Verso/Doc/Elab/Basic.lean +++ b/verso/src/verso/Verso/Doc/Elab/Basic.lean @@ -111,7 +111,7 @@ public def PartFrame.close (fr : PartFrame) (endPos : String.Pos.Raw) : Finished /-- References that must be local to the current blob of concrete document syntax -/ public structure DocDef (α : Type) where - defSite : TSyntax `str + defSite : Syntax val : α deriving Repr diff --git a/verso/src/verso/Verso/Doc/Elab/Block.lean b/verso/src/verso/Verso/Doc/Elab/Block.lean index 8da3023c5..21de70940 100644 --- a/verso/src/verso/Verso/Doc/Elab/Block.lean +++ b/verso/src/verso/Verso/Doc/Elab/Block.lean @@ -19,18 +19,19 @@ open Verso.ArgParse (SigDoc) set_option backward.privateInPublic false -def decorateClosing : TSyntax `block → DocElabM Unit - | `(block|:::%$s $_ $_* { $_* }%$e) - | `(block|```%$s $_ $_* | $_ ```%$e) - | `(block|%%%%$s $_* %%%%$e) => closes s e +/-- Records the delimiters of a block that has both, so that each hover mentions the other. -/ +def decorateClosing : Lean.Doc.BlockView → DocElabM Unit + | .directive v => closes v.opener v.closer + | .codeblock v => closes v.openFence v.closeFence + | .metadata v => closes v.opener v.closer | _ => pure () /-- Elaborates a parsed block into syntax denoting an expression of type `Block genre`. -/ -public partial def elabBlock (block : TSyntax `block) : DocElabM (TSyntax `term) := +public partial def elabBlock (block : TSyntax ``Lean.Doc.Parser.block) : + DocElabM (TSyntax `term) := withTraceNode `Elab.Verso.block (fun _ => pure m!"Block {block}") <| withRef block <| withFreshMacroScope <| withIncRecDepth <| do - decorateClosing block match block.raw with | .missing => ``(sorryAx (Block _) (synthetic := true)) @@ -43,10 +44,13 @@ public partial def elabBlock (block : TSyntax `block) : DocElabM (TSyntax `term) withRef stxNew <| elabBlock ⟨stxNew⟩ | none => + let some view := Lean.Doc.BlockView.of ⟨stx⟩ + | throwUnexpected stx + decorateClosing view let exp ← blockExpandersFor kind for e in exp do try - let termStx ← withFreshMacroScope <| e stx + let termStx ← withFreshMacroScope <| e view return termStx catch | ex@(.internal id) => diff --git a/verso/src/verso/Verso/Doc/Elab/Inline.lean b/verso/src/verso/Verso/Doc/Elab/Inline.lean index a1301db1b..e1a81898c 100644 --- a/verso/src/verso/Verso/Doc/Elab/Inline.lean +++ b/verso/src/verso/Verso/Doc/Elab/Inline.lean @@ -21,7 +21,8 @@ set_option backward.privateInPublic false public def throwUnexpected [Monad m] [MonadError m] (stx : Syntax) : m α := throwErrorAt stx "unexpected syntax{indentD stx}" -public partial def elabInline (inline : TSyntax `inline) : DocElabM (TSyntax `term) := +public partial def elabInline (inline : TSyntax ``Lean.Doc.Parser.inline) : + DocElabM (TSyntax `term) := withRef inline <| withFreshMacroScope <| withIncRecDepth <| do match inline.raw with | .missing => @@ -35,10 +36,12 @@ public partial def elabInline (inline : TSyntax `inline) : DocElabM (TSyntax `te withRef stxNew <| elabInline ⟨stxNew⟩ | none => + let some view := Lean.Doc.InlineView.of ⟨stx⟩ + | throwUnexpected stx let exp ← inlineExpandersFor kind for e in exp do try - let termStx ← withFreshMacroScope <| e stx + let termStx ← withFreshMacroScope <| e view return termStx catch | ex@(.internal id) => diff --git a/verso/src/verso/Verso/Doc/Elab/Monad.lean b/verso/src/verso/Verso/Doc/Elab/Monad.lean index a2995f908..4c77d7c2d 100644 --- a/verso/src/verso/Verso/Doc/Elab/Monad.lean +++ b/verso/src/verso/Verso/Doc/Elab/Monad.lean @@ -10,6 +10,7 @@ import Std.Data.HashSet import Lean.Elab.DeclUtil import Lean.Meta.Reduce import Lean.DocString.Syntax +public import Lean.DocString.View import Lean.DocString import SubVerso.Highlighting @@ -43,47 +44,40 @@ class HasLink (name : String) (doc : Name) where class HasNote (name : String) (doc : Name) (genre : Genre) where contents : Array (Inline genre) -private def linkRefName [Monad m] [MonadQuotation m] (docName : Name) (ref : TSyntax `str) : m Term := do - ``(HasLink.url $(quote ref.getString) $(quote docName)) +private def linkRefName [Monad m] [MonadQuotation m] (docName : Name) (ref : String) : m Term := do + ``(HasLink.url $(quote ref) $(quote docName)) -private def footnoteRefName [Monad m] [MonadQuotation m] (genre : Term) (docName : Name) (ref : TSyntax `str) : m Term := - ``(HasNote.contents $(quote ref.getString) $(quote docName) (genre := $genre)) +private def footnoteRefName [Monad m] [MonadQuotation m] (genre : Term) (docName : Name) (ref : String) : m Term := + ``(HasNote.contents $(quote ref) $(quote docName) (genre := $genre)) -- For use in IDE features and previews and such -@[inline_to_string Lean.Doc.Syntax.text] -public meta def _root_.Lean.Doc.Syntax.text.inline_to_string : InlineToString - | _, `(inline| $s:str) => some s.getString - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.text] +public meta def _root_.Lean.Doc.Parser.Inline.text.inline_to_string : InlineToString + | _, stx => Lean.Doc.TextView.of ⟨stx⟩ |>.map (·.getVersoText) -@[inline_to_string Lean.Doc.Syntax.linebreak] -public meta def _root_.Lean.Doc.Syntax.linebreak.inline_to_string : InlineToString - | _, `(inline|line! $_) => some " " - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.linebreak] +public meta def _root_.Lean.Doc.Parser.Inline.linebreak.inline_to_string : InlineToString + | _, stx => Lean.Doc.LinebreakView.of ⟨stx⟩ |>.map fun _ => " " -@[inline_to_string Lean.Doc.Syntax.emph] -public meta def _root_.Lean.Doc.Syntax.emph.inline_to_string : InlineToString - | env, `(inline| _[ $args* ]) => - some <| String.intercalate " " (Array.map (inlineToString env) args).toList - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.emph] +public meta def _root_.Lean.Doc.Parser.Inline.emph.inline_to_string : InlineToString + | env, stx => Lean.Doc.EmphView.of ⟨stx⟩ |>.map fun v => + String.intercalate " " (v.content.toList.map (inlineToString env ·.raw)) -@[inline_to_string Lean.Doc.Syntax.bold] -public meta def _root_.Lean.Doc.Syntax.bold.inline_to_string : InlineToString - | env, `(inline| *[ $args* ]) => - some <| String.intercalate " " (Array.map (inlineToString env) args).toList - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.bold] +public meta def _root_.Lean.Doc.Parser.Inline.bold.inline_to_string : InlineToString + | env, stx => Lean.Doc.BoldView.of ⟨stx⟩ |>.map fun v => + String.intercalate " " (v.content.toList.map (inlineToString env ·.raw)) -@[inline_to_string Lean.Doc.Syntax.code] -public meta def _root_.Lean.Doc.Syntax.code.inline_to_string : InlineToString - | _, `(inline| code( $str )) => - some str.getString - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.code] +public meta def _root_.Lean.Doc.Parser.Inline.code.inline_to_string : InlineToString + | _, stx => Lean.Doc.CodeView.of ⟨stx⟩ |>.map (·.getVersoCode) -@[inline_to_string Lean.Doc.Syntax.role] -public meta def _root_.Lean.Doc.Syntax.role.inline_to_string : InlineToString - | env, `(inline| role{ $_ $_* }[ $body* ]) => - String.join (body.toList.map (inlineToString env <| ·.raw)) - | _, _ => none +@[inline_to_string Lean.Doc.Parser.Inline.role] +public meta def _root_.Lean.Doc.Parser.Inline.role.inline_to_string : InlineToString + | env, stx => Lean.Doc.RoleView.of ⟨stx⟩ |>.map fun v => + String.join (v.content.toList.map (inlineToString env ·.raw)) @[inline_to_string null] public meta def nullInline_to_string : InlineToString @@ -109,9 +103,11 @@ public def inlineSyntaxToString (env : Environment) (inlines : Syntax) : String dbg_trace "didn't understand inline sequence {inlines} for string" "" -public def headerStxToString (env : Environment) : Syntax → String - | `(block|header($_){$inlines*}) => inlinesToString env inlines - | headerStx => dbg_trace "didn't understand {headerStx} for string" +public def headerStxToString (env : Environment) (headerStx : Syntax) : String := + match Lean.Doc.HeaderView.of ⟨headerStx⟩ with + | some v => inlinesToString env (v.content.map (·.raw)) + | none => + dbg_trace "didn't understand {headerStx} for string" "" /-- @@ -350,8 +346,9 @@ public def PartElabM.addBlock (block : TSyntax `term) (blockInternalDocReconstru public def PartElabM.addPart (finished : FinishedPart) : PartElabM Unit := modifyThe State fun st => { st with partContext.priorParts := st.partContext.priorParts.push finished } -public def PartElabM.addLinkDef (refName : TSyntax `str) (url : String) : PartElabM Unit := do - let strName := refName.getString +public def PartElabM.addLinkDef (refName : Lean.Doc.VersoRefName) (url : String) : + PartElabM Unit := do + let strName := refName.getVersoRefName let docName ← currentDocName match (← getThe State).linkDefs[strName]? with | none => @@ -367,13 +364,13 @@ public def PartElabM.addLinkDef (refName : TSyntax `str) (url : String) : PartEl } setReducibilityStatus n .implicitReducible Meta.addInstance n AttributeKind.global (eval_prio default) - modifyThe State fun st => {st with linkDefs := st.linkDefs.insert strName ⟨refName, url⟩} + modifyThe State fun st => {st with linkDefs := st.linkDefs.insert strName ⟨refName.raw, url⟩} | some ⟨_, url'⟩ => throwErrorAt refName "Already defined link [{strName}] as '{url'}'" -public def DocElabM.addLinkRef (refName : TSyntax `str) : DocElabM (TSyntax `term) := do - let strName := refName.getString +public def DocElabM.addLinkRef (refName : Lean.Doc.VersoRefName) : DocElabM (TSyntax `term) := do + let strName := refName.getVersoRefName match (← readThe DocElabContext).refsAllowed with | .always => pure () | .onlyIfDefined => @@ -382,15 +379,16 @@ public def DocElabM.addLinkRef (refName : TSyntax `str) : DocElabM (TSyntax `ter match (← getThe State).linkRefs[strName]? with | none => - modifyThe State fun st => {st with linkRefs := st.linkRefs.insert strName ⟨#[refName]⟩} - linkRefName (← currentDocName) refName + modifyThe State fun st => {st with linkRefs := st.linkRefs.insert strName ⟨#[refName.raw]⟩} + linkRefName (← currentDocName) strName | some ⟨uses⟩ => - modifyThe State fun st => {st with linkRefs := st.linkRefs.insert strName ⟨uses.push refName⟩} - linkRefName (← currentDocName) refName + modifyThe State fun st => {st with linkRefs := st.linkRefs.insert strName ⟨uses.push refName.raw⟩} + linkRefName (← currentDocName) strName -public def PartElabM.addFootnoteDef (refName : TSyntax `str) (content : Array (TSyntax `term)) : PartElabM Unit := do - let strName := refName.getString +public def PartElabM.addFootnoteDef (refName : Lean.Doc.VersoRefName) + (content : Array (TSyntax `term)) : PartElabM Unit := do + let strName := refName.getVersoRefName let docName ← currentDocName let genre := (← readThe DocElabContext).genre match (← getThe State).footnoteDefs[strName]? with @@ -410,12 +408,13 @@ public def PartElabM.addFootnoteDef (refName : TSyntax `str) (content : Array (T } setReducibilityStatus n .implicitReducible Meta.addInstance n AttributeKind.global (eval_prio default) - modifyThe State fun st => {st with footnoteDefs := st.footnoteDefs.insert strName ⟨refName, content⟩} + modifyThe State fun st => {st with footnoteDefs := st.footnoteDefs.insert strName ⟨refName.raw, content⟩} | some _ => throwErrorAt refName m!"Already defined footnote [^{strName}]" -public def DocElabM.addFootnoteRef (refName : TSyntax `str) : DocElabM (TSyntax `term) := do - let strName := refName.getString +public def DocElabM.addFootnoteRef (refName : Lean.Doc.VersoRefName) : + DocElabM (TSyntax `term) := do + let strName := refName.getVersoRefName let genre := (← readThe DocElabContext).genreSyntax match (← readThe DocElabContext).refsAllowed with | .always => pure () @@ -425,11 +424,11 @@ public def DocElabM.addFootnoteRef (refName : TSyntax `str) : DocElabM (TSyntax match (← getThe State).footnoteRefs[strName]? with | none => - modifyThe State fun st => {st with footnoteRefs := st.footnoteRefs.insert strName ⟨#[refName]⟩} - footnoteRefName ⟨genre⟩ (← currentDocName) refName + modifyThe State fun st => {st with footnoteRefs := st.footnoteRefs.insert strName ⟨#[refName.raw]⟩} + footnoteRefName ⟨genre⟩ (← currentDocName) strName | some ⟨uses⟩ => - modifyThe State fun st => {st with footnoteRefs := st.footnoteRefs.insert strName ⟨uses.push refName⟩} - footnoteRefName ⟨genre⟩ (← currentDocName) refName + modifyThe State fun st => {st with footnoteRefs := st.footnoteRefs.insert strName ⟨uses.push refName.raw⟩} + footnoteRefName ⟨genre⟩ (← currentDocName) strName public def PartElabM.push (fr : PartFrame) : PartElabM Unit := modifyThe State fun st => {st with partContext := st.partContext.push fr} @@ -453,7 +452,7 @@ public def closes (openTok closeTok : Syntax) : DocElabM Unit := do let lineStr := if lineStr.startsWith "`" || lineStr.endsWith "`" then " " ++ lineStr ++ " " else lineStr.copy Hover.addCustomHover closeTok (.markdown s!"Closes line {line + 1}: ``````````{lineStr}``````````") -public abbrev InlineExpander := Syntax → DocElabM (TSyntax `term) +public abbrev InlineExpander := Lean.Doc.InlineView → DocElabM (TSyntax `term) initialize inlineExpanderAttr : KeyedDeclsAttribute InlineExpander ← mkDocExpanderAttribute `inline_expander ``InlineExpander "Indicates that this function expands inline elements of a given name" `inlineExpanderAttr @@ -538,7 +537,7 @@ public def FinishedPart.toVersoDoc ``(VersoDoc.mk (fun $docReconstructionPlaceholder => $finishedSyntax) $(quote reconstJson.compress)) -public abbrev BlockExpander := Syntax → DocElabM (TSyntax `term) +public abbrev BlockExpander := Lean.Doc.BlockView → DocElabM (TSyntax `term) initialize blockExpanderAttr : KeyedDeclsAttribute BlockExpander ← mkDocExpanderAttribute `block_expander ``BlockExpander "Indicates that this function expands block elements of a given name" `blockExpanderAttr @@ -581,7 +580,7 @@ initialize expanderSignatureExt : public def sig (α) [inst : FromArgs α DocElabM] : Option ArgParse.SigDoc := inst.fromArgs.signature -public abbrev PartCommand := Syntax → PartElabM Unit +public abbrev PartCommand := Lean.Doc.BlockView → PartElabM Unit initialize partCommandAttr : KeyedDeclsAttribute PartCommand ← mkDocExpanderAttribute `part_command ``PartCommand "Indicates that this function is used for side effects on the structure of the document" `partCommandAttr @@ -616,14 +615,16 @@ private def mkExpanderExtension (name : Name) : IO ExpanderExtension := .uniform entries.toArray } -public abbrev RoleExpander := Array Arg → TSyntaxArray `inline → DocElabM (Array (TSyntax `term)) +public abbrev RoleExpander := + Array Arg → TSyntaxArray ``Lean.Doc.Parser.inline → DocElabM (Array (TSyntax `term)) -public abbrev RoleExpanderOf α := α → TSyntaxArray `inline → DocElabM Term +public abbrev RoleExpanderOf α := α → TSyntaxArray ``Lean.Doc.Parser.inline → DocElabM Term initialize roleExpanderAttr : KeyedDeclsAttribute RoleExpander ← mkDocExpanderAttribute `role_expander ``RoleExpander "Indicates that this function is used to implement a given role" `roleExpanderAttr -public def toRole {α : Type} [FromArgs α DocElabM] (expander : α → TSyntaxArray `inline → DocElabM Term) : RoleExpander := +public def toRole {α : Type} [FromArgs α DocElabM] + (expander : α → TSyntaxArray ``Lean.Doc.Parser.inline → DocElabM Term) : RoleExpander := fun args inlines => do let v ← ArgParse.parse args return #[← expander v inlines] @@ -762,15 +763,17 @@ unsafe initialize registerBuiltinAttribute { } -public abbrev CodeBlockExpander := Array Arg → TSyntax `str → DocElabM (Array (TSyntax `term)) +public abbrev CodeBlockExpander := + Array Arg → Lean.Doc.VersoCodeBlock → DocElabM (Array (TSyntax `term)) -public abbrev CodeBlockExpanderOf α := α → StrLit → DocElabM Term +public abbrev CodeBlockExpanderOf α := α → Lean.Doc.VersoCodeBlock → DocElabM Term initialize codeBlockExpanderAttr : KeyedDeclsAttribute CodeBlockExpander ← mkDocExpanderAttribute `code_block_expander ``CodeBlockExpander "Indicates that this function is used to implement a given code block" `codeBlockExpanderAttr -public def toCodeBlock {α : Type} [FromArgs α DocElabM] (expander : α → StrLit → DocElabM Term) : CodeBlockExpander := +public def toCodeBlock {α : Type} [FromArgs α DocElabM] + (expander : α → Lean.Doc.VersoCodeBlock → DocElabM Term) : CodeBlockExpander := fun args str => do let v ← ArgParse.parse args return #[← expander v str] @@ -853,15 +856,17 @@ private def registeredCodeBlockNamesImpl : DocElabM (Array Name) := @[implemented_by registeredCodeBlockNamesImpl] public opaque registeredCodeBlockNames : DocElabM (Array Name) -public abbrev DirectiveExpander := Array Arg → TSyntaxArray `block → DocElabM (Array (TSyntax `term)) +public abbrev DirectiveExpander := + Array Arg → TSyntaxArray ``Lean.Doc.Parser.block → DocElabM (Array (TSyntax `term)) -public abbrev DirectiveExpanderOf α := α → TSyntaxArray `block → DocElabM Term +public abbrev DirectiveExpanderOf α := α → TSyntaxArray ``Lean.Doc.Parser.block → DocElabM Term initialize directiveExpanderAttr : KeyedDeclsAttribute DirectiveExpander ← mkDocExpanderAttribute `directive_expander ``DirectiveExpander "Indicates that this function is used to implement a given directive" `directiveExpanderAttr -public def toDirective {α : Type} [FromArgs α DocElabM] (expander : α → TSyntaxArray `block → DocElabM Term) : DirectiveExpander := +public def toDirective {α : Type} [FromArgs α DocElabM] + (expander : α → TSyntaxArray ``Lean.Doc.Parser.block → DocElabM Term) : DirectiveExpander := fun args blocks => do let v ← ArgParse.parse args return #[← expander v blocks] diff --git a/verso/src/verso/Verso/Doc/Helpers.lean b/verso/src/verso/Verso/Doc/Helpers.lean index 8a089c77f..d94a09b56 100644 --- a/verso/src/verso/Verso/Doc/Helpers.lean +++ b/verso/src/verso/Verso/Doc/Helpers.lean @@ -5,6 +5,7 @@ Author: David Thrane Christiansen -/ module import Lean.DocString.Syntax +public import Lean.DocString.View public import Lean.Exception public import Lean.Log @@ -18,36 +19,39 @@ namespace Verso.Doc If {name}`inlines` contains exactly one code inline, its contents are returned. Throws an error otherwise. -/ -public def oneCodeStr [Monad m] [MonadError m] (inlines : Array (TSyntax `inline)) : m StrLit := do +public def oneCodeStr [Monad m] [MonadError m] + (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m Lean.Doc.VersoCode := do let #[code] := inlines - | (if inlines.size == 0 then (throwError ·) else (throwErrorAt (mkNullNode inlines) ·)) "Expected one code element" - let `(inline|code($code:str)) := code + | (if inlines.size == 0 then (throwError ·) + else (throwErrorAt (mkNullNode (inlines.map (·.raw))) ·)) "Expected one code element" + let some v := Lean.Doc.CodeView.of code | throwErrorAt code "Expected a code element" - return code + return v.content /-- If {name}`inlines` contains exactly one code inline, its contents are returned. Otherwise, an error is logged and {name}`none` is returned. -/ public def oneCodeStr? [Monad m] [MonadError m] [MonadLog m] [AddMessageContext m] [MonadOptions m] - (inlines : Array (TSyntax `inline)) : m (Option StrLit) := do + (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m (Option Lean.Doc.VersoCode) := do let #[code] := inlines | if inlines.size == 0 then Lean.logError "Expected a code element" else - logErrorAt (mkNullNode inlines) "Expected one code element" + logErrorAt (mkNullNode (inlines.map (·.raw))) "Expected one code element" return none - let `(inline|code($code)) := code + let some v := Lean.Doc.CodeView.of code | logErrorAt code "Expected a code element" return none - return some code + return some v.content /-- If {name}`inlines` contains exactly one Lean name, it is returned with its source location as an identifier. Otherwise, an error is thrown. -/ -public def oneCodeName [Monad m] [MonadError m] (inlines : Array (TSyntax `inline)) : m Ident := do +public def oneCodeName [Monad m] [MonadError m] + (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m Ident := do let code ← oneCodeStr inlines - let str := code.getString + let str := code.getVersoCode let name := if str.contains '.' then str.toName else Name.str .anonymous str return mkIdentFrom code name diff --git a/verso/src/verso/Verso/Doc/Lsp.lean b/verso/src/verso/Verso/Doc/Lsp.lean index 3d1a4bcfb..f3f40039a 100644 --- a/verso/src/verso/Verso/Doc/Lsp.lean +++ b/verso/src/verso/Verso/Doc/Lsp.lean @@ -425,125 +425,114 @@ meta def decodeLeanTokens (data : Array Nat) : Array SemanticTokenEntry := Id.ru deriving instance Repr, BEq for SemanticTokenType -meta partial def versoTokens (text : FileMap) (stx : Syntax) : Array SemanticTokenEntry := Id.run do - match stx with - | `(inline|$_s:str) => - mkTok text .string stx - | `(inline|_[%$s $inlines* ]%$e) | `(inline|*[%$s $inlines* ]%$e) => - mkTok text .keyword s ++ versoTokens text (mkNullNode inlines) ++ mkTok text .keyword e - | `(inline|role{%$s $f $args*}%$e [%$s' $inlines* ]%$e') => - mkTok text .keyword s ++ - mkTok text .function f ++ - versoTokens text (mkNullNode args) ++ - mkTok text .keyword e ++ - mkTok text .keyword s' ++ - versoTokens text (mkNullNode inlines) ++ - mkTok text .keyword e' - | `(inline|link[%$s $inlines* ]%$e (%$s' $tgt )%$e') - | `(inline|link[%$s $inlines* ]%$e [%$s' $tgt ]%$e') => - mkTok text .keyword s ++ - versoTokens text (mkNullNode inlines) ++ - mkTok text .keyword e ++ - mkTok text .keyword s' ++ - mkTok text .parameter tgt ++ - mkTok text .keyword e' - | `(inline|image(%$s $alt )%$e (%$s' $tgt )%$e') - | `(inline|image(%$s $alt )%$e [%$s' $tgt ]%$e') => - mkTok text .keyword s ++ - versoTokens text alt ++ - mkTok text .keyword e ++ - mkTok text .keyword s' ++ - mkTok text .parameter tgt ++ - mkTok text .keyword e' - | `(inline|footnote(%$s $note )%$e) => - mkTok text .keyword s ++ - mkTok text .parameter note ++ - mkTok text .keyword e - | `(inline|code(%$s $_str )%$e) => - mkTok text .keyword s ++ - -- None for str, so Lean's can pass through - mkTok text .keyword e - | `(inline|\math%$m code(%$s $str )%$e) - | `(inline|\displaymath%$m code(%$s $str )%$e) => - mkTok text .keyword m ++ - mkTok text .keyword s ++ - -- Enum member arbitrarily chosen for uniqueness - mkTok text .enumMember str ++ - mkTok text .keyword e - | `(desc_item| :%$s $inlines* =>%$e $blocks*) => - mkTok text .keyword s ++ - versoTokens text (mkNullNode inlines) ++ - mkTok text .keyword e ++ - versoTokens text (mkNullNode blocks) - | `(list_item| *%$bulletOrNum $contents*) => - mkTok text .keyword bulletOrNum ++ - versoTokens text (mkNullNode contents) - | `(block| :::%$s $f $args* {%$s' $body* }%$e) => - mkTok text .keyword s ++ - mkTok text .function f ++ - versoTokens text (mkNullNode args) ++ - mkTok text .keyword s' ++ - versoTokens text (mkNullNode body) ++ - mkTok text .keyword e - | `(block| ```%$s $f $args* |%$s' $_code ```%$e) => - mkTok text .keyword s ++ - mkTok text .function f ++ - versoTokens text (mkNullNode args) ++ - mkTok text .keyword s' ++ - -- No token for the code, because we want Lean's tokens to shine through - mkTok text .keyword e - | `(block| >%$s $blocks*) => - mkTok text .keyword s ++ versoTokens text (mkNullNode blocks) - | `(block|header(%$s $n )%$e {%$s' $txt* }%$e') => - mkTok text .keyword s ++ - versoTokens text n ++ - mkTok text .keyword e ++ - mkTok text .keyword s' ++ - versoTokens text (mkNullNode txt) ++ - mkTok text .keyword e' - | `(block|[^%$s $n ]:%$e $txt*) => - mkTok text .keyword s ++ - mkTok text .parameter n ++ - mkTok text .keyword e ++ - versoTokens text (mkNullNode txt) - | `(block|[%$s $n ]:%$e $url) => - mkTok text .keyword s ++ - mkTok text .parameter n ++ - mkTok text .keyword e ++ - mkTok text .parameter url - | `(block| %%%%$s $_defs* %%%%$e) => - mkTok text .keyword s ++ - -- No tokens for defs, because Lean should supply them - mkTok text .keyword e - | `(block| command{%$s $f $args* }%$e) => - mkTok text .keyword s ++ - mkTok text .function f ++ - versoTokens text (mkNullNode args) ++ - mkTok text .keyword e - | `(doc_arg| $x:ident :=%$eq $v:arg_val) => - mkTok text .parameter x ++ - mkTok text .keyword eq ++ - versoTokens text v - | `(doc_arg| $v:arg_val) => - versoTokens text v - -- In the next three cases, no token is returned. This is to allow Lean's to shine through, if - -- there are any. It would be nice to add a priority mechanism to fall back to these defaults if - -- Lean didn't provide any. - | `(arg_val| $_v:num) => - --mkTok text .number v - #[] - | `(arg_val| $_v:ident) => - -- mkTok text .variable v - #[] - | `(arg_val| $_v:str) => - -- mkTok text .string v - #[] - | _ => +meta partial def versoTokens (text : FileMap) (stx : Syntax) : Array SemanticTokenEntry := + Id.run do + if let some v := Lean.Doc.InlineView.of ⟨stx⟩ then inlineTokens text v + else if let some v := Lean.Doc.BlockView.of ⟨stx⟩ then blockTokens text v + else if let some v := Lean.Doc.DescItemView.of ⟨stx⟩ then + mkTok text .keyword v.marker ++ + versoTokens text (mkNullNode (v.term.map (·.raw))) ++ + versoTokens text (mkNullNode (v.desc.map (·.raw))) + else if let some v := Lean.Doc.UnorderedListItemView.of ⟨stx⟩ then + mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.contents.map (·.raw))) + else if let some v := Lean.Doc.OrderedListItemView.of ⟨stx⟩ then + mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.contents.map (·.raw))) + else if let some v := Lean.Doc.ArgView.of ⟨stx⟩ then + match v with + | .anon _ val => versoTokens text val + | .named _ _ x eq val => + mkTok text .parameter x ++ mkTok text .keyword eq ++ versoTokens text val + | .flag _ sign x _ => mkTok text .keyword sign ++ mkTok text .parameter x + -- An argument value yields no token, so that Lean's own tokens show through. + else if (Lean.Doc.ArgValView.of ⟨stx⟩).isSome then #[] + else Id.run do let mut out := #[] for arg in stx.getArgs do out := out ++ versoTokens text arg return out where + inlineTokens (text : FileMap) : Lean.Doc.InlineView → Array SemanticTokenEntry + | .text v => mkTok text .string v.stx + | .linebreak _ => #[] + | .emph v => + mkTok text .keyword v.opener ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) ++ + mkTok text .keyword v.closer + | .bold v => + mkTok text .keyword v.opener ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) ++ + mkTok text .keyword v.closer + | .role v => + mkTok text .keyword v.braceOpen ++ + mkTok text .function v.name ++ + versoTokens text (mkNullNode (v.args.map (·.raw))) ++ + mkTok text .keyword v.braceClose ++ + (v.brackets.map (fun (o, _) => mkTok text .keyword o) |>.getD #[]) ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) ++ + (v.brackets.map (fun (_, c) => mkTok text .keyword c) |>.getD #[]) + | .link v => + mkTok text .keyword v.opener ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) ++ + mkTok text .keyword v.closer ++ + targetTokens text v.target + | .image v => + mkTok text .keyword v.opener ++ + mkTok text .string v.alt ++ + mkTok text .keyword v.closer ++ + targetTokens text v.target + | .footnote v => + mkTok text .keyword v.opener ++ + mkTok text .parameter v.name ++ + mkTok text .keyword v.closer + -- No token for the code itself, so that Lean's tokens show through. + | .code v => mkTok text .keyword v.opener ++ mkTok text .keyword v.closer + | .math v => + mkTok text .keyword v.marker ++ + mkTok text .keyword v.code.opener ++ + -- Enum member arbitrarily chosen for uniqueness + mkTok text .enumMember v.code.content ++ + mkTok text .keyword v.code.closer + + targetTokens (text : FileMap) : Lean.Doc.LinkTargetView → Array SemanticTokenEntry + | .url _ o url c | .ref _ o url c => + mkTok text .keyword o ++ mkTok text .parameter url ++ mkTok text .keyword c + + blockTokens (text : FileMap) : Lean.Doc.BlockView → Array SemanticTokenEntry + | .para v => versoTokens text (mkNullNode (v.content.map (·.raw))) + | .ul v => versoTokens text (mkNullNode (v.items.map (·.stx.raw))) + | .ol v => versoTokens text (mkNullNode (v.items.map (·.stx.raw))) + | .dl v => versoTokens text (mkNullNode (v.items.map (·.stx.raw))) + | .blockquote v => + mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.content.map (·.raw))) + | .directive v => + mkTok text .keyword v.opener ++ + mkTok text .function v.name ++ + versoTokens text (mkNullNode (v.args.map (·.raw))) ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) ++ + mkTok text .keyword v.closer + -- No token for the code, because Lean's tokens should show through. + | .codeblock v => + mkTok text .keyword v.openFence ++ + (v.name?.map (mkTok text .function ·) |>.getD #[]) ++ + versoTokens text (mkNullNode (v.args.map (·.raw))) ++ + mkTok text .keyword v.closeFence + | .command v => + mkTok text .function v.name ++ versoTokens text (mkNullNode (v.args.map (·.raw))) + | .header v => + mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.content.map (·.raw))) + | .footnoteRef v => + mkTok text .keyword v.opener ++ + mkTok text .parameter v.name ++ + mkTok text .keyword v.closer ++ + versoTokens text (mkNullNode (v.content.map (·.raw))) + | .linkRef v => + mkTok text .keyword v.opener ++ + mkTok text .parameter v.name ++ + mkTok text .keyword v.closer ++ + mkTok text .parameter v.url + -- No tokens for the fields, because Lean should supply them. + | .metadata v => mkTok text .keyword v.opener ++ mkTok text .keyword v.closer + mkTok (text : FileMap) (tokenType : SemanticTokenType) (stx : Syntax) : Array SemanticTokenEntry := Id.run do let (some startPos, some endPos) := (stx.getPos?, stx.getTailPos?) | return #[] diff --git a/verso/src/verso/Verso/ExpectString.lean b/verso/src/verso/Verso/ExpectString.lean index 1b71170ea..6b4c7fff4 100644 --- a/verso/src/verso/Verso/ExpectString.lean +++ b/verso/src/verso/Verso/ExpectString.lean @@ -4,6 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: David Thrane Christiansen -/ module +public import Verso.Literal public import Lean.Log import Lean.Elab.Command import Lean.Elab.InfoTree @@ -45,10 +46,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectStringOrDiff (expected : StrLit) (actual : String) +def expectStringOrDiff [Verso.VersoLiteral k] (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m (Option MessageData) := do - let expectedLines := expected.getString.splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (Verso.decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do @@ -71,10 +72,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectString (what : String) (expected : StrLit) (actual : String) +def expectString [Verso.VersoLiteral k] (what : String) (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m Bool := do - let expectedLines := expected.getString.splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (Verso.decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do diff --git a/verso/src/verso/Verso/Linters.lean b/verso/src/verso/Verso/Linters.lean index 32edd9fb0..32ca6d592 100644 --- a/verso/src/verso/Verso/Linters.lean +++ b/verso/src/verso/Verso/Linters.lean @@ -76,9 +76,10 @@ def typography : Linter where let h ← liftTermElabM <| MessageData.hint m!"Replace with Unicode" #[{suggestion := replacement}] (ref? := strLit) logLintIf linter strLit (m!"Use {what} ('{replacement}')" ++ h) - discard <| stx.replaceM fun - | `(inline|$s:str) => do - if let some ⟨start, stop⟩ := s.raw.getRange? then + discard <| stx.replaceM fun stx => do + let some (.text s) := Lean.Doc.InlineView.of ⟨stx⟩ + | pure none + if let some ⟨start, stop⟩ := s.content.raw.getRange? then let mut state : PunctuationState := if start == 0 || (start.prev text.source).get text.source ∈ ['\n', ' '] then .atBeginning (start.prev text.source) @@ -130,7 +131,6 @@ def typography : Linter where | _, _ => state := if c.isDigit then .afterDigit else .none pure none - | _ => pure none initialize addLinter typography @@ -189,12 +189,13 @@ def emphasisMinimization : Linter where let text ← getFileMap - discard <| stx.replaceM fun - | `(inline|_[%$tk1 $e* ]%$tk2) => do - lintDelimited linter.verso.markup.emph text tk1 tk2 '_' + discard <| stx.replaceM fun stx => do + match Lean.Doc.InlineView.of ⟨stx⟩ with + | some (.emph e) => + lintDelimited linter.verso.markup.emph text e.opener e.closer '_' pure none - | `(inline|*[%$tk1 $e* ]%$tk2) => do - lintDelimited linter.verso.markup.emph text tk1 tk2 '*' + | some (.bold b) => + lintDelimited linter.verso.markup.emph text b.opener b.closer '*' pure none | _ => pure none @@ -211,13 +212,13 @@ def codeMinimization : Linter where let text ← getFileMap - discard <| stx.replaceM fun - | `(inline|code(%$tk1 $_ )%$tk2) => do - lintDelimited linter.verso.markup.code text tk1 tk2 '`' + discard <| stx.replaceM fun stx => do + if let some (.code c) := Lean.Doc.InlineView.of ⟨stx⟩ then + lintDelimited linter.verso.markup.code text c.opener c.closer '`' pure none - | `(block|```%$tk1 $[$_ $_*]? | $_ ```%$tk2) => do - lintDelimited linter.verso.markup.codeBlock text tk1 tk2 '`' (minimal := 3) + else if let some (.codeblock c) := Lean.Doc.BlockView.of ⟨stx⟩ then + lintDelimited linter.verso.markup.codeBlock text c.openFence c.closeFence '`' (minimal := 3) pure none - | _ => pure none + else pure none initialize addLinter codeMinimization diff --git a/verso/src/verso/Verso/Literal.lean b/verso/src/verso/Verso/Literal.lean new file mode 100644 index 000000000..46d030917 --- /dev/null +++ b/verso/src/verso/Verso/Literal.lean @@ -0,0 +1,60 @@ +/- +Copyright (c) 2026 Lean FRO LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Author: David Thrane Christiansen +-/ +module + +public import Lean.DocString.View + +public section + +namespace Verso + +open Lean + +/-- +A syntax kind whose tokens denote a string. + +Each kind writes its string with delimiters and escapes of its own, so code that reads a literal's +text, or replaces it with new text, works at any of them. +-/ +class VersoLiteral (k : SyntaxNodeKinds) where + /-- The text that `stx` denotes. -/ + decode (stx : TSyntax k) : String + /-- A token at `src`'s position that denotes `value`. -/ + encode (src : Syntax) (value : String) : TSyntax k + +export VersoLiteral (decode encode) + +instance : VersoLiteral strLitKind where + decode stx := stx.getString + encode src value := Syntax.mkStrLit value (info := src.getHeadInfo) + +instance : VersoLiteral ``Doc.Parser.versoText where + decode stx := stx.getVersoText + encode src value := Doc.mkVersoTextFrom src value + +instance : VersoLiteral ``Doc.Parser.versoRef where + decode stx := stx.getVersoRefName + encode src value := Doc.mkVersoRefNameFrom src value + +instance : VersoLiteral ``Doc.Parser.versoLinkUrl where + decode stx := stx.getVersoLinkUrl + encode src value := Doc.mkVersoLinkUrlFrom src value + +instance : VersoLiteral ``Doc.Parser.versoLinkRefUrl where + decode stx := stx.getVersoLinkRefUrl + encode src value := Doc.mkVersoLinkRefUrlFrom src value + +instance : VersoLiteral ``Doc.Parser.versoImageAlt where + decode stx := stx.getVersoImageAlt + encode src value := Doc.mkVersoImageAltFrom src value + +instance : VersoLiteral ``Doc.Parser.versoCode where + decode stx := stx.getVersoCode + encode src value := Doc.mkVersoCodeFrom src value + +instance : VersoLiteral ``Doc.Parser.versoCodeBlock where + decode stx := stx.getVersoCodeBlock + encode src value := Doc.mkVersoCodeBlockFrom src value diff --git a/verso/src/verso/Verso/Parser.lean b/verso/src/verso/Verso/Parser.lean index a5676ca0d..2dc1a39b4 100644 --- a/verso/src/verso/Verso/Parser.lean +++ b/verso/src/verso/Verso/Parser.lean @@ -185,32 +185,6 @@ def bolThen (p : ParserFn) (description : String) : ParserFn := fun c s => else s else s.mkErrorAt description s.pos -/-- -We can only start a nestable block if we're immediately after a newline followed by a sequence of nestable block openers --/ -def onlyBlockOpeners : ParserFn := fun c s => - let position := c.fileMap.toPosition s.pos - let lineStart := c.fileMap.lineStart position.line - let ok : Bool := Id.run do - let mut iter := {String.Legacy.iter c.inputString with i := lineStart} - while iter.i < s.pos && iter.hasNext do - if iter.curr.isDigit then - while iter.curr.isDigit && iter.i < s.pos && iter.hasNext do - iter := iter.next - if !iter.hasNext then return false - else if iter.curr == '.' || iter.curr == ')' then iter := iter.next - else if iter.curr == ' ' then iter := iter.next - else if iter.curr == '>' then iter := iter.next - else if iter.curr == '*' then iter := iter.next - else if iter.curr == '+' then iter := iter.next - else if iter.curr == '-' then iter := iter.next - else return false - true - - if ok then s - else s.mkErrorAt s!"beginning of line or sequence of nestable block openers at {position}" s.pos - - def fakeAtom (str : String) (info : SourceInfo := SourceInfo.none) : ParserFn := fun _c s => let atom := mkAtom info str s.pushSyntax atom @@ -229,120 +203,6 @@ def strFn (str : String) : ParserFn := asStringFn <| fun c s => let s := go (String.Legacy.iter str) s if s.hasError then s.mkErrorAt s!"'{str}'" iniPos (some iniSz) else s -inductive OrderedListType where - /-- Items like 1. -/ - | numDot - /-- Items like 1) -/ - | parenAfter -deriving Repr, BEq, Ord, DecidableEq - -def OrderedListType.all : List OrderedListType := - [.numDot, .parenAfter] - -theorem OrderedListType.all_complete : ∀ x : OrderedListType, x ∈ all := by - unfold all; intro x; cases x <;> repeat constructor - -inductive UnorderedListType where - /-- Items like * -/ - | asterisk - /-- Items like - -/ - | dash - /-- Items like + -/ - | plus -deriving Repr, BEq, Ord, DecidableEq - -def UnorderedListType.all : List UnorderedListType := - [.asterisk, .dash, .plus] - -theorem UnorderedListType.all_complete : ∀ x : UnorderedListType, x ∈ all := by - unfold all; intro x; cases x <;> repeat constructor - -def unorderedListIndicator (type : UnorderedListType) : ParserFn := - asStringFn <| - match type with - | .asterisk => chFn '*' - | .dash => chFn '-' - | .plus => chFn '+' - -def orderedListIndicator (type : OrderedListType) : ParserFn := - asStringFn <| - takeWhile1Fn (·.isDigit) "digits" >> - match type with - | .numDot => chFn '.' - | .parenAfter => chFn ')' - -def bullet := atomicFn (go UnorderedListType.all) -where - go - | [] => fun _ s => s.mkError "no list type" - | [x] => atomicFn (unorderedListIndicator x) - | x :: xs => atomicFn (unorderedListIndicator x) <|> go xs - -def numbering := atomicFn (go OrderedListType.all) -where - go - | [] => fun _ s => s.mkError "no list type" - | [x] => atomicFn (orderedListIndicator x) - | x :: xs => atomicFn (orderedListIndicator x) <|> go xs - -def inlineTextChar : ParserFn := fun c s => - let i := s.pos - if h : c.atEnd i then s.mkEOIError - else - let curr := c.get' i h - match curr with - | '\\' => - let s := s.next' c i h - let i := s.pos - if h : c.atEnd i then s.mkEOIError - else s.next' c i h - | '*' | '_' | '\n' | '[' | ']' | '{' | '}' | '`' => s.mkUnexpectedErrorAt s!"'{curr}'" i - | '!' => - let s := s.next' c i h - let i' := s.pos - if h : c.atEnd i' then s - else if c.get' i' h == '[' - then s.mkUnexpectedErrorAt "![" i - else s - | '$' => - let s := s.next' c i h - let i' := s.pos - if h : c.atEnd i' then - s - else if c.get' i' h == '`' then - s.mkUnexpectedErrorAt "$`" i - else if c.get' i' h == '$' then - let s := s.next' c i' h - let i' := s.pos - if h : c.atEnd i' then - s - else if c.get' i' h == '`' then - s.mkUnexpectedErrorAt "$$`" i - else s - else s - | _ => s.next' c i h - -/-- Return some inline text up to the next inline opener or the end of -the line, whichever is first. Always consumes at least one -logical character on success, taking escaping into account. -/ -def inlineText : ParserFn := asStringFn (transform := unescapeStr) <| atomicFn inlineTextChar >> manyFn inlineTextChar - -/-- Block opener prefixes -/ -def blockOpener := atomicFn <| - takeWhileEscFn (· == ' ') >> - (atomicFn ((bullet >> chFn ' ')) <|> - atomicFn ((numbering >> chFn ' ')) <|> - atomicFn (strFn ": ") <|> - atomicFn (atLeastFn 3 (chFn ':')) <|> - atomicFn (atLeastFn 3 (chFn '`')) <|> - atomicFn (strFn "%%%") <|> - atomicFn (chFn '>')) - -def val : ParserFn := - nodeFn ``arg_num docNumLitFn <|> - nodeFn ``arg_ident docIdentFn <|> - nodeFn ``arg_str docStrLitFn - def withCurrentStackSize (p : Nat → ParserFn) : ParserFn := fun c s => p s.stxStack.size c s @@ -426,563 +286,54 @@ def recoverHereWithKeeping (stxs : Array Syntax) (keep : Nat) (p : ParserFn) : P show ParserFn from fun _ s => stxs.foldl (init := s.restore (rctx.initialSize + keep) rctx.initialPos) (·.pushSyntax ·) -def arg : ParserFn := - withCurrentStackSize fun iniSz => - flag <|> withParens iniSz <|> potentiallyNamed iniSz <|> (val >> mkAnon iniSz) -where - mkNamed (iniSz : Nat) : ParserFn := fun _ s => s.mkNode ``Lean.Doc.Syntax.named iniSz - mkNamedNoParen (iniSz : Nat) : ParserFn := fun _ s => s.mkNode ``Lean.Doc.Syntax.named_no_paren iniSz - mkAnon (iniSz : Nat) : ParserFn := fun _ s => s.mkNode ``Lean.Doc.Syntax.anon iniSz - mkIdent (iniSz : Nat) : ParserFn := fun _ s => s.mkNode ``Lean.Doc.Syntax.arg_ident iniSz - flag : ParserFn := - nodeFn ``Lean.Doc.Syntax.flag_on (asStringFn (strFn "+") >> recoverNonSpace noSpace >> recoverWs (docIdentFn (reportAs := "flag name"))) <|> - nodeFn ``Lean.Doc.Syntax.flag_off (asStringFn (strFn "-") >> recoverNonSpace noSpace >> recoverWs (docIdentFn (reportAs := "flag name"))) - noSpace : ParserFn := fun c s => - if h : c.atEnd s.pos then s - else - let ch := c.get' s.pos h - if ch == ' ' then - s.mkError "no space before" - else s - potentiallyNamed iniSz := - atomicFn docIdentFn >> eatSpaces >> - ((atomicFn (asStringFn <| strFn ":=") >> eatSpaces >> val >> eatSpaces >> mkNamedNoParen iniSz) <|> (mkIdent iniSz >> mkAnon iniSz)) - withParens iniSz := - atomicFn (asStringFn <| strFn "(") >> eatSpaces >> - recoverWs (docIdentFn (reportAs := "argument name")) >> eatSpaces >> - recoverWs (asStringFn <| strFn ":=") >> eatSpaces >> - recoverWs val >> eatSpaces >> - recoverEol (asStringFn <| strFn ")") >> eatSpaces >> - mkNamed iniSz - -/-- -Skip whitespace for name and arguments. If the argument is `none`, -it's in a single-line context and whitespace may only be the space -character. If it's `some N`, then newlines are allowed, but `N` is the -minimum indentation column. +/-! +Verso's markup is Lean's docstring markup, so the productions below are `Lean.Doc.Parser`'s. They +produce the syntax that Verso's elaborators consume. -/ -def nameArgWhitespace : (multiline : Option Nat) → ParserFn - | none => eatSpaces - | some n => takeWhileFn (fun c => c == ' ' || c == '\n') >> guardMinColumn n -def args (multiline : Option Nat := none) : ParserFn := - sepByFn true arg (nameArgWhitespace multiline) +export Lean.Doc.Parser ( + OrderedListType UnorderedListType InlineCtxt InList BlockCtxt + inlineTextCharFn blockOpenerFn valFn argEndWs argFn argsFn nameAndArgsFn + textFn emphFn boldFn codeFn mathFn linkFn imageFn footnoteFn roleFn + delimitedInlineFn inlineFn + paraFn headerFn codeBlockFn directiveFn blockCommandFn linkRefFn footnoteRefFn + listItemFn descItemFn blockquoteFn unorderedListFn orderedListFn definitionListFn + blockFn blocksFn blocks1Fn documentFn + metadataContents metadataBlockFn + lookaheadOrderedListMarker lookaheadUnorderedListMarker) -def nameAndArgs (multiline : Option Nat := none) (reportNameAs : String := "identifier") : ParserFn := - nameArgWhitespace multiline >> docIdentFn (reportAs := reportNameAs) >> - nameArgWhitespace multiline >> args (multiline := multiline) +/-- One or more inline elements. With `allowNewlines`, they may continue onto the following lines. -/ +def textLine (allowNewlines := true) : ParserFn := many1Fn (inlineFn { allowNewlines }) -structure InlineCtxt where - allowNewlines := true - -- The minimum indentation of a continuation line for the current paragraph - minIndent : Nat := 0 - -- How many asterisks introduced the current level of boldness? `none` means no bold here. - boldDepth : Option Nat := none - -- How many underscores introduced the current level of emphasis? `none` means no emphasis here. - emphDepth : Option Nat := none +/-- An inline element. -/ +def inline (ctxt : InlineCtxt) : ParserFn := inlineFn ctxt - -- Are we in a link? - inLink : Bool := false +/-- A block element. -/ +def block (ctxt : BlockCtxt) : ParserFn := blockFn ctxt -deriving Inhabited +/-- Zero or more block elements. -/ +def blocks (ctxt : BlockCtxt) : ParserFn := blocksFn ctxt -/- Parsing inlines: - * Inline parsers may not consume trailing whitespace, and must be robust in the face of leading whitespace --/ +/-- One or more block elements. -/ +def blocks1 (ctxt : BlockCtxt) : ParserFn := blocks1Fn ctxt -/-- -A linebreak that isn't a block break (that is, there's non-space content on the next line) --/ -def linebreak (ctxt : InlineCtxt) : ParserFn := - if ctxt.allowNewlines then - nodeFn ``linebreak <| - andthenFn (withInfoSyntaxFn skip.fn (fun info => fakeAtom "line!" info)) <| - nodeFn strLitKind <| - asStringFn (quoted := true) <| - atomicFn (chFn '\n' >> lookaheadFn (manyFn (chFn ' ') >> notFollowedByFn (chFn '\n' <|> blockOpener) "newline")) - else - errorFn "Newlines not allowed here" - -mutual - partial def emphLike (name : SyntaxNodeKind) (char : Char) (what plural : String) (getter : InlineCtxt → Option Nat) (setter : InlineCtxt → Option Nat → InlineCtxt) (ctxt : InlineCtxt) : ParserFn := - nodeFn name <| - withCurrentColumn fun c => - atomicFn (asStringFn (asStringFn (opener ctxt) >> notFollowedByFn (chFn ' ' false <|> chFn '\n' false) "space or newline after opener")) >> - (recoverSkip <| - withCurrentColumn fun c' => - let count := c' - c - manyFn (inline (setter ctxt (some count))) >> - asStringFn (atomicFn (noSpaceBefore >> repFn count (satisfyFn (· == char) s!"'{tok count}'")))) - - where - tok (count : Nat) : String := "".pushn char count - opener (ctxt : InlineCtxt) : ParserFn := - match getter ctxt with - | none => many1Fn (satisfyFn (· == char) s!"any number of {char}s") - | some 1 | some 0 => fun _ s => s.mkError s!"Can't {what} here" - | some d => atMostFn (d - 1) (satisfyFn (· == char) s!"{char}") s!"at most {d} {plural}" - noSpaceBefore : ParserFn := fun c s => - if s.pos == 0 then s - else - let prior := c.get (c.prev s.pos) - if prior.isWhitespace then - s.mkError s!"'{char}' without preceding space" - else s - - partial def emph := emphLike ``emph '_' "emphasize" "underscores" (·.emphDepth) ({· with emphDepth := ·}) - partial def bold := emphLike ``bold '*' "bold" "asterisks" (·.boldDepth) ({· with boldDepth := ·}) - - partial def code : ParserFn := - nodeFn ``code <| - withCurrentColumn fun c => - atomicFn opener >> - ( atomicFn <| - withCurrentColumn fun c' => - let count := c' - c - recoverCode <| - nodeFn strLitKind - (asStringFn (many1Fn <| codeContentsFn (count - 1)) (quoted := true) >> - normFn) >> - closer count) - where - opener : ParserFn := asStringFn (many1Fn (satisfyFn (· == '`') s!"any number of backticks")) - closer (count : Nat) : ParserFn := - asStringFn (atomicFn (repFn count (satisfyFn' (· == '`') s!"expected '{String.ofList (.replicate count '`')}' to close inline code"))) >> - notFollowedByFn (satisfyFn (· == '`') "`") "backtick" - takeBackticksFn : Nat → ParserFn - | 0 => satisfyFn (fun _ => false) - | n+1 => optionalFn (chFn '`' >> takeBackticksFn n) - recoverCode (p : ParserFn) : ParserFn := - recoverFn p fun rctx => - (show ParserFn from fun _ s => s.restore rctx.initialSize rctx.initialPos) >> - atomicFn (nodeFn strLitKind (asStringFn (takeWhileFn (· ≠ '\n')) true) >> ignoreFn (chFn '\n' <|> eoiFn) >> pushMissing) - codeContentsFn (maxCount : Nat) : ParserFn := - atomicFn (asStringFn (satisfyFn (maxCount > 0 && · == '`') >> atMostFn (maxCount - 1) (chFn '`') s!"at most {maxCount} backticks")) <|> - satisfyFn (· != '`') "expected character other than backtick ('`')" - normFn : ParserFn := fun _c s => Id.run <| do - let str := s.stxStack.back - if let .atom info str := str then - if str.startsWith "\" " && str.endsWith " \"" then - let core := str.drop 2 |>.dropEnd 2 - if core.copy.any (· != ' ') then - let str := "\"" ++ core ++ "\"" - let info : SourceInfo := - match info with - | .none => .none - | .synthetic start stop c => .synthetic (⟨start.byteIdx + 1⟩) (⟨stop.byteIdx - 1⟩) c - | .original leading start trailing stop => - .original - { leading with stopPos := ⟨leading.stopPos.byteIdx + 1⟩ } ⟨start.byteIdx + 1⟩ - { trailing with startPos := ⟨trailing.startPos.byteIdx - 1⟩ } ⟨stop.byteIdx - 1⟩ - return s.popSyntax.pushSyntax (.atom info str) - return s - - takeContentsFn (maxCount : Nat) : ParserFn := fun c s => - let i := s.pos - if h : c.atEnd i then s.mkEOIError - else - let ch := c.get' i h - let s := s.next' c i h - let i := s.pos - if ch == '\\' then - if h : c.atEnd i then s.mkEOIError - else - let ch := c.get' i h - let s := s.next' c i h - if ch ∈ ['`', '\\'] then takeContentsFn maxCount c s - else - s.mkError "expected 'n', '\\', or '`'" - else if ch == '`' then - optionalFn (atomicFn (takeBackticksFn maxCount) >> takeContentsFn maxCount) c s - else if ch == '\n' then - s.mkError "unexpected newline" - else takeContentsFn maxCount c s - - partial def math : ParserFn := - atomicFn (nodeFn ``display_math <| strFn "$$" >> code) <|> - atomicFn (nodeFn ``inline_math <| strFn "$" >> code) - - -- Read a prefix of a line of text, stopping at a text-mode special character - partial def text := - nodeFn ``text <| - nodeFn strLitKind <| - asStringFn (transform := unescapeStr) (quoted := true) <| - many1Fn inlineTextChar - - partial def link (ctxt : InlineCtxt) := - nodeFn ``link <| - (atomicFn (notInLink ctxt >> strFn "[" >> notFollowedByFn (chFn '^') "'^'" )) >> - (recoverEol <| - many1Fn (inline {ctxt with inLink := true}) >> - strFn "]" >> linkTarget) - - partial def footnote (ctxt : InlineCtxt) := - nodeFn ``footnote <| - (atomicFn (notInLink ctxt >> strFn "[^" )) >> - (recoverLine <| - nodeFn `str (asStringFn (quoted := true) (many1Fn (satisfyEscFn (fun c => c != ']' && c != '\n') "other than ']' or newline"))) >> - strFn "]") - - partial def linkTarget := ref <|> url - where - notUrlEnd := satisfyEscFn (· ∉ ")\n".toList) "not ')' or newline" >> takeUntilEscFn (· ∈ ")\n".toList) - notRefEnd := satisfyEscFn (· ∉ "]\n".toList) "not ']' or newline" >> takeUntilEscFn (· ∈ "]\n".toList) - ref : ParserFn := - nodeFn ``Lean.Doc.Syntax.ref <| - (atomicFn <| strFn "[") >> - recoverEol (nodeFn strLitKind (asStringFn notRefEnd (quoted := true)) >> strFn "]") - url : ParserFn := - nodeFn ``Lean.Doc.Syntax.url <| - (atomicFn <| strFn "(") >> - recoverEol (nodeFn strLitKind (asStringFn notUrlEnd (quoted := true)) >> strFn ")") - - partial def notInLink (ctxt : InlineCtxt) : ParserFn := fun _ s => - if ctxt.inLink then s.mkError "Already in a link" else s - - partial def image : ParserFn := - nodeFn ``image <| - atomicFn (strFn "![") >> - (recoverSkip <| - nodeFn strLitKind (asStringFn (takeUntilEscFn (· ∈ "]\n".toList)) (quoted := true)) >> - strFn "]" >> - linkTarget) - - partial def role (ctxt : InlineCtxt) : ParserFn := - nodeFn ``role <| - intro >> (bracketed <|> atomicFn nonBracketed) - where - intro := atomicFn (chFn '{') >> recoverBlock (eatSpaces >> nameAndArgs >> eatSpaces >> chFn '}') - bracketed := atomicFn (chFn '[') >> recoverBlock (manyFn (inline ctxt) >> chFn ']') - fakeOpen := mkAtom SourceInfo.none "[" - fakeClose := mkAtom SourceInfo.none "]" - nonBracketed : ParserFn := fun c s => - let s := s.pushSyntax fakeOpen - let s := nodeFn nullKind (delimitedInline ctxt) c s - s.pushSyntax fakeClose - - partial def delimitedInline (ctxt : InlineCtxt) : ParserFn := emph ctxt <|> bold ctxt <|> code <|> math <|> role ctxt <|> image <|> link ctxt <|> footnote ctxt - - partial def inline (ctxt : InlineCtxt) : ParserFn := - text <|> linebreak ctxt <|> delimitedInline ctxt -end +/-- A metadata block. -/ +def metadataBlock : ParserFn := metadataBlockFn /-- -One or more inline elements. With `allowNewlines`, they may continue onto the following lines. +Some number of blank lines followed by zero or more blocks. + +`documentFn` wraps the blocks in a node of its own, while Verso's elaborators consume the sequence +of blocks, so the wrapper is removed here. -/ -def textLine (allowNewlines := true) : ParserFn := many1Fn (inline { allowNewlines }) - -open Lean.Parser Term in -def metadataContents : Parser := - structInstFields (sepByIndent structInstField ", " (allowTrailingSep := true)) - -open Lean.Parser.Term in -def metadataBlock : ParserFn := - nodeFn ``metadata_block <| - opener >> - (adaptUncacheableContextFn addTriplePercent metadataContents.fn) >> - takeWhileFn (·.isWhitespace) >> - closer -where - opener := atomicFn (bolThen (eatSpaces >> strFn "%%%") "%%% (at line beginning)") >> eatSpaces >> ignoreFn (chFn '\n') - closer := bolThen (eatSpaces >> strFn "%%%") "%%% (at line beginning)" >> eatSpaces >> ignoreFn (chFn '\n' <|> eoiFn) - - -- This is necessary in Lean v4.24.0-rc1. Check if it's still necessary in subsequent releases. - addTriplePercent c := { c with tokens := c.tokens.insert "%%%" "%%%" } - - -structure InList where - indentation : Nat - type : OrderedListType ⊕ UnorderedListType -deriving Repr - -structure BlockCtxt where - minIndent : Nat := 0 - maxDirective : Option Nat := none - inLists : List InList := [] -deriving Inhabited, Repr - -def lookaheadOrderedListIndicator (ctxt : BlockCtxt) (p : OrderedListType → Int → ParserFn) : ParserFn := fun c s => - let iniPos := s.pos - let iniSz := s.stxStack.size - let s := (onlyBlockOpeners >> takeWhileFn (· == ' ') >> guardMinColumn ctxt.minIndent) c s - if s.hasError then s.setPos iniPos |>.shrinkStack iniSz - else - let numPos := s.pos - let s := ignoreFn (takeWhile1Fn (·.isDigit) "digits") c s - if s.hasError then {s with pos := iniPos}.shrinkStack iniSz else - let digits := c.extract numPos s.pos - match digits.toNat? with - | none => {s.mkError s!"digits, got '{digits}'" with pos := iniPos} - | some n => - let i := s.pos - if h : c.atEnd i then {s.mkEOIError with pos := iniPos} - else - let (s, next, type) := match c.get' i h with - | '.' => (s.next' c i h, (chFn ' ' <|> chFn '\n'), OrderedListType.numDot) - | ')' => (s.next' c i h, (chFn ' ' <|> chFn '\n'), OrderedListType.parenAfter) - | other => (s.setError {unexpected := s!"unexpected '{other}'", expected := ["'.'", "')'"]}, skipFn, .numDot) - if s.hasError then {s with pos := iniPos} - else - let s := next c s - if s.hasError then {s with pos := iniPos} - else - let leading := c.mkEmptySubstringAt numPos - let trailing := c.mkEmptySubstringAt i - let num := Syntax.mkNumLit digits (info := .original leading numPos trailing i) - p type n c (s.shrinkStack iniSz |>.setPos numPos |>.pushSyntax num) - -def lookaheadUnorderedListIndicator (ctxt : BlockCtxt) (p : UnorderedListType → ParserFn) : ParserFn := fun c s => - let iniPos := s.pos +def document (blockContext : BlockCtxt := {}) : ParserFn := fun c s => let iniSz := s.stxStack.size - let s := (onlyBlockOpeners >> takeWhileFn (· == ' ') >> guardMinColumn ctxt.minIndent) c s - let bulletPos := s.pos - if s.hasError then s.setPos iniPos |>.shrinkStack iniSz - else if h : c.atEnd s.pos then s.mkEOIError.setPos iniPos |>.shrinkStack iniSz - else let (s, type) : (_ × UnorderedListType) := match c.get' s.pos h with - | '*' => (s.next' c s.pos h, .asterisk) - | '-' => (s.next' c s.pos h, .dash) - | '+' => (s.next' c s.pos h, .plus) - | other => (s.setError {expected := ["*", "-", "+"], unexpected := s!"'{other}'"}, .plus) - if s.hasError then s.setPos iniPos + let s := documentFn blockContext c s + if s.hasError || s.stxStack.size != iniSz + 1 then s else - let s := (chFn ' ' <|> chFn '\n') c s - if s.hasError then s.setPos iniPos - else p type c (s.shrinkStack iniSz |>.setPos bulletPos) - -def skipUntilDedent (indent : Nat) : ParserFn := - skipRestOfLine >> - manyFn (chFn ' ' >> takeWhileFn (· == ' ') >> guardColumn (· ≥ indent) s!"indentation at {indent}" >> skipRestOfLine) - -def recoverUnindent (indent : Nat) (p : ParserFn) (finish : ParserFn := skipFn) : ParserFn := recoverFn p (fun _ => ignoreFn (skipUntilDedent indent) >> finish) - -mutual - partial def listItem (ctxt : BlockCtxt) : ParserFn := - nodeFn ``li (bulletFn >> withCurrentColumn fun c => ignoreFn (manyFn (chFn ' ' <|> chFn '\n')) >> blocks1 {ctxt with minIndent := c}) - where - bulletFn := - match ctxt.inLists.head? with - | none => fun _ s => s.mkError "not in a list" - | some ⟨col, .inr type⟩ => - atomicFn <| - takeWhileFn (· == ' ') >> - guardColumn (· == col) s!"indentation at {col}" >> - unorderedListIndicator type >> ignoreFn (lookaheadFn (chFn ' ' <|> chFn '\n')) - | some ⟨col, .inl type⟩ => - atomicFn <| - takeWhileFn (· == ' ') >> - guardColumn (· == col) s!"indentation at {col}" >> - orderedListIndicator type >> ignoreFn (lookaheadFn (chFn ' ' <|> chFn '\n')) - - partial def descItem (ctxt : BlockCtxt) : ParserFn := - nodeFn ``desc <| - colonFn >> - withCurrentColumn fun c => textLine >> ignoreFn (manyFn blankLine) >> - fakeAtom "=>" >> - takeWhileFn (· == ' ') >> - recoverSkip (guardColumn (· ≥ c) s!"indentation at least {c}" >> - blocks1 { ctxt with minIndent := c}) >> - ignoreFn (manyFn blankLine) - where - colonFn := atomicFn <| - takeWhileFn (· == ' ') >> - guardColumn (· == ctxt.minIndent) s!"indentation at {ctxt.minIndent}" >> - asStringFn (chFn ':' false) >> ignoreFn (lookaheadFn (chFn ' ')) - - partial def blockquote (ctxt : BlockCtxt) : ParserFn := - atomicFn <| nodeFn ``blockquote <| - takeWhileFn (· == ' ') >> guardMinColumn ctxt.minIndent >> chFn '>' >> - withCurrentColumn fun c => blocks { ctxt with minIndent := c } - - partial def unorderedList (ctxt : BlockCtxt) : ParserFn := - nodeFn ``ul <| - lookaheadUnorderedListIndicator ctxt fun type => - withCurrentColumn fun c => - fakeAtom "ul{" >> - many1Fn (listItem {ctxt with minIndent := c + 1 , inLists := ⟨c, .inr type⟩ :: ctxt.inLists}) >> - fakeAtom "}" - - partial def orderedList (ctxt : BlockCtxt) : ParserFn := - nodeFn ``ol <| - fakeAtom "ol(" >> - lookaheadOrderedListIndicator ctxt fun type _start => -- TODO? Validate list numbering? - withCurrentColumn fun c => - fakeAtom ")" >> fakeAtom "{" >> - many1Fn (listItem {ctxt with minIndent := c + 1 , inLists := ⟨c, .inl type⟩ :: ctxt.inLists}) >> - fakeAtom "}" - - partial def definitionList (ctxt : BlockCtxt) : ParserFn := - nodeFn ``dl <| - atomicFn (onlyBlockOpeners >> takeWhileFn (· == ' ') >> ignoreFn (lookaheadFn (chFn ':' >> chFn ' ')) >> guardMinColumn ctxt.minIndent) >> - withInfoSyntaxFn skip.fn (fun info => fakeAtom "dl{" info) >> - withCurrentColumn (fun c => many1Fn (descItem {ctxt with minIndent := c})) >> - withInfoSyntaxFn skip.fn (fun info => fakeAtom "}" info) - - partial def para (ctxt : BlockCtxt) : ParserFn := - nodeFn ``para <| - atomicFn (takeWhileFn (· == ' ') >> notFollowedByFn blockOpener "block opener" >> guardMinColumn ctxt.minIndent) >> - withInfoSyntaxFn skip.fn (fun info => fakeAtom "para{" (info := info)) >> - textLine >> - withInfoSyntaxFn skip.fn (fun info => fakeAtom "}" (info := info)) - - partial def header (ctxt : BlockCtxt) : ParserFn := - nodeFn ``header <| - guardMinColumn ctxt.minIndent >> - atomicFn (bol >> - withCurrentColumn fun c => - withInfoSyntaxFn (many1Fn (skipChFn '#')) (fun info => fakeAtom "header(" (info := info)) >> - withCurrentColumn fun c' => - skipChFn ' ' >> takeWhileFn (· == ' ') >> lookaheadFn (satisfyFn (· != '\n') "non-newline") >> - (show ParserFn from fun _ s => s.pushSyntax <| Syntax.mkNumLit (toString <| c' - c - 1)) >> - fakeAtom ")") >> - fakeAtom "{" >> - textLine (allowNewlines := false) >> - fakeAtom "}" - - partial def codeBlock (ctxt : BlockCtxt) : ParserFn := - nodeFn ``codeblock <| - -- Opener - leaves indent info and open token on the stack - atomicFn (takeWhileFn (· == ' ') >> guardMinColumn ctxt.minIndent >> pushColumn >> asStringFn (atLeastFn 3 (skipChFn '`'))) >> - withIndentColumn fun c => - recoverUnindent c <| - withCurrentColumn fun c' => - let fenceWidth := c' - c - takeWhileFn (· == ' ') >> - optionalFn nameAndArgs >> - asStringFn (satisfyFn (· == '\n') "newline") >> - nodeFn strLitKind (asStringFn (manyFn (atomicFn blankLine <|> codeFrom c fenceWidth)) (transform := deIndent c) (quoted := true)) >> - closeFence c fenceWidth - where - withIndentColumn (p : Nat → ParserFn) : ParserFn := fun c s => - let colStx := s.stxStack.get! (s.stxStack.size - 2) - match colStx with - | .node _ `column #[.atom _ col] => - if let some colNat := col.toNat? then - let opener := s.stxStack.get! (s.stxStack.size - 1) - p colNat c (s.popSyntax.popSyntax.pushSyntax opener) - else - s.mkError s!"Internal error - not a Nat {col}" - | other => s.mkError s!"Internal error - not a column node {other}" - - deIndent (n : Nat) (str : String) : String := Id.run do - let str := if str != "" && str.back == '\n' then str.dropEnd 1 |>.copy else str - let mut out := "" - for line in str.splitOn "\n" do - out := out ++ line.drop n ++ "\n" - out - - codeFrom (col width : Nat) := - atomicFn (bol >> takeWhileFn (· == ' ') >> guardMinColumn col >> - notFollowedByFn (atLeastFn width (skipChFn '`')) "ending fence") >> - manyFn (satisfyFn (· != '\n') "non-newline") >> satisfyFn (· == '\n') "newline" - - closeFence (col width : Nat) := - bol >> takeWhileFn (· == ' ') >> guardColumn (· == col) s!"column {col}" >> - atomicFn (asStringFn (repFn width (skipChFn '`'))) >> - notFollowedByFn (skipChFn '`') "extra `" >> - takeWhileFn (· == ' ') >> (satisfyFn (· == '\n') "newline" <|> eoiFn) - - partial def directive (ctxt : BlockCtxt) : ParserFn := - nodeFn ``directive <| - -- Opener - leaves indent info and open token on the stack - atomicFn - (eatSpaces >> guardMinColumn ctxt.minIndent >> - asStringFn (atLeastFn 3 (skipChFn ':')) >> - guardOpenerSize >> - eatSpaces >> - recoverEolWith #[.missing, .node .none nullKind #[]] (nameAndArgs (reportNameAs := "directive name (identifier)") >> - satisfyFn (· == '\n') "newline")) >> - fakeAtom "\n" >> - ignoreFn (manyFn blankLine) >> - (withFencePos 3 fun ⟨l, col⟩ => - withFenceSize 3 fun fenceWidth => - blocks {ctxt with minIndent := col, maxDirective := fenceWidth} >> - recoverHereWith #[.missing] - (closeFence l fenceWidth >> - withFence 0 fun info _ c s => - if (c.fileMap.toPosition info.getPos!).column != col then - s.mkErrorAt s!"closing '{String.ofList <| List.replicate fenceWidth ':'}' from directive on line {l} at column {col}, but it's at column {(c.fileMap.toPosition info.getPos!).column}" info.getPos! - else - s)) - - where - withFence (atDepth : Nat) (p : SourceInfo → String → ParserFn) : ParserFn := fun c s => - match s.stxStack.get! (s.stxStack.size - (atDepth + 1)) with - | .atom info str => - if str.all (· == ':') then - p info str c s - else - s.mkError s!"Internal error - index {atDepth} wasn't the directive fence - it was the atom {str}" - | .missing => s.pushSyntax .missing - | stx => - s.mkError s!"Internal error - index {atDepth} wasn't the directive fence - it was {stx} in {s.stxStack.back}, {s.stxStack.pop.back}, {s.stxStack.pop.pop.back}, {s.stxStack.pop.pop.pop.back}" - - withFenceSize (atDepth : Nat) (p : Nat → ParserFn) : ParserFn := - withFence atDepth fun _ str => p str.length - - withFencePos (atDepth : Nat) (p : Position → ParserFn) : ParserFn := - withFence atDepth fun info _ c s => p (c.fileMap.toPosition info.getPos!) c s - - withIndentColumn (atDepth : Nat) (p : Nat → ParserFn) : ParserFn := - withFence atDepth fun info _ c s => - let col := c.fileMap.toPosition info.getPos! |>.column - p col c s - - guardOpenerSize : ParserFn := withFenceSize 0 fun x => - if let some m := ctxt.maxDirective then - if x < m then skipFn else fun _ s => s.mkError "Too many ':'s here" - else skipFn - - closeFence (line width : Nat) := - let str := String.ofList (.replicate width ':') - bolThen (description := s!"closing '{str}' for directive from line {line}") - (eatSpaces >> - asStringFn (strFn str) >> notFollowedByFn (chFn ':') "':'" >> - eatSpaces >> - (ignoreFn <| atomicFn (satisfyFn (· == '\n') "newline") <|> eoiFn)) - - -- This low-level definition is to get exactly the right amount of lookahead - -- together with column tracking - partial def block_command (ctxt : BlockCtxt) : ParserFn := fun c s => - let iniPos := s.pos - let iniSz := s.stxStack.size - let restorePosOnErr : ParserState → ParserState - | ⟨stack, lhsPrec, _, cache, some msg, errs⟩ => ⟨stack, lhsPrec, iniPos, cache, some msg, errs⟩ - | other => other - let s := eatSpaces c s - if s.hasError then restorePosOnErr s - else - let s := (intro >> eatSpaces >> ignoreFn (satisfyFn (· == '\n') "newline" <|> eoiFn)) c s - if s.hasError then restorePosOnErr s - else - s.mkNode ``Lean.Doc.Syntax.command iniSz - where - eatSpaces := takeWhileFn (· == ' ') - intro := guardMinColumn (ctxt.minIndent) >> atomicFn (chFn '{') >> nameAndArgs >> nameArgWhitespace none >> chFn '}' - - partial def linkRef (c : BlockCtxt) : ParserFn := - nodeFn ``link_ref <| - atomicFn (ignoreFn (bol >> eatSpaces >> guardMinColumn c.minIndent) >> chFn '[' >> nodeFn strLitKind (asStringFn (quoted := true) (nameStart >> manyFn (satisfyEscFn (· != ']') "not ']'"))) >> strFn "]:") >> - eatSpaces >> - nodeFn strLitKind (asStringFn (quoted := true) (takeWhileFn (· != '\n'))) >> - ignoreFn (satisfyFn (· == '\n') "newline" <|> eoiFn) - where nameStart := satisfyEscFn (fun c => c != ']' && c != '^') "not ']' or '^'" - - partial def footnoteRef (c : BlockCtxt) : ParserFn := - nodeFn ``footnote_ref <| - atomicFn (ignoreFn (bol >> eatSpaces >> guardMinColumn c.minIndent) >> strFn "[^" >> nodeFn strLitKind (asStringFn (quoted := true) (many1Fn (satisfyEscFn (· != ']') "not ']'"))) >> strFn "]:") >> - eatSpaces >> - notFollowedByFn blockOpener "block opener" >> guardMinColumn c.minIndent >> textLine - - partial def block (c : BlockCtxt) : ParserFn := - block_command c <|> unorderedList c <|> orderedList c <|> definitionList c <|> header c <|> codeBlock c <|> directive c <|> blockquote c <|> linkRef c <|> footnoteRef c <|> para c <|> metadataBlock - - partial def blocks (c : BlockCtxt) : ParserFn := sepByFn true (block c) (ignoreFn (manyFn blankLine)) - - partial def blocks1 (c : BlockCtxt) : ParserFn := sepBy1Fn true (block c) (ignoreFn (manyFn blankLine)) - - partial def document (blockContext : BlockCtxt := {}) : ParserFn := ignoreFn (manyFn blankLine) >> blocks blockContext -end + let stx := s.stxStack.back + s.popSyntax.pushSyntax (stx.getArg 0) end Verso.Parser diff --git a/verso/src/verso/Verso/SyntaxUtils.lean b/verso/src/verso/Verso/SyntaxUtils.lean index e77da035e..bfac2f2ee 100644 --- a/verso/src/verso/Verso/SyntaxUtils.lean +++ b/verso/src/verso/Verso/SyntaxUtils.lean @@ -6,6 +6,7 @@ Author: David Thrane Christiansen module public import Lean.Parser.Types public import Std.Data.TreeMap +public import Verso.Literal public meta import Verso.Instances import Verso.Instances import Verso.Method @@ -154,8 +155,8 @@ public def strLitInputContext [Monad m] [MonadFileMap m] [MonadError m] (str : S Given a string literal, constructs a Lean string that can be parsed by the Lean parser, yielding correct source positions for items in the string literal. -/ -public def parserInputString [Monad m] [MonadFileMap m] - (str : TSyntax `str) : +public def parserInputString [Monad m] [MonadFileMap m] [Verso.VersoLiteral k] + (str : TSyntax k) : m String := do let text ← getFileMap let preString := (0 : String.Pos.Raw).extract text.source (str.raw.getPos?.getD 0) @@ -172,7 +173,7 @@ public def parserInputString [Monad m] [MonadFileMap m] let strOriginal? : Option String := do let ⟨start, stop⟩ ← str.raw.getRange? start.extract text.source stop - code := code ++ strOriginal?.getD str.getString + code := code ++ strOriginal?.getD (Verso.decode str) return code @@ -335,14 +336,15 @@ actual string contents. When the literal's source text differs from its contents sequences, the decoded contents are parsed and the resulting positions are mapped back to the source. -/ -public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] (p : ParserFn) (input : StrLit) : m Syntax := do +public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Verso.VersoLiteral k] (p : ParserFn) (input : TSyntax k) : m Syntax := do let text ← getFileMap + let inputText := Verso.decode input if let some startPos := input.raw.getPos? then let endPos := input.raw.getTailPos?.getD startPos let stopPos := if endPos > text.source.rawEndPos then text.source.rawEndPos else endPos - if startPos.extract text.source stopPos != input.getString then + if startPos.extract text.source stopPos != inputText then let (decoded, posMap) := decodeContentWithMap text.source startPos stopPos - if decoded == input.getString then + if decoded == inputText then return ← parseDecoded p decoded (mapDecodedPos posMap) text -- The contents appear verbatim in the source: parse it directly for exact source positions. let (ictx, startPos) ← strLitInputContext input.raw (← getFileName) @@ -382,7 +384,7 @@ Parses an original string literal as part of a syntax category. The provided string literal is used only for source positions; the `FileMap` is used to acquire the actual string contents. -/ -public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] (catName : Name) (input : StrLit) : m Syntax := +public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Verso.VersoLiteral k] (catName : Name) (input : TSyntax k) : m Syntax := parseStrLitWith (andthenFn whitespace (categoryParserFnImpl catName)) input /-- diff --git a/verso/test-projects/textbook/DemoTextbook/Meta/Lean.lean b/verso/test-projects/textbook/DemoTextbook/Meta/Lean.lean index ef6ccc517..dabba0f3f 100644 --- a/verso/test-projects/textbook/DemoTextbook/Meta/Lean.lean +++ b/verso/test-projects/textbook/DemoTextbook/Meta/Lean.lean @@ -38,7 +38,7 @@ Lean code that is saved to the examples file. def savedLean : CodeBlockExpanderOf InlineLean.LeanBlockConfig | args, code => do let underlying ← InlineLean.lean args code - ``(Block.other (Block.savedLean $(quote (← getFileName)) $(quote (code.getString))) #[$underlying]) + ``(Block.other (Block.savedLean $(quote (← getFileName)) $(quote (code.getVersoCodeBlock))) #[$underlying]) /-- An import of some other module, to be located in the saved code. Not rendered. @@ -46,7 +46,7 @@ An import of some other module, to be located in the saved code. Not rendered. @[code_block] def savedImport : CodeBlockExpanderOf Unit | (), code => do - ``(Block.other (Block.savedImport $(quote (← getFileName)) $(quote (code.getString))) #[]) + ``(Block.other (Block.savedImport $(quote (← getFileName)) $(quote (code.getVersoCodeBlock))) #[]) /-- Comments to be added as module docstrings to the examples file. @@ -54,6 +54,6 @@ Comments to be added as module docstrings to the examples file. @[code_block] def savedComment : CodeBlockExpanderOf Unit | (), code => do - let str := code.getString.trimAsciiEnd.copy + let str := code.getVersoCodeBlock.trimAsciiEnd.copy let comment := s!"/-!\n{str}\n-/" ``(Block.other (Block.savedLean $(quote (← getFileName)) $(quote comment)) #[]) diff --git a/verso/test-projects/tutorial-test/TutorialExample/Data.lean b/verso/test-projects/tutorial-test/TutorialExample/Data.lean index af2a26eac..35a45879c 100644 --- a/verso/test-projects/tutorial-test/TutorialExample/Data.lean +++ b/verso/test-projects/tutorial-test/TutorialExample/Data.lean @@ -323,7 +323,7 @@ example : SubList [1, 3] [1, 2, 3, 4] := .cons₂ (.cons (.cons₂ .nil)) ``` -#. Advanced Topics +# Advanced Topics ## Mutually Inductive Types diff --git a/verso/test-projects/website/DemoSite/About.lean b/verso/test-projects/website/DemoSite/About.lean index b840df775..e4cdc43f2 100644 --- a/verso/test-projects/website/DemoSite/About.lean +++ b/verso/test-projects/website/DemoSite/About.lean @@ -54,21 +54,28 @@ def galleryImpl : DirectiveExpanderOf Unit let #[stx] := stxs | logErrorAt (mkNullNode stxs) "Expected one block" return (← `(sorry)) - let `(block| dl{ $item*}) := stx + let some dl := Lean.Doc.DescListView.of stx | throwErrorAt stx "Expected definition list" - let items ← item.mapM getItem + let items ← dl.items.mapM getItem ``(Block.other (Blog.BlockExt.component $(quote `gallery) Json.null) #[$(items),*]) where - getItem : TSyntax `desc_item → DocElabM Term - | `(desc_item|: $inls* => $desc $descs*) => do - let #[inl] := inls.filter (fun - | `(inline|$s:str) => s.getString.any (not ∘ Char.isWhitespace) - | _ => true) - | throwErrorAt (mkNullNode inls) "Expected one inline" - let `(inline|image($alt)($url)) := inl - | throwErrorAt inl "Expected an image" - `(Block.other (.component $(quote `image) (.arr #[$alt, $url])) #[$(← elabBlock desc), $(← descs.mapM elabBlock),*]) - | stx => throwErrorAt stx "Expected an image and description, got {stx}" + getItem (item : Lean.Doc.DescItemView) : DocElabM Term := do + let #[inl] := item.term.filter (fun i => + match Lean.Doc.TextView.of i with + | some t => t.getVersoText.any (not ∘ Char.isWhitespace) + | none => true) + | throwErrorAt (mkNullNode (item.term.map (·.raw))) "Expected one inline" + let some img := Lean.Doc.ImageView.of inl + | throwErrorAt inl "Expected an image" + let .url _ _ url _ := img.target + | throwErrorAt inl "Expected an image with a URL" + let some desc := item.desc[0]? + | throwErrorAt item.stx "Expected a description" + let descs := item.desc.extract 1 item.desc.size + `(Block.other + (.component $(quote `image) + (.arr #[$(quote img.getAlt), $(quote url.getVersoLinkUrl)])) + #[$(← elabBlock desc), $(← descs.mapM elabBlock),*]) block_component +directive button' (onclick : String) where toHtml id _ _ goB contents := do From d3f04e6ba7bece1967d986a637e76b81500c8b5b Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 15:52:14 +0200 Subject: [PATCH 05/15] refactor: verso names the parser's syntax as briefly as it can The views, the content tokens and the literal class are named through `open`, so a signature reads `TSyntaxArray ``inline` rather than spelling out a namespace. Each file opens the names it uses: `Lean.Doc` as a whole brings in the document types, whose `Inline` and `Block` collide with Verso's own inside a quotation. A view is destructured by field, so a pattern says which part of an element it takes. The remaining full names are the ones that resolution does not reach: an `inline_to_string` key, which the attribute reads literally, and the syntax kinds a linter compares against, which are `Name` literals. `Verso.Parser` no longer re-exports the productions it used to define, which named the same things as `Lean.Doc.Parser` and shadowed them. Every element that Verso reads is read through a view: the language server's delimiter pairing, directive resizing and semantic tokens, and the roles that reach for their own syntax. `Lean.Doc.Syntax` is gone from Verso, including the `open` that `#doc` used to add to a document's scope. `warnLongLines` takes a code block and measures each line token, which the parser has already separated, rather than recovering the lines from the file. Its note and hint are built only for a line that is too long. A test covers the limit, a line that exceeds it, and the option that turns it off. `DocDef.defSite` is the name it defines. --- verso/doc/UsersGuide/Markup.lean | 35 ++++--- verso/src/tests/Tests/Linters.lean | 56 +++++++++++ verso/src/verso-blog/VersoBlog.lean | 16 +-- .../src/verso-illuminate/VersoIlluminate.lean | 3 +- .../verso-manual/VersoManual/Docstring.lean | 6 +- .../VersoManual/Docstring/Progress.lean | 29 +++--- .../src/verso-manual/VersoManual/Imports.lean | 1 - .../verso-manual/VersoManual/InlineLean.lean | 35 +++---- .../VersoManual/InlineLean/IO.lean | 19 ++-- .../VersoManual/InlineLean/IO/Context.lean | 13 +-- .../VersoManual/InlineLean/LongLines.lean | 45 +++++---- .../VersoManual/InlineLean/Option.lean | 4 +- .../src/verso-manual/VersoManual/Linters.lean | 11 ++- .../verso-manual/VersoManual/Literate.lean | 1 - verso/src/verso-manual/VersoManual/Table.lean | 7 +- verso/src/verso/Verso/Code/External.lean | 21 ++-- verso/src/verso/Verso/Doc/ArgParse.lean | 4 +- verso/src/verso/Verso/Doc/Concrete.lean | 4 +- verso/src/verso/Verso/Doc/Elab.lean | 27 ++--- verso/src/verso/Verso/Doc/Elab/Basic.lean | 4 +- verso/src/verso/Verso/Doc/Elab/Block.lean | 7 +- verso/src/verso/Verso/Doc/Elab/Inline.lean | 5 +- verso/src/verso/Verso/Doc/Elab/Monad.lean | 46 +++++---- verso/src/verso/Verso/Doc/Helpers.lean | 12 ++- verso/src/verso/Verso/Doc/Lsp.lean | 99 ++++++++++--------- verso/src/verso/Verso/ExpectString.lean | 9 +- verso/src/verso/Verso/Linters.lean | 10 +- verso/src/verso/Verso/Parser.lean | 18 +--- verso/src/verso/Verso/SyntaxUtils.lean | 11 ++- .../test-projects/website/DemoSite/About.lean | 12 +-- 30 files changed, 318 insertions(+), 252 deletions(-) diff --git a/verso/doc/UsersGuide/Markup.lean b/verso/doc/UsersGuide/Markup.lean index 46ff01a5f..7b99bbb4c 100644 --- a/verso/doc/UsersGuide/Markup.lean +++ b/verso/doc/UsersGuide/Markup.lean @@ -12,7 +12,7 @@ open Verso Genre Manual section open Lean -open Lean.Doc.Syntax +open Lean.Doc (ArgValView ArgView BlockView CodeBlockView DescItemView InlineView LinebreakView OrderedListItemView ParaView UnorderedListItemView mkVersoTextFromRef) variable [Monad m] [MonadError m] [MonadQuotation m] @@ -21,8 +21,8 @@ def newlinesToSpace (inls : TSyntaxArray ``Lean.Doc.Parser.inline) : let mut out := #[] for h : i in [:inls.size] do let inl := inls[i] - if (Lean.Doc.LinebreakView.of inl).isSome && i < inls.size - 1 then - out := out.push (← `(Lean.Doc.Parser.inline| $(← Lean.Doc.mkVersoTextFromRef " "):versoText)) + if (LinebreakView.of inl).isSome && i < inls.size - 1 then + out := out.push (← `(Lean.Doc.Parser.inline| $(← mkVersoTextFromRef " "):versoText)) else out := out.push inl return out @@ -38,8 +38,6 @@ def asCode (s : String) : String := (lw - s.length).fold (init := s) fun _ _ => (" " ++ ·) (lines.mapIdx fun i l => (s!"{toString (i + 1) |> pad}|{l}⏎\n")) |> String.join |>.trimAsciiEnd |>.copy -open Lean.Doc - mutual partial def previewInline (v : InlineView) : m Std.Format := do match v with @@ -66,14 +64,14 @@ mutual let txt ← newlinesToSpace l.content let contents ← txt.toList.mapM (preview ∘ TSyntax.raw) match l.target with - | .url _ _ url _ => + | .url (url := url) .. => pure <| .group <| .nest 2 (s!"" ++ .line ++ .fill (.join contents)) ++ .line ++ "" - | .ref _ _ name _ => + | .ref (name := name) .. => pure <| .fill <| s!"" ++ .join contents ++ "" | .image i => let src := match i.target with - | .url _ _ url _ => url.getVersoLinkUrl - | .ref _ _ name _ => s!"value of «{name.getVersoRefName}»" + | .url (url := url) .. => url.getVersoLinkUrl + | .ref (name := name) .. => s!"value of «{name.getVersoRefName}»" pure <| .group <| .nest 2 <| "" | .code c => pure s!"{c.getVersoCode.quote}" | .footnote f => pure s!"" @@ -290,6 +288,7 @@ r#" section open Lean +open Lean.Doc (CodeBlockView InlineView ParaView) open ArgParse open Doc.Elab @@ -306,26 +305,26 @@ private def withNl (s : String) : String := if s.endsWith "\n" then s else s.pus open Verso Doc Elab in open Lean Elab in +open Lean.Doc (CodeBlockView InlineView ParaView) in open Verso.Parser in -open Lean.Doc.Syntax in @[directive] def markupPreview : DirectiveExpanderOf MarkupPreviewConfig | {title}, contents => do let #[blk1, blk2] := contents.filter nonempty | throwError "Expected precisely two code blocks, got {contents.filter nonempty}" - let some cb1 := Lean.Doc.CodeBlockView.of blk1 + let some cb1 := CodeBlockView.of blk1 | throwErrorAt blk1 "Expected anonymous code block" - let some cb2 := Lean.Doc.CodeBlockView.of blk2 + let some cb2 := CodeBlockView.of blk2 | throwErrorAt blk2 "Expected anonymous code block" let (contents, expected) := (cb1.content, cb2.content) - let stx ← blocks {} |>.parseString contents.getVersoCodeBlock.trimAsciiEnd.copy + let stx ← blocksFn {} |>.parseString contents.getVersoCodeBlock.trimAsciiEnd.copy let p ← preview stx let p := p.pretty (width := 35) withOptions (verso.code.warnLineLength.set · 35) do - warnLongLines none contents.raw - warnLongLines none expected.raw + warnLongLines contents + warnLongLines expected unless eq expected.getVersoCodeBlock p do let hint ← MessageData.hint m!"Replace with actual output" #[withNl p] (ref? := expected) @@ -344,12 +343,12 @@ where -- A paragraph of only whitespace separates the two code blocks that the directive expects. nonemptyI (inl : TSyntax ``Lean.Doc.Parser.inline) : Bool := - match Lean.Doc.InlineView.of inl with + match InlineView.of inl with | some (.text t) => !t.getVersoText.isEmpty | some (.linebreak _) => false | _ => true nonempty (blk : TSyntax ``Lean.Doc.Parser.block) : Bool := - match Lean.Doc.ParaView.of blk with + match ParaView.of blk with | some p => p.content.any nonemptyI | none => true @@ -359,7 +358,7 @@ open Verso.Parser in def markupPreviewPre : CodeBlockExpanderOf MarkupPreviewConfig | {title}, contents => do - let stx ← blocks {} |>.parseString contents.getVersoCodeBlock + let stx ← blocksFn {} |>.parseString contents.getVersoCodeBlock let p ← preview stx let p := p.pretty (width := 35) diff --git a/verso/src/tests/Tests/Linters.lean b/verso/src/tests/Tests/Linters.lean index 14eb2115c..ccedfb50a 100644 --- a/verso/src/tests/Tests/Linters.lean +++ b/verso/src/tests/Tests/Linters.lean @@ -296,3 +296,59 @@ set_option linter.verso.manual.headerTags true in Some text. ::::::: + +/-! +# `verso.code.warnLineLength` (default: 60, `Manual` genre only) + +The limit applies to a line's own width. A code block's indentation is the leading whitespace of +each of its line tokens, so indenting a block does not make its lines count as longer. +-/ + +/-! +A line within the limit produces no warning. +-/ +#guard_msgs in +set_option verso.code.warnLineLength 20 in +#docs (Verso.Genre.Manual) shortLines "Short lines" := +::::::: + +```Genre.Manual.InlineLean.lean +def a := 1 +``` + +::::::: + +/-! +A line past the limit is reported, at the line itself. +-/ +/-- +warning: Line 1 is too long (33 columns exceeds 20). + +Note: Example code is shown on mobile devices and other narrow contexts. Long lines are likely to be truncated in the rendered output. + +Hint: The limit of this linter can be changed with the option `verso.code.warnLineLength`. This linter can be disabled by setting this option to 0. +-/ +#guard_msgs in +set_option verso.code.warnLineLength 20 in +#docs (Verso.Genre.Manual) longLines "Long lines" := +::::::: + +```Genre.Manual.InlineLean.lean +def someRatherLongNameIndeed := 1 +``` + +::::::: + +/-! +Setting the option to 0 disables the linter. +-/ +#guard_msgs in +set_option verso.code.warnLineLength 0 in +#docs (Verso.Genre.Manual) lineLengthOff "Line length off" := +::::::: + +```Genre.Manual.InlineLean.lean +def anotherRatherLongNameHere := 1 +``` + +::::::: diff --git a/verso/src/verso-blog/VersoBlog.lean b/verso/src/verso-blog/VersoBlog.lean index 06dd1b0e1..d6583ce2c 100644 --- a/verso/src/verso-blog/VersoBlog.lean +++ b/verso/src/verso-blog/VersoBlog.lean @@ -34,7 +34,7 @@ open Verso.Output Html namespace Verso.Genre.Blog -open Lean.Doc.Syntax +open Lean.Doc (CodeView RoleView) open Verso ArgParse Doc Elab open Lean Elab open Verso.SyntaxUtils (parserInputString strLitInputContext) @@ -373,7 +373,7 @@ meta instance : FromArgs NoArgs m where @[role] meta def leanKw : RoleExpanderOf NoArgs | ⟨⟩, #[arg] => do - let some ⟨_, _, kw, _⟩ := Lean.Doc.CodeView.of arg + let some { content := kw, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the keyword" let hl : SubVerso.Highlighting.Highlighted := .token ⟨.keyword none none none, kw.getVersoCode⟩ ``(Inline.other (Blog.InlineExt.customHighlight $(quote hl)) #[Inline.code $(quote kw.getVersoCode)]) @@ -396,7 +396,7 @@ meta instance : FromArgs LeanTermArgs DocElabM where @[role] meta def leanTerm : RoleExpanderOf LeanTermArgs | {project, showProofStates}, #[arg] => withTraceNode `Elab.Verso.block.lean (fun _ => pure m!"leanTerm") <| do - let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let exampleName := name.getVersoCode.toName let projectExamples ← getSubproject project @@ -611,7 +611,7 @@ private meta def leanInlineImpl : RoleExpanderOf LeanInlineConfig | config, elts => withTraceNode `Elab.Verso.block.lean (fun _ => pure m!"lean block") <| do let #[code] := elts | throwError "Expected precisely one code element" - let some ⟨_, _, str, _⟩ := Lean.Doc.CodeView.of code + let some { content := str, .. } := CodeView.of code | throwErrorAt code "Expected an inline code element" let x := config.exampleContext let (commandState, _) ← match exampleContextExt.getState (← getEnv) |>.contexts.find? x.getId with @@ -677,9 +677,9 @@ private meta def leanInlineImpl : RoleExpanderOf LeanInlineConfig pushInfoTree tree - if let `(inline|role{%$s $f $_*}%$e[$_*]) ← getRef then - Hover.addCustomHover (mkNullNode #[s, e]) type - Hover.addCustomHover f type + if let some v := RoleView.of ⟨← getRef⟩ then + Hover.addCustomHover (mkNullNode #[v.braceOpen, v.braceClose]) type + Hover.addCustomHover v.name type for msg in newMsgs.toArray do logMessage {msg with @@ -815,7 +815,7 @@ elab "define_lexed_text" blockName:ident " ← " lexerName:ident : command => do elabCommand <| ← `(@[role] def $(mkIdent <| blockName.getId ++ `role) : Doc.Elab.RoleExpanderOf NoArgs | ⟨⟩, #[inl] => do - let some ⟨_, _, str, _⟩ := Lean.Doc.CodeView.of inl + let some { content := str, .. } := CodeView.of inl | throwErrorAt inl "Expected code" let out ← Verso.Genre.Blog.LexedText.highlight $(mkIdentFrom lexerName lexer) str.getVersoCode ``(Inline.other (Blog.InlineExt.lexedText $$(quote out)) #[]) diff --git a/verso/src/verso-illuminate/VersoIlluminate.lean b/verso/src/verso-illuminate/VersoIlluminate.lean index f339dc4b8..76e517004 100644 --- a/verso/src/verso-illuminate/VersoIlluminate.lean +++ b/verso/src/verso-illuminate/VersoIlluminate.lean @@ -17,6 +17,7 @@ open Verso ArgParse Doc Elab open Verso.SyntaxUtils (parseStrLitAsCategory) open Illuminate +open Lean.Doc namespace Verso.ArgParse.ValDesc @@ -145,7 +146,7 @@ section variables for the Manual genre). It defaults to the identity. Genre-specific code-block expanders call this to do the shared evaluation work and then emit their own `GenreDiagram.diagramBlock` term. -/ -public def elabAndStoreDiagram [Verso.VersoLiteral k] (str : TSyntax k) +public def elabAndStoreDiagram [VersoLiteral k] (str : TSyntax k) (scope : {α : Type} → TermElabM α → TermElabM α := fun act => act) : DocElabM (String × Float) := do let stx ← parseStrLitAsCategory `term str diff --git a/verso/src/verso-manual/VersoManual/Docstring.lean b/verso/src/verso-manual/VersoManual/Docstring.lean index ab4883e9a..9e986acb7 100644 --- a/verso/src/verso-manual/VersoManual/Docstring.lean +++ b/verso/src/verso-manual/VersoManual/Docstring.lean @@ -55,7 +55,7 @@ open Verso.Doc.Elab.PartElabM open Verso.Code open Verso.ArgParse open Verso.Code.Highlighted.WebAssets -open Lean.Doc.Syntax +open Lean.Doc (CodeView) open SubVerso.Highlighting @@ -1833,7 +1833,7 @@ meta def tacticInline : RoleExpanderOf TacticInlineOptions | {«show»}, inlines => do let #[arg] := inlines | throwError "Expected exactly one argument" - let some ⟨_, _, tac, _⟩ := Lean.Doc.CodeView.of arg + let some { content := tac, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the tactic name" let tacTok := tac.getVersoCode let tacName := tac.getVersoCode.toName @@ -1990,7 +1990,7 @@ meta def convInline : RoleExpander | _args, inlines => do let #[arg] := inlines | throwError "Expected exactly one argument" - let some ⟨_, _, convTac, _⟩ := Lean.Doc.CodeView.of arg + let some { content := convTac, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the conv tactic name" let convTacName := convTac.getVersoCode.toName let convTacDoc ← getConvTactic (.inr (mkIdent convTacName)) none diff --git a/verso/src/verso-manual/VersoManual/Docstring/Progress.lean b/verso/src/verso-manual/VersoManual/Docstring/Progress.lean index 3f7b65649..9cd0919b0 100644 --- a/verso/src/verso-manual/VersoManual/Docstring/Progress.lean +++ b/verso/src/verso-manual/VersoManual/Docstring/Progress.lean @@ -18,7 +18,7 @@ namespace Verso.Genre.Manual open Lean open Verso.Output.Html open Verso.Doc.Elab -open Lean.Doc.Syntax +open Lean.Doc (CodeBlockView) /-- A progress tracker that shows how many symbols are documented. @@ -68,20 +68,19 @@ public meta def progress : DirectiveExpanderOf Unit let mut namespaces : NameSet := {} let mut exceptions : NameSet := {} for block in blocks do - match block with - | `(block|```$nameStx:ident $_argsStx* | $contents```) => - let contents := contents.getString - match nameStx.getId with - | `namespace => - for str in contents.splitToList Char.isWhitespace do - if !str.isEmpty then - namespaces := namespaces.insert str.toName - | `exceptions => - for str in contents.splitToList Char.isWhitespace do - if !str.isEmpty then - exceptions := exceptions.insert str.toName - | _ => throwErrorAt nameStx "Expected 'namespace' or 'exceptions'" - | _ => throwErrorAt block "Expected code block named 'namespace' or 'exceptions'" + let some { name? := some nameStx, content := code, .. } := CodeBlockView.of block + | throwErrorAt block "Expected code block named 'namespace' or 'exceptions'" + let contents := code.getVersoCodeBlock + match nameStx.getId with + | `namespace => + for str in contents.splitToList Char.isWhitespace do + if !str.isEmpty then + namespaces := namespaces.insert str.toName + | `exceptions => + for str in contents.splitToList Char.isWhitespace do + if !str.isEmpty then + exceptions := exceptions.insert str.toName + | _ => throwErrorAt nameStx "Expected 'namespace' or 'exceptions'" let mut present : Lean.NameMap NameSet := {} for ns in namespaces do diff --git a/verso/src/verso-manual/VersoManual/Imports.lean b/verso/src/verso-manual/VersoManual/Imports.lean index fa7038fda..bc28d31d2 100644 --- a/verso/src/verso-manual/VersoManual/Imports.lean +++ b/verso/src/verso-manual/VersoManual/Imports.lean @@ -13,7 +13,6 @@ public meta import Verso.Doc.Elab.Monad public section -open scoped Lean.Doc.Syntax open Verso Doc Elab open Lean diff --git a/verso/src/verso-manual/VersoManual/InlineLean.lean b/verso/src/verso-manual/VersoManual/InlineLean.lean index ac1a5de73..1a7860127 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean.lean @@ -37,7 +37,7 @@ open SubVerso.Highlighting open Verso.SyntaxUtils (runParserCategory' SyntaxError parseStrLitAsCategory strLitInputContext) -open Lean.Doc.Syntax +open Lean.Doc (CodeView RoleView) open Lean.Elab.Tactic.GuardMsgs namespace Verso.Genre.Manual.InlineLean @@ -200,7 +200,7 @@ private meta def quoteHighlightViaSerialization (hls : Highlighted) : DocElabM T De-indents and returns (syntax of) a Block representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanBlock [Verso.VersoLiteral k] (shouldShow : Bool) +private meta def toHighlightedLeanBlock [VersoLiteral k] (shouldShow : Bool) (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Block.concat #[]) @@ -214,18 +214,18 @@ private meta def toHighlightedLeanBlock [Verso.VersoLiteral k] (shouldShow : Boo let range := range.map (← getFileMap).utf8RangeToLspRange ``(Block.other (Block.lean $(← quoteHighlightViaSerialization hls) (some $(quote (← getFileName))) $(quote range)) - #[Block.code $(quote (Verso.decode str))]) + #[Block.code $(quote (decode str))]) /-- Returns (syntax of) an Inline representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanInline [Verso.VersoLiteral k] (shouldShow : Bool) +private meta def toHighlightedLeanInline [VersoLiteral k] (shouldShow : Bool) (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Inline.concat #[]) - ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote (Verso.decode str))]) + ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote (decode str))]) /-- @@ -248,14 +248,14 @@ private meta partial def disableUnusedVarLinterInInfoTree : InfoTree → InfoTre .node info (children.map disableUnusedVarLinterInInfoTree) | .hole id => .hole id -meta def elabCommands [Verso.VersoLiteral k] (config : LeanBlockConfig) (str : TSyntax k) +meta def elabCommands [VersoLiteral k] (config : LeanBlockConfig) (str : TSyntax k) (toHighlightedLeanContent : (shouldShow : Bool) → (hls : Highlighted) → (str : TSyntax k) → DocElabM Term) (minCommands : Option Nat := none) (maxCommands : Option Nat := none) : DocElabM Term := withoutAsync <| do - PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (Verso.decode str))) + PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (decode str))) (kind := Lsp.SymbolKind.file) (detail? := some ("Lean code" ++ config.outlineMeta)) @@ -352,8 +352,6 @@ meta def elabCommands [Verso.VersoLiteral k] (config : LeanBlockConfig) (str : T reportMessages config.error str cmdState.messages - if config.show then - warnLongLines col? str.raw where runCommand (act : Command.CommandElabM Unit) (stx : Syntax) (cctx : Command.Context) (cmdState : Command.State) : @@ -378,7 +376,10 @@ Elaborates the provided Lean command in the context of the current Verso module. -/ @[code_block] meta def lean : CodeBlockExpanderOf LeanBlockConfig - | config, str => elabCommands config str toHighlightedLeanBlock + | config, str => do + -- Only a code block is rendered wide enough for its line lengths to matter. + if config.show then warnLongLines str + elabCommands config str toHighlightedLeanBlock @[role] meta def leanCommand : RoleExpanderOf LeanBlockConfig @@ -469,7 +470,7 @@ meta def leanInline : RoleExpanderOf LeanInlineConfig | config, inlines => withoutAsync do let #[arg] := inlines | throwError "Expected exactly one argument" - let some ⟨_, _, term, _⟩ := Lean.Doc.CodeView.of arg + let some { content := term, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let leveller := @@ -528,9 +529,9 @@ meta def leanInline : RoleExpanderOf LeanInlineConfig pushInfoTree (disableUnusedVarLinterInInfoTree tree) - if let `(inline|role{%$s $f $_*}%$e[$_*]) ← getRef then - Hover.addCustomHover (mkNullNode #[s, e]) type - Hover.addCustomHover f type + if let some v := RoleView.of ⟨← getRef⟩ then + Hover.addCustomHover (mkNullNode #[v.braceOpen, v.braceClose]) type + Hover.addCustomHover v.name type if config.error then if newMsgs.hasErrors then @@ -559,7 +560,7 @@ meta def inst : RoleExpanderOf LeanBlockConfig | config, inlines => withoutAsync <| do let #[arg] := inlines | throwError "Expected exactly one argument" - let some ⟨_, _, term, _⟩ := Lean.Doc.CodeView.of arg + let some { content := term, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let stx ← parseStrLitAsCategory `term term @@ -861,7 +862,7 @@ meta def constTok [Monad m] [MonadEnv m] [MonadLiftT MetaM m] [MonadLiftT IO m] @[role] meta def name : RoleExpanderOf NameConfig | cfg, #[arg] => do - let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let exampleName := name.getVersoCode.toName let identStx := mkIdentFrom arg (cfg.full.getD exampleName) (canonical := true) @@ -889,7 +890,7 @@ meta def name : RoleExpanderOf NameConfig @[role] meta def module : RoleExpanderOf Unit | (), #[arg] => do - let some ⟨_, _, name, _⟩ := Lean.Doc.CodeView.of arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the module's name" let exampleName := name.getVersoCode.toName let identStx := mkIdentFrom arg exampleName (canonical := true) diff --git a/verso/src/verso-manual/VersoManual/InlineLean/IO.lean b/verso/src/verso-manual/VersoManual/InlineLean/IO.lean index b8c2dbd58..1f1247ff3 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/IO.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/IO.lean @@ -28,6 +28,7 @@ public section open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open SubVerso.Highlighting Highlighted open Lean Elab +open Lean.Doc (VersoCodeBlock) open Lean.Elab.Tactic.GuardMsgs @@ -300,7 +301,7 @@ meta def startExample [Monad m] [MonadEnv m] [MonadError m] [MonadQuotation m] [ modifyEnv fun env => ioExampleCtx.setState env (some {leanCodeName}) -meta def saveLeanCode (src : Lean.Doc.VersoCodeBlock) : DocElabM Ident := do +meta def saveLeanCode (src : VersoCodeBlock) : DocElabM Ident := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't set Lean code - not in an IO example" | some st => @@ -311,19 +312,19 @@ meta def saveLeanCode (src : Lean.Doc.VersoCodeBlock) : DocElabM Ident := do else throwError "Code already specified" -meta def saveInputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : Lean.Doc.VersoCodeBlock) : m Unit := do +meta def saveInputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save file - not in an IO example" | some st => modifyEnv fun env => ioExampleCtx.setState env (some {st with inputFiles := st.inputFiles.push (name, contents)}) -meta def saveOutputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : Lean.Doc.VersoCodeBlock) : m Unit := do +meta def saveOutputFile [Monad m] [MonadEnv m] [MonadError m] (name : System.FilePath) (contents : VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save file - not in an IO example" | some st => modifyEnv fun env => ioExampleCtx.setState env (some {st with outputFiles := st.outputFiles.push (name, contents)}) -meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do +meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stdin - not in an IO example" | some st => @@ -331,7 +332,7 @@ meta def saveStdin [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.Ve | none => modifyEnv fun env => ioExampleCtx.setState env (some {st with stdin := some contents}) | some _ => throwError "stdin already specified" -meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do +meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stdout - not in an IO example" | some st => @@ -339,7 +340,7 @@ meta def saveStdout [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.V | none => modifyEnv fun env => ioExampleCtx.setState env (some {st with stdout := some contents}) | some _ => throwError "stdout already specified" -meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.VersoCodeBlock) : m Unit := do +meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : VersoCodeBlock) : m Unit := do match ioExampleCtx.getState (← getEnv) with | none => throwError "Can't save stderr - not in an IO example" | some st => @@ -349,9 +350,9 @@ meta def saveStderr [Monad m] [MonadEnv m] [MonadError m] (contents : Lean.Doc.V meta def check - (leanCode : Lean.Doc.VersoCodeBlock) (leanCodeName : Name) - (inputFiles outputFiles : Array (System.FilePath × Lean.Doc.VersoCodeBlock)) - (stdin stdout stderr : Option Lean.Doc.VersoCodeBlock) : DocElabM Highlighted := + (leanCode : VersoCodeBlock) (leanCodeName : Name) + (inputFiles outputFiles : Array (System.FilePath × VersoCodeBlock)) + (stdin stdout stderr : Option VersoCodeBlock) : DocElabM Highlighted := IO.FS.withTempDir fun dirname => do let toolchain : String ← IO.FS.readFile "lean-toolchain" let leanCodeName : String := diff --git a/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean b/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean index dac61b293..a764eb791 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/IO/Context.lean @@ -13,15 +13,16 @@ public section namespace Verso.Genre.Manual.InlineLean.IOExample open Lean +open Lean.Doc structure IOExampleContext where leanCodeName : Ident - code : Option Doc.VersoCodeBlock := none - inputFiles : Array (System.FilePath × Doc.VersoCodeBlock) := #[] - outputFiles : Array (System.FilePath × Doc.VersoCodeBlock) := #[] - stdin : Option Doc.VersoCodeBlock := none - stdout : Option Doc.VersoCodeBlock := none - stderr : Option Doc.VersoCodeBlock := none + code : Option VersoCodeBlock := none + inputFiles : Array (System.FilePath × VersoCodeBlock) := #[] + outputFiles : Array (System.FilePath × VersoCodeBlock) := #[] + stdin : Option VersoCodeBlock := none + stdout : Option VersoCodeBlock := none + stderr : Option VersoCodeBlock := none deriving Repr initialize ioExampleCtx : EnvExtension (Option IOExampleContext) ← diff --git a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean index f11580af5..92429f938 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean @@ -6,6 +6,7 @@ Author: David Thrane Christiansen module public import Lean.Data.Lsp.Utf16 +public import Lean.DocString.View public import Lean.Data.Options public import Lean.Data.Position public import Lean.Log @@ -13,6 +14,7 @@ public import Lean.Log public section open Lean MonadOptions +open Lean.Doc register_option verso.code.warnLineLength : Nat := { defValue := 60 @@ -25,26 +27,27 @@ def getWarnLineLength [Monad m] [MonadOptions m] : m (Option Nat) := do let val := (← getOptions).get verso.code.warnLineLength.name verso.code.warnLineLength.defValue if val = 0 then return none else return some val -def warnLongLines [Monad m] [MonadFileMap m] [MonadLog m] [AddMessageContext m] [MonadOptions m] (indent? : Option Nat) (str : Syntax) : m Unit := do +/-- +Warns about the lines of `code` that are too long to render in a narrow context. + +A code block's indentation is the leading whitespace of each of its line tokens, so a line's width +is the width of its token's contents. +-/ +def warnLongLines [Monad m] [MonadLog m] [AddMessageContext m] [MonadOptions m] + (code : VersoCodeBlock) : m Unit := do let some maxCodeColumns ← getWarnLineLength | pure () - let fileMap ← getFileMap - let maxCol := maxCodeColumns + indent?.getD 0 - if let some startPos := str.getPos? then - if let some stopPos := str.getTailPos? then - let ⟨startLine, _⟩ := fileMap.toPosition startPos - let ⟨stopLine, _⟩ := fileMap.toPosition stopPos - for l in [startLine:stopLine] do - let nextStart := fileMap.lineStart (l + 1) - let ⟨_, endCol⟩ := fileMap.utf8PosToLspPos (nextStart.prev fileMap.source) - if endCol > maxCol then - let thisStart := fileMap.lineStart l - let fakeLiteral := Syntax.mkStrLit (thisStart.extract fileMap.source nextStart) (.synthetic thisStart nextStart) - let note := - MessageData.note m!"Example code is shown on mobile devices and other narrow contexts. \ - Long lines are likely to be truncated in the rendered output." - let hint := - MessageData.hint' m!"The limit of this linter can be changed with the option \ - `{.ofConstName ``verso.code.warnLineLength}`. This linter can be disabled by setting \ - this option to 0." - logWarningAt fakeLiteral m!"Line {l} is too long ({endCol} columns exceeds {maxCol}).{note}{hint}" + let lines := code.getVersoCodeBlockLines + for h : i in [0:lines.size] do + let line := lines[i] + let width := line.getVersoCodeBlockLine.trimAsciiEnd.length + if width > maxCodeColumns then + let note := + MessageData.note m!"Example code is shown on mobile devices and other narrow contexts. \ + Long lines are likely to be truncated in the rendered output." + let hint := + MessageData.hint' m!"The limit of this linter can be changed with the option \ + `{.ofConstName ``verso.code.warnLineLength}`. This linter can be disabled by setting \ + this option to 0." + logWarningAt line + m!"Line {i + 1} is too long ({width} columns exceeds {maxCodeColumns}).{note}{hint}" diff --git a/verso/src/verso-manual/VersoManual/InlineLean/Option.lean b/verso/src/verso-manual/VersoManual/InlineLean/Option.lean index 92e1b16fc..40a33e703 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/Option.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/Option.lean @@ -18,7 +18,7 @@ open Verso Genre Manual ArgParse Doc Elab open Verso Output Html open Verso Code Highlighted WebAssets open Lean -open Lean.Doc.Syntax +open Lean.Doc (CodeView) namespace Verso.Genre.Manual.InlineLean @@ -29,7 +29,7 @@ meta def option : RoleExpanderOf Unit | (), inlines => withoutAsync do let #[arg] := inlines | throwError "Expected exactly one argument" - let some ⟨_, _, optName, _⟩ := Lean.Doc.CodeView.of arg + let some { content := optName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the option name" let optName := optName.getVersoCode.toName let optDecl ← getOptionDecl optName diff --git a/verso/src/verso-manual/VersoManual/Linters.lean b/verso/src/verso-manual/VersoManual/Linters.lean index 020561027..a1edbc3e7 100644 --- a/verso/src/verso-manual/VersoManual/Linters.lean +++ b/verso/src/verso-manual/VersoManual/Linters.lean @@ -15,7 +15,8 @@ public section set_option linter.missingDocs true open Lean Linter Elab Command -open Lean.Doc.Syntax +open Lean.Doc (HeaderView InlineView MetadataView) +open Lean.Doc.Parser /-- Lints for tagless headers. @@ -48,7 +49,7 @@ meta partial def headerTagLinter : Linter where let text ← getFileMap discard <| stx.replaceM fun block => do - if let some hdr := Lean.Doc.HeaderView.of ⟨block⟩ then + if let some hdr := HeaderView.of ⟨block⟩ then let inls := hdr.content let some ⟨start, stop⟩ := block.getRange? | return none @@ -68,7 +69,7 @@ meta partial def headerTagLinter : Linter where } let toks := Parser.getTokenTable (← getEnv) let s := { cache := { tokenCache := {}, parserCache := {} }, pos := nextLine.i } - let s := Verso.Parser.metadataBlock.run ictx pmctx toks s + let s := metadataBlockFn.run ictx pmctx toks s let tagNote := MessageData.note <| "The tag is used as a permanent name for the section or chapter. Writers "++ @@ -90,7 +91,7 @@ meta partial def headerTagLinter : Linter where if s.stxStack.size = 1 then pure (s.stxStack.get! 0) else return none - if let some metaView := Lean.Doc.MetadataView.of ⟨nextStx⟩ then + if let some metaView := MetadataView.of ⟨nextStx⟩ then let fieldOrAbbrev := metaView.fields let metadataStx : Term := ⟨(← `(Lean.Parser.Term.structInst| { $[$fieldOrAbbrev],* })).raw⟩ @@ -134,7 +135,7 @@ where suggestId' (name : TSyntaxArray ``Lean.Doc.Parser.inline) : String := Id.run do let mut strTitle := "" for inl in name do - match Lean.Doc.InlineView.of inl with + match InlineView.of inl with | some (.text s) => strTitle := strTitle ++ s.getVersoText.toLower | some (.code c) => strTitle := strTitle ++ c.getVersoCode | some (.emph e) => strTitle := strTitle ++ suggestId' e.content diff --git a/verso/src/verso-manual/VersoManual/Literate.lean b/verso/src/verso-manual/VersoManual/Literate.lean index adebad0e1..b8dc47a2d 100644 --- a/verso/src/verso-manual/VersoManual/Literate.lean +++ b/verso/src/verso-manual/VersoManual/Literate.lean @@ -85,7 +85,6 @@ instance : LoadLiterate Manual where docstringPart lvl title contents := .other (Block.literateDocstringPart lvl) (#[.para title] ++ contents) -open Lean.Doc.Syntax open Verso.Doc Elab Concrete open Lean.Elab Command Term open PartElabM diff --git a/verso/src/verso-manual/VersoManual/Table.lean b/verso/src/verso-manual/VersoManual/Table.lean index a36fe8910..6ff180688 100644 --- a/verso/src/verso-manual/VersoManual/Table.lean +++ b/verso/src/verso-manual/VersoManual/Table.lean @@ -15,7 +15,8 @@ public section open Verso Doc Elab open Verso.Genre Manual open Verso.ArgParse -open Lean.Doc.Syntax +open Lean.Doc (UnorderedListView) +open Lean.Doc.Parser open Lean Elab @@ -194,13 +195,13 @@ meta def table : DirectiveExpanderOf TableConfig -- The table should be a list of lists. Extract them! let #[oneBlock] := contents | throwError "Expected a single unordered list" - let some outer := Lean.Doc.UnorderedListView.of oneBlock + let some outer := UnorderedListView.of oneBlock | throwErrorAt oneBlock "Expected a single unordered list" let preRows := outer.items.map (·.contents) let rows ← preRows.mapM fun blks => do let #[oneInRow] := blks.filter (·.raw.isOfKind ``Lean.Doc.Parser.Block.ul) | throwError "Each row should have exactly one list in it" - let some inner := Lean.Doc.UnorderedListView.of oneInRow + let some inner := UnorderedListView.of oneInRow | throwErrorAt oneInRow "Each row should have exactly one list in it" pure (inner.items.map (·.contents)) if h : rows.size = 0 then diff --git a/verso/src/verso/Verso/Code/External.lean b/verso/src/verso/Verso/Code/External.lean index 37f0f7305..41f1009ad 100644 --- a/verso/src/verso/Verso/Code/External.lean +++ b/verso/src/verso/Verso/Code/External.lean @@ -42,7 +42,7 @@ open SubVerso Highlighting open Lean Meta Hint open Std -open Lean.Doc.Syntax +open Lean.Doc (RoleView VersoCodeBlock) namespace Verso.Code.External @@ -193,7 +193,7 @@ where if k > n then n := k n.fold (fun _ _ s => s.push '`') "" -meta def moduleContentBlock (args : Array Arg) (code : Lean.Doc.VersoCodeBlock) : +meta def moduleContentBlock (args : Array Arg) (code : VersoCodeBlock) : DocElabM (Array Term) := do let cfg@{ module := moduleName, project, anchor?, showProofStates := _, defSite := _ } ← parseThe CodeContext args withAnchored project moduleName anchor? fun hl => do @@ -399,7 +399,7 @@ public meta def anchorTerm : RoleExpander else throwError "Expected a positional argument first (the anchor name)" -public meta def moduleTermBlock (args : Array Arg) (term : Lean.Doc.VersoCodeBlock) : +public meta def moduleTermBlock (args : Array Arg) (term : VersoCodeBlock) : DocElabM (Array Term) := do let cfg@{module := moduleName, project, anchor?, showProofStates := _, defSite := _} ← parseThe CodeContext args @@ -471,7 +471,7 @@ private meta partial def findTrace? (header : String) : MessageContents Highligh if msg.toString == header then pure t else chs.findSome? (findTrace? header) -public meta def outputBlock (args : Array Arg) (str : Lean.Doc.VersoCodeBlock) : +public meta def outputBlock (args : Array Arg) (str : VersoCodeBlock) : DocElabM (Array Term) := do let {module := moduleName, project, anchor?, severity, expandTraces, onlyTrace, showProofStates := _, defSite := _} ← parseThe MessageContext args let strText := str.getVersoCodeBlock @@ -627,7 +627,7 @@ public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray ``Lean.D -- A role's content is the better place to report at than the role as a whole. let ref := - match Lean.Doc.RoleView.of ⟨← getRef⟩ with + match RoleView.of ⟨← getRef⟩ with | some v => if h : v.content.size = 1 then v.content[0].raw else str.raw | none => str.raw @@ -652,11 +652,14 @@ public meta def moduleOutInline (args : Array Arg) (inls : TSyntaxArray ``Lean.D else pure <| msg let err := m!"Expected one of:{indentD (m!"\n".joinSep <| candidates.toList.map (·.toString (expandTraces := expandTraces)))}" Lean.logError m!"No expected term provided. {err}" - if let `(inline|role{$_ $_*} [%$tok1 $contents* ]%$tok2) := (← getRef) then + if let some v := RoleView.of ⟨← getRef⟩ then let stx := - if tok1.getHeadInfo matches .original .. && tok2.getHeadInfo matches .original .. then - mkNullNode #[tok1, tok2] - else mkNullNode contents + match v.brackets with + | some (tok1, tok2) => + if tok1.getHeadInfo matches .original .. && tok2.getHeadInfo matches .original .. then + mkNullNode #[tok1, tok2] + else mkNullNode (v.content.map (·.raw)) + | none => mkNullNode (v.content.map (·.raw)) for (msg, _) in infos do let str := msg.toString |>.trimAscii |>.copy Suggestion.saveSuggestion stx (quoteCode <| ExpectString.abbreviateString str) (quoteCode str) diff --git a/verso/src/verso/Verso/Doc/ArgParse.lean b/verso/src/verso/Verso/Doc/ArgParse.lean index 838220ebf..5ca810134 100644 --- a/verso/src/verso/Verso/Doc/ArgParse.lean +++ b/verso/src/verso/Verso/Doc/ArgParse.lean @@ -18,6 +18,8 @@ public section open Lean Elab open Verso Doc +open Lean.Doc +open Lean.Doc.Parser namespace Verso @@ -699,7 +701,7 @@ open Verso.Parser in Parses a sequence of Verso inline elements from a string literal. The resulting syntax is adjusted so that escapes in the string literal are accounted for in source positions. -/ -def ValDesc.inlinesString [MonadFileMap m] : ValDesc m (TSyntaxArray ``Lean.Doc.Parser.inline) where +def ValDesc.inlinesString [MonadFileMap m] : ValDesc m (TSyntaxArray ``Parser.inline) where description := doc!"a string that contains a sequence of inline elements" signature := .String get diff --git a/verso/src/verso/Verso/Doc/Concrete.lean b/verso/src/verso/Verso/Doc/Concrete.lean index 6a220e0ac..ea4e6cf56 100644 --- a/verso/src/verso/Verso/Doc/Concrete.lean +++ b/verso/src/verso/Verso/Doc/Concrete.lean @@ -255,7 +255,7 @@ private meta def versoBlockCommandFn : ParserFn := fun c s => let iniSz := s.stackSize let lastPos? := lastVersoEndPosExt.getState c.env let s := lastPos? |>.map s.setPos |>.getD s - let s := recoverBlockWith #[.missing] (Verso.Parser.block {}) c s + let s := recoverBlockWith #[.missing] (blockFn {}) c s if s.hasError then s else let s := ignoreFn (manyFn blankLine) c s @@ -367,8 +367,6 @@ private meta def finishDoc : Command.CommandElabM Unit:= do syntax (name := replaceDoc) "#doc " "(" term ") " str " =>" : command elab_rules : command | `(command|#doc ( $genreSyntax:term ) $title:str =>%$tok) => open Lean Parser Elab Command in do - elabCommand <| ← `(open scoped Lean.Doc.Syntax) - startDoc genreSyntax title -- Sets up basic incremental evaluation of documents by replacing Lean's command-by-command parser diff --git a/verso/src/verso/Verso/Doc/Elab.lean b/verso/src/verso/Verso/Doc/Elab.lean index 87737f736..719ddbff0 100644 --- a/verso/src/verso/Verso/Doc/Elab.lean +++ b/verso/src/verso/Verso/Doc/Elab.lean @@ -19,7 +19,8 @@ namespace Verso.Doc.Elab open Lean Elab open PartElabM open DocElabM -open Lean.Doc.Syntax +open Lean.Doc (ArgView ArgValView BlockView DescItemView MathMode) +open Lean.Doc.Parser open Verso.ArgParse (SigDoc) set_option backward.privateInPublic false @@ -48,7 +49,7 @@ public meta def _root_.Lean.Doc.Parser.Inline.bold.expand : InlineExpander | _ => throwUnsupportedSyntax meta def parseArgVal (val : TSyntax ``Lean.Doc.Parser.argVal) : DocElabM ArgVal := do - match Lean.Doc.ArgValView.of val with + match ArgValView.of val with | some (.str s _) => pure <| .str s | some (.name x) => pure <| .name x | some (.num n _) => pure <| .num n @@ -58,10 +59,10 @@ public meta def parseArgs (argStx : TSyntaxArray ``Lean.Doc.Parser.arg) : DocElabM (Array Arg) := do let mut argVals := #[] for arg in argStx do - match Lean.Doc.ArgView.of arg with - | some (.anon _ v) => + match ArgView.of arg with + | some (.anon (val := v) ..) => argVals := argVals.push (.anon (← parseArgVal v)) - | some (.named _ none x _ v) => do + | some (.named (parens := none) (name := x) (val := v) ..) => do -- A named argument without parentheses is the deprecated spelling. let src := (← getFileMap).source if let some ⟨s, e⟩ := x.raw.getRange? (canonicalOnly := true) then @@ -69,9 +70,9 @@ public meta def parseArgs (argStx : TSyntaxArray ``Lean.Doc.Parser.arg) : let hint ← MessageData.hint m!"Replace with the updated syntax:" #[s!"({s.extract src e} := {s'.extract src e'})"] (ref? := some arg) logWarningAt arg m!"Deprecated named argument syntax for `{x}`{hint}" argVals := argVals.push (.named arg x (← parseArgVal v)) - | some (.named _ (some _) x _ v) => + | some (.named (parens := some _) (name := x) (val := v) ..) => argVals := argVals.push (.named arg x (← parseArgVal v)) - | some (.flag _ _ x isOn) => + | some (.flag (name := x) (isOn := isOn) ..) => argVals := argVals.push (.flag arg x isOn) | none => throwErrorAt arg "Can't decode argument '{repr arg}'" pure argVals @@ -149,8 +150,8 @@ public meta def _root_.Lean.Doc.Parser.Inline.link.expand : InlineExpander | .link v => do let url : TSyntax `term ← match v.target with - | .url _ _ u _ => pure (quote u.getVersoLinkUrl) - | .ref _ _ name _ => addLinkRef name + | .url (url := u) .. => pure (quote u.getVersoLinkUrl) + | .ref (name := name) .. => addLinkRef name ``(Inline.link #[$[$(← v.content.mapM elabInline)],*] $url) | _ => throwUnsupportedSyntax @@ -166,8 +167,8 @@ public meta def _root_.Lean.Doc.Parser.Inline.image.expand : InlineExpander | .image v => do let url : TSyntax `term ← match v.target with - | .url _ _ u _ => pure (quote u.getVersoLinkUrl) - | .ref _ _ name _ => addLinkRef name + | .url (url := u) .. => pure (quote u.getVersoLinkUrl) + | .ref (name := name) .. => addLinkRef name ``(Inline.image $(quote v.getAlt) $url) | _ => throwUnsupportedSyntax @@ -198,7 +199,7 @@ public meta def partCommand (cmd : TSyntax ``Lean.Doc.Parser.block) : PartElabM withRef cmd <| withFreshMacroScope <| do match cmd.raw with | stx@(.node _ kind _) => - let some view := Lean.Doc.BlockView.of ⟨stx⟩ + let some view := BlockView.of ⟨stx⟩ | fallback let exp ← partCommandsFor kind for e in exp do @@ -386,7 +387,7 @@ public meta def _root_.Lean.Doc.Parser.Block.ol.expand : BlockExpander | _ => throwUnsupportedSyntax -meta def elabDesc (item : Lean.Doc.DescItemView) : DocElabM (Syntax × TSyntax `term) := +meta def elabDesc (item : DescItemView) : DocElabM (Syntax × TSyntax `term) := withRef item.stx <| do let genre := (← readThe DocElabContext).genreSyntax let item' ← ``(DescItem.mk (α := Inline $(⟨genre⟩)) (β := Block $(⟨genre⟩)) #[$[$(← item.term.mapM elabInline)],*] #[$[$(← item.desc.mapM elabBlock)],*]) diff --git a/verso/src/verso/Verso/Doc/Elab/Basic.lean b/verso/src/verso/Verso/Doc/Elab/Basic.lean index 6ea4940d7..e8767ca22 100644 --- a/verso/src/verso/Verso/Doc/Elab/Basic.lean +++ b/verso/src/verso/Verso/Doc/Elab/Basic.lean @@ -111,7 +111,7 @@ public def PartFrame.close (fr : PartFrame) (endPos : String.Pos.Raw) : Finished /-- References that must be local to the current blob of concrete document syntax -/ public structure DocDef (α : Type) where - defSite : Syntax + defSite : Lean.Doc.VersoRefName val : α deriving Repr @@ -176,7 +176,7 @@ public def internalRefs (defs : HashMap String (DocDef α)) (refs : HashMap Stri let mut refInfo := #[] for k in keys do refInfo := refInfo.push { - defSite := defs[k]? |>.map (·.defSite), + defSite := defs[k]? |>.map (·.defSite.raw), useSites := refs[k]? |>.map (·.useSites) |>.getD #[] } refInfo diff --git a/verso/src/verso/Verso/Doc/Elab/Block.lean b/verso/src/verso/Verso/Doc/Elab/Block.lean index 21de70940..c7a18d925 100644 --- a/verso/src/verso/Verso/Doc/Elab/Block.lean +++ b/verso/src/verso/Verso/Doc/Elab/Block.lean @@ -14,13 +14,14 @@ namespace Verso.Doc.Elab open Lean Elab open PartElabM open DocElabM -open Lean.Doc.Syntax +open Lean.Doc (BlockView) +open Lean.Doc.Parser open Verso.ArgParse (SigDoc) set_option backward.privateInPublic false /-- Records the delimiters of a block that has both, so that each hover mentions the other. -/ -def decorateClosing : Lean.Doc.BlockView → DocElabM Unit +def decorateClosing : BlockView → DocElabM Unit | .directive v => closes v.opener v.closer | .codeblock v => closes v.openFence v.closeFence | .metadata v => closes v.opener v.closer @@ -44,7 +45,7 @@ public partial def elabBlock (block : TSyntax ``Lean.Doc.Parser.block) : withRef stxNew <| elabBlock ⟨stxNew⟩ | none => - let some view := Lean.Doc.BlockView.of ⟨stx⟩ + let some view := BlockView.of ⟨stx⟩ | throwUnexpected stx decorateClosing view let exp ← blockExpandersFor kind diff --git a/verso/src/verso/Verso/Doc/Elab/Inline.lean b/verso/src/verso/Verso/Doc/Elab/Inline.lean index e1a81898c..9dc86c7e0 100644 --- a/verso/src/verso/Verso/Doc/Elab/Inline.lean +++ b/verso/src/verso/Verso/Doc/Elab/Inline.lean @@ -13,7 +13,8 @@ namespace Verso.Doc.Elab open Lean Elab open PartElabM open DocElabM -open Lean.Doc.Syntax +open Lean.Doc (InlineView) +open Lean.Doc.Parser open Verso.ArgParse (SigDoc) set_option backward.privateInPublic false @@ -36,7 +37,7 @@ public partial def elabInline (inline : TSyntax ``Lean.Doc.Parser.inline) : withRef stxNew <| elabInline ⟨stxNew⟩ | none => - let some view := Lean.Doc.InlineView.of ⟨stx⟩ + let some view := InlineView.of ⟨stx⟩ | throwUnexpected stx let exp ← inlineExpandersFor kind for e in exp do diff --git a/verso/src/verso/Verso/Doc/Elab/Monad.lean b/verso/src/verso/Verso/Doc/Elab/Monad.lean index 4c77d7c2d..655f3fb39 100644 --- a/verso/src/verso/Verso/Doc/Elab/Monad.lean +++ b/verso/src/verso/Verso/Doc/Elab/Monad.lean @@ -30,7 +30,9 @@ namespace Verso.Doc.Elab open Lean open Lean.Elab -open Lean.Doc.Syntax +open Lean.Doc (BlockView BoldView CodeView EmphView HeaderView InlineView LinebreakView RoleView + TextView VersoCodeBlock VersoRefName) +open Lean.Doc.Parser open Std (HashMap HashSet) open Verso.ArgParse (FromArgs SigDoc) @@ -51,32 +53,34 @@ private def footnoteRefName [Monad m] [MonadQuotation m] (genre : Term) (docName ``(HasNote.contents $(quote ref) $(quote docName) (genre := $genre)) --- For use in IDE features and previews and such +-- For use in IDE features and previews and such. +-- `inline_to_string` takes its key literally rather than resolving it, so these names are +-- written in full. @[inline_to_string Lean.Doc.Parser.Inline.text] public meta def _root_.Lean.Doc.Parser.Inline.text.inline_to_string : InlineToString - | _, stx => Lean.Doc.TextView.of ⟨stx⟩ |>.map (·.getVersoText) + | _, stx => TextView.of ⟨stx⟩ |>.map (·.getVersoText) @[inline_to_string Lean.Doc.Parser.Inline.linebreak] public meta def _root_.Lean.Doc.Parser.Inline.linebreak.inline_to_string : InlineToString - | _, stx => Lean.Doc.LinebreakView.of ⟨stx⟩ |>.map fun _ => " " + | _, stx => LinebreakView.of ⟨stx⟩ |>.map fun _ => " " @[inline_to_string Lean.Doc.Parser.Inline.emph] public meta def _root_.Lean.Doc.Parser.Inline.emph.inline_to_string : InlineToString - | env, stx => Lean.Doc.EmphView.of ⟨stx⟩ |>.map fun v => + | env, stx => EmphView.of ⟨stx⟩ |>.map fun v => String.intercalate " " (v.content.toList.map (inlineToString env ·.raw)) @[inline_to_string Lean.Doc.Parser.Inline.bold] public meta def _root_.Lean.Doc.Parser.Inline.bold.inline_to_string : InlineToString - | env, stx => Lean.Doc.BoldView.of ⟨stx⟩ |>.map fun v => + | env, stx => BoldView.of ⟨stx⟩ |>.map fun v => String.intercalate " " (v.content.toList.map (inlineToString env ·.raw)) @[inline_to_string Lean.Doc.Parser.Inline.code] public meta def _root_.Lean.Doc.Parser.Inline.code.inline_to_string : InlineToString - | _, stx => Lean.Doc.CodeView.of ⟨stx⟩ |>.map (·.getVersoCode) + | _, stx => CodeView.of ⟨stx⟩ |>.map (·.getVersoCode) @[inline_to_string Lean.Doc.Parser.Inline.role] public meta def _root_.Lean.Doc.Parser.Inline.role.inline_to_string : InlineToString - | env, stx => Lean.Doc.RoleView.of ⟨stx⟩ |>.map fun v => + | env, stx => RoleView.of ⟨stx⟩ |>.map fun v => String.join (v.content.toList.map (inlineToString env ·.raw)) @[inline_to_string null] @@ -104,7 +108,7 @@ public def inlineSyntaxToString (env : Environment) (inlines : Syntax) : String "" public def headerStxToString (env : Environment) (headerStx : Syntax) : String := - match Lean.Doc.HeaderView.of ⟨headerStx⟩ with + match HeaderView.of ⟨headerStx⟩ with | some v => inlinesToString env (v.content.map (·.raw)) | none => dbg_trace "didn't understand {headerStx} for string" @@ -346,7 +350,7 @@ public def PartElabM.addBlock (block : TSyntax `term) (blockInternalDocReconstru public def PartElabM.addPart (finished : FinishedPart) : PartElabM Unit := modifyThe State fun st => { st with partContext.priorParts := st.partContext.priorParts.push finished } -public def PartElabM.addLinkDef (refName : Lean.Doc.VersoRefName) (url : String) : +public def PartElabM.addLinkDef (refName : VersoRefName) (url : String) : PartElabM Unit := do let strName := refName.getVersoRefName let docName ← currentDocName @@ -364,12 +368,12 @@ public def PartElabM.addLinkDef (refName : Lean.Doc.VersoRefName) (url : String) } setReducibilityStatus n .implicitReducible Meta.addInstance n AttributeKind.global (eval_prio default) - modifyThe State fun st => {st with linkDefs := st.linkDefs.insert strName ⟨refName.raw, url⟩} + modifyThe State fun st => {st with linkDefs := st.linkDefs.insert strName ⟨refName, url⟩} | some ⟨_, url'⟩ => throwErrorAt refName "Already defined link [{strName}] as '{url'}'" -public def DocElabM.addLinkRef (refName : Lean.Doc.VersoRefName) : DocElabM (TSyntax `term) := do +public def DocElabM.addLinkRef (refName : VersoRefName) : DocElabM (TSyntax `term) := do let strName := refName.getVersoRefName match (← readThe DocElabContext).refsAllowed with | .always => pure () @@ -386,7 +390,7 @@ public def DocElabM.addLinkRef (refName : Lean.Doc.VersoRefName) : DocElabM (TSy linkRefName (← currentDocName) strName -public def PartElabM.addFootnoteDef (refName : Lean.Doc.VersoRefName) +public def PartElabM.addFootnoteDef (refName : VersoRefName) (content : Array (TSyntax `term)) : PartElabM Unit := do let strName := refName.getVersoRefName let docName ← currentDocName @@ -408,11 +412,11 @@ public def PartElabM.addFootnoteDef (refName : Lean.Doc.VersoRefName) } setReducibilityStatus n .implicitReducible Meta.addInstance n AttributeKind.global (eval_prio default) - modifyThe State fun st => {st with footnoteDefs := st.footnoteDefs.insert strName ⟨refName.raw, content⟩} + modifyThe State fun st => {st with footnoteDefs := st.footnoteDefs.insert strName ⟨refName, content⟩} | some _ => throwErrorAt refName m!"Already defined footnote [^{strName}]" -public def DocElabM.addFootnoteRef (refName : Lean.Doc.VersoRefName) : +public def DocElabM.addFootnoteRef (refName : VersoRefName) : DocElabM (TSyntax `term) := do let strName := refName.getVersoRefName let genre := (← readThe DocElabContext).genreSyntax @@ -452,7 +456,7 @@ public def closes (openTok closeTok : Syntax) : DocElabM Unit := do let lineStr := if lineStr.startsWith "`" || lineStr.endsWith "`" then " " ++ lineStr ++ " " else lineStr.copy Hover.addCustomHover closeTok (.markdown s!"Closes line {line + 1}: ``````````{lineStr}``````````") -public abbrev InlineExpander := Lean.Doc.InlineView → DocElabM (TSyntax `term) +public abbrev InlineExpander := InlineView → DocElabM (TSyntax `term) initialize inlineExpanderAttr : KeyedDeclsAttribute InlineExpander ← mkDocExpanderAttribute `inline_expander ``InlineExpander "Indicates that this function expands inline elements of a given name" `inlineExpanderAttr @@ -537,7 +541,7 @@ public def FinishedPart.toVersoDoc ``(VersoDoc.mk (fun $docReconstructionPlaceholder => $finishedSyntax) $(quote reconstJson.compress)) -public abbrev BlockExpander := Lean.Doc.BlockView → DocElabM (TSyntax `term) +public abbrev BlockExpander := BlockView → DocElabM (TSyntax `term) initialize blockExpanderAttr : KeyedDeclsAttribute BlockExpander ← mkDocExpanderAttribute `block_expander ``BlockExpander "Indicates that this function expands block elements of a given name" `blockExpanderAttr @@ -580,7 +584,7 @@ initialize expanderSignatureExt : public def sig (α) [inst : FromArgs α DocElabM] : Option ArgParse.SigDoc := inst.fromArgs.signature -public abbrev PartCommand := Lean.Doc.BlockView → PartElabM Unit +public abbrev PartCommand := BlockView → PartElabM Unit initialize partCommandAttr : KeyedDeclsAttribute PartCommand ← mkDocExpanderAttribute `part_command ``PartCommand "Indicates that this function is used for side effects on the structure of the document" `partCommandAttr @@ -764,16 +768,16 @@ unsafe initialize registerBuiltinAttribute { public abbrev CodeBlockExpander := - Array Arg → Lean.Doc.VersoCodeBlock → DocElabM (Array (TSyntax `term)) + Array Arg → VersoCodeBlock → DocElabM (Array (TSyntax `term)) -public abbrev CodeBlockExpanderOf α := α → Lean.Doc.VersoCodeBlock → DocElabM Term +public abbrev CodeBlockExpanderOf α := α → VersoCodeBlock → DocElabM Term initialize codeBlockExpanderAttr : KeyedDeclsAttribute CodeBlockExpander ← mkDocExpanderAttribute `code_block_expander ``CodeBlockExpander "Indicates that this function is used to implement a given code block" `codeBlockExpanderAttr public def toCodeBlock {α : Type} [FromArgs α DocElabM] - (expander : α → Lean.Doc.VersoCodeBlock → DocElabM Term) : CodeBlockExpander := + (expander : α → VersoCodeBlock → DocElabM Term) : CodeBlockExpander := fun args str => do let v ← ArgParse.parse args return #[← expander v str] diff --git a/verso/src/verso/Verso/Doc/Helpers.lean b/verso/src/verso/Verso/Doc/Helpers.lean index d94a09b56..b44c51b8e 100644 --- a/verso/src/verso/Verso/Doc/Helpers.lean +++ b/verso/src/verso/Verso/Doc/Helpers.lean @@ -12,6 +12,8 @@ public import Lean.Log set_option doc.verso true open Lean Doc.Syntax +open Lean.Doc +open Lean.Doc.Parser namespace Verso.Doc @@ -20,11 +22,11 @@ If {name}`inlines` contains exactly one code inline, its contents are returned. otherwise. -/ public def oneCodeStr [Monad m] [MonadError m] - (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m Lean.Doc.VersoCode := do + (inlines : TSyntaxArray ``Parser.inline) : m VersoCode := do let #[code] := inlines | (if inlines.size == 0 then (throwError ·) else (throwErrorAt (mkNullNode (inlines.map (·.raw))) ·)) "Expected one code element" - let some v := Lean.Doc.CodeView.of code + let some v := CodeView.of code | throwErrorAt code "Expected a code element" return v.content @@ -33,14 +35,14 @@ If {name}`inlines` contains exactly one code inline, its contents are returned. is logged and {name}`none` is returned. -/ public def oneCodeStr? [Monad m] [MonadError m] [MonadLog m] [AddMessageContext m] [MonadOptions m] - (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m (Option Lean.Doc.VersoCode) := do + (inlines : TSyntaxArray ``Parser.inline) : m (Option VersoCode) := do let #[code] := inlines | if inlines.size == 0 then Lean.logError "Expected a code element" else logErrorAt (mkNullNode (inlines.map (·.raw))) "Expected one code element" return none - let some v := Lean.Doc.CodeView.of code + let some v := CodeView.of code | logErrorAt code "Expected a code element" return none return some v.content @@ -50,7 +52,7 @@ If {name}`inlines` contains exactly one Lean name, it is returned with its sourc identifier. Otherwise, an error is thrown. -/ public def oneCodeName [Monad m] [MonadError m] - (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m Ident := do + (inlines : TSyntaxArray ``Parser.inline) : m Ident := do let code ← oneCodeStr inlines let str := code.getVersoCode let name := if str.contains '.' then str.toName else Name.str .anonymous str diff --git a/verso/src/verso/Verso/Doc/Lsp.lean b/verso/src/verso/Verso/Doc/Lsp.lean index f3f40039a..26a0faa6a 100644 --- a/verso/src/verso/Verso/Doc/Lsp.lean +++ b/verso/src/verso/Verso/Doc/Lsp.lean @@ -24,7 +24,8 @@ namespace Verso.Lsp open Verso.Doc.Elab (DocListInfo DocRefInfo TOC) open Verso.Doc (PointOfInterest) open Verso.Hover -open Lean.Doc.Syntax +open Lean.Doc +open Lean.Doc.Parser open Lean @@ -195,36 +196,41 @@ where -- Tested in Emacs and the problem isn't server side. syntactic (text : FileMap) (pos : String.Pos.Raw) (stx : Syntax) : Option (Array Syntax) := do if includes stx pos |>.getD true then - match stx with - | `(block|:::%$opener $_name $_args* {$_contents*}%$closer ) - | `(block|```%$opener | $_contents ```%$closer) - | `(block|```%$opener $_name $_args* | $_contents ```%$closer) => - if (includes opener pos).getD false || (includes closer pos).getD false then - return #[opener, closer] - | _ => - match stx with - | `(inline| \math%$opener1 code(%$opener2 $_ )%$closer1) - | `(inline| \displaymath%$opener1 code(%$opener2 $_ )%$closer1) => - if (includes opener1 pos).getD false || (includes closer1 pos).getD false || (includes opener2 pos).getD false then - return #[opener1, closer1, opener2] - | `(inline| link[%$opener1 $_* ]%$closer1 (%$opener2 $_ )%$closer2) - | `(inline| link[%$opener1 $_* ]%$closer1 [%$opener2 $_ ]%$closer2) => - if (includes opener1 pos).getD false || (includes closer1 pos).getD false || (includes opener2 pos).getD false || (includes closer2 pos).getD false then - return #[opener1, closer1, opener2, closer2] - | `(inline| code(%$opener $_ )%$closer) => - if (includes opener pos).getD false || (includes closer pos).getD false then - return #[opener, closer] - | `(inline| role{%$opener1 $name $_* }%$closer1 [%$opener2 $subjects ]%$closer2) => - if (includes opener1 pos).getD false || (includes closer1 pos).getD false || - (includes opener2 pos).getD false || (includes closer2 pos).getD false || - (includes name pos).getD false then - return #[opener1, closer1, opener2, closer2, subjects.raw] - | _ => pure () + if let some (toHighlight, alsoTriggers) := delimiters stx then + if (toHighlight ++ alsoTriggers).any (fun s => (includes s pos).getD false) then + return toHighlight if let .node _ _ contents := stx then for s in contents do if let some r := syntactic text pos s then return r failure + /-- + An element's delimiters, which are highlighted together, paired with the syntax that highlights + them without being highlighted itself. + -/ + delimiters (stx : Syntax) : Option (Array Syntax × Array Syntax) := + match BlockView.of ⟨stx⟩ with + | some (.directive v) => some (#[v.opener, v.closer], #[]) + | some (.codeblock v) => some (#[v.openFence, v.closeFence], #[]) + | some (.metadata v) => some (#[v.opener, v.closer], #[]) + | _ => + match InlineView.of ⟨stx⟩ with + | some (.math v) => some (#[v.marker, v.code.opener, v.code.closer], #[]) + | some (.code v) => some (#[v.opener, v.closer], #[]) + | some (.emph v) => some (#[v.opener, v.closer], #[]) + | some (.bold v) => some (#[v.opener, v.closer], #[]) + | some (.footnote v) => some (#[v.opener, v.closer], #[]) + | some (.link v) => some (#[v.opener, v.closer] ++ targetDelimiters v.target, #[]) + | some (.image v) => some (#[v.opener, v.closer] ++ targetDelimiters v.target, #[]) + | some (.role v) => + let brackets := v.brackets.map (fun (o, c) => #[o, c]) |>.getD #[] + some (#[v.braceOpen, v.braceClose] ++ brackets, #[v.name]) + | _ => none + + targetDelimiters : LinkTargetView → Array Syntax + | .url (opener := opener) (closer := closer) .. + | .ref (opener := opener) (closer := closer) .. => #[opener, closer] + includes (stx : Syntax) (pos : String.Pos.Raw) : Option Bool := stx.getRange?.map (fun r => pos ≥ r.start && pos < r.stop) @@ -427,31 +433,32 @@ deriving instance Repr, BEq for SemanticTokenType meta partial def versoTokens (text : FileMap) (stx : Syntax) : Array SemanticTokenEntry := Id.run do - if let some v := Lean.Doc.InlineView.of ⟨stx⟩ then inlineTokens text v - else if let some v := Lean.Doc.BlockView.of ⟨stx⟩ then blockTokens text v - else if let some v := Lean.Doc.DescItemView.of ⟨stx⟩ then + if let some v := InlineView.of ⟨stx⟩ then inlineTokens text v + else if let some v := BlockView.of ⟨stx⟩ then blockTokens text v + else if let some v := DescItemView.of ⟨stx⟩ then mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.term.map (·.raw))) ++ versoTokens text (mkNullNode (v.desc.map (·.raw))) - else if let some v := Lean.Doc.UnorderedListItemView.of ⟨stx⟩ then + else if let some v := UnorderedListItemView.of ⟨stx⟩ then mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.contents.map (·.raw))) - else if let some v := Lean.Doc.OrderedListItemView.of ⟨stx⟩ then + else if let some v := OrderedListItemView.of ⟨stx⟩ then mkTok text .keyword v.marker ++ versoTokens text (mkNullNode (v.contents.map (·.raw))) - else if let some v := Lean.Doc.ArgView.of ⟨stx⟩ then + else if let some v := ArgView.of ⟨stx⟩ then match v with - | .anon _ val => versoTokens text val - | .named _ _ x eq val => + | .anon (val := val) .. => versoTokens text val + | .named (name := x) (assign := eq) (val := val) .. => mkTok text .parameter x ++ mkTok text .keyword eq ++ versoTokens text val - | .flag _ sign x _ => mkTok text .keyword sign ++ mkTok text .parameter x + | .flag (sign := sign) (name := x) .. => + mkTok text .keyword sign ++ mkTok text .parameter x -- An argument value yields no token, so that Lean's own tokens show through. - else if (Lean.Doc.ArgValView.of ⟨stx⟩).isSome then #[] + else if (ArgValView.of ⟨stx⟩).isSome then #[] else Id.run do let mut out := #[] for arg in stx.getArgs do out := out ++ versoTokens text arg return out where - inlineTokens (text : FileMap) : Lean.Doc.InlineView → Array SemanticTokenEntry + inlineTokens (text : FileMap) : InlineView → Array SemanticTokenEntry | .text v => mkTok text .string v.stx | .linebreak _ => #[] | .emph v => @@ -493,11 +500,12 @@ where mkTok text .enumMember v.code.content ++ mkTok text .keyword v.code.closer - targetTokens (text : FileMap) : Lean.Doc.LinkTargetView → Array SemanticTokenEntry - | .url _ o url c | .ref _ o url c => - mkTok text .keyword o ++ mkTok text .parameter url ++ mkTok text .keyword c + targetTokens (text : FileMap) : LinkTargetView → Array SemanticTokenEntry + | .url (opener := o) (url := target) (closer := c) .. + | .ref (opener := o) (name := target) (closer := c) .. => + mkTok text .keyword o ++ mkTok text .parameter target ++ mkTok text .keyword c - blockTokens (text : FileMap) : Lean.Doc.BlockView → Array SemanticTokenEntry + blockTokens (text : FileMap) : BlockView → Array SemanticTokenEntry | .para v => versoTokens text (mkNullNode (v.content.map (·.raw))) | .ul v => versoTokens text (mkNullNode (v.items.map (·.stx.raw))) | .ol v => versoTokens text (mkNullNode (v.items.map (·.stx.raw))) @@ -735,7 +743,8 @@ meta partial def directiveResizings (parents : Array (Syntax × Syntax)) (subject : Syntax) : StateM (Array (Bool × Syntax × Syntax × TextEditBatch)) Unit := do - if let `(block|:::%$opener $_name $_args* { $contents* }%$closer ) := subject then + if let some v := DirectiveView.of ⟨subject⟩ then + let (opener, closer, contents) := (v.opener.raw, v.closer.raw, v.content.map (·.raw)) let parents := parents.push (opener, closer) if onLine opener || onLine closer then if let some edit := parents.flatMapM getIncreases then @@ -783,8 +792,8 @@ where pure (outer ++ inner) getDecreasesIn (stx : Syntax) : Option TextEditBatch := - if let `(block|:::%$opener $_name $_args* {$contents*}%$closer) := stx then - getDecreases (opener, closer) contents + if let some v := DirectiveView.of ⟨stx⟩ then + getDecreases (v.opener.raw, v.closer.raw) (v.content.map (·.raw)) else if let .node _ _ children := stx then children.flatMapM getDecreasesIn else pure #[] @@ -872,8 +881,6 @@ where | `Lean.Doc.Parser.Block.codeblock | `Lean.Doc.Parser.Block.directive | `Lean.Doc.Parser.Block.metadata_block | `Lean.Doc.Parser.Block.blockquote | `Lean.Doc.Parser.Block.ol | `Lean.Doc.Parser.Block.ul | `Lean.Doc.Parser.Block.dl => true - | `Lean.Doc.Syntax.codeblock | `Lean.Doc.Syntax.directive | `Lean.Doc.Syntax.metadata_block | `Lean.Doc.Syntax.blockquote - | `Lean.Doc.Syntax.ol | `Lean.Doc.Syntax.ul | `Lean.Doc.Syntax.dl => true | `Verso.Syntax.codeblock | `Verso.Syntax.directive | `Verso.Syntax.metadata_block | `Verso.Syntax.blockquote | `Verso.Syntax.ol | `Verso.Syntax.ul | `Verso.Syntax.dl => true | _ => false diff --git a/verso/src/verso/Verso/ExpectString.lean b/verso/src/verso/Verso/ExpectString.lean index 6b4c7fff4..c4c32c427 100644 --- a/verso/src/verso/Verso/ExpectString.lean +++ b/verso/src/verso/Verso/ExpectString.lean @@ -19,6 +19,7 @@ public section set_option doc.verso true open Lean Elab +open Lean.Doc namespace Verso.ExpectString @@ -46,10 +47,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectStringOrDiff [Verso.VersoLiteral k] (expected : TSyntax k) (actual : String) +def expectStringOrDiff [VersoLiteral k] (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m (Option MessageData) := do - let expectedLines := (Verso.decode expected).splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do @@ -72,10 +73,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectString [Verso.VersoLiteral k] (what : String) (expected : TSyntax k) (actual : String) +def expectString [VersoLiteral k] (what : String) (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m Bool := do - let expectedLines := (Verso.decode expected).splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do diff --git a/verso/src/verso/Verso/Linters.lean b/verso/src/verso/Verso/Linters.lean index 32ca6d592..694d6c910 100644 --- a/verso/src/verso/Verso/Linters.lean +++ b/verso/src/verso/Verso/Linters.lean @@ -15,7 +15,7 @@ set_option linter.missingDocs true public section open Lean Linter Elab Command -open Lean.Doc.Syntax +open Lean.Doc (BlockView InlineView) /-- Generates curly-quote suggestions -/ register_option linter.typography.quotes : Bool := { @@ -77,7 +77,7 @@ def typography : Linter where logLintIf linter strLit (m!"Use {what} ('{replacement}')" ++ h) discard <| stx.replaceM fun stx => do - let some (.text s) := Lean.Doc.InlineView.of ⟨stx⟩ + let some (.text s) := InlineView.of ⟨stx⟩ | pure none if let some ⟨start, stop⟩ := s.content.raw.getRange? then let mut state : PunctuationState := @@ -190,7 +190,7 @@ def emphasisMinimization : Linter where let text ← getFileMap discard <| stx.replaceM fun stx => do - match Lean.Doc.InlineView.of ⟨stx⟩ with + match InlineView.of ⟨stx⟩ with | some (.emph e) => lintDelimited linter.verso.markup.emph text e.opener e.closer '_' pure none @@ -213,10 +213,10 @@ def codeMinimization : Linter where let text ← getFileMap discard <| stx.replaceM fun stx => do - if let some (.code c) := Lean.Doc.InlineView.of ⟨stx⟩ then + if let some (.code c) := InlineView.of ⟨stx⟩ then lintDelimited linter.verso.markup.code text c.opener c.closer '`' pure none - else if let some (.codeblock c) := Lean.Doc.BlockView.of ⟨stx⟩ then + else if let some (.codeblock c) := BlockView.of ⟨stx⟩ then lintDelimited linter.verso.markup.codeBlock text c.openFence c.closeFence '`' (minimal := 3) pure none else pure none diff --git a/verso/src/verso/Verso/Parser.lean b/verso/src/verso/Verso/Parser.lean index 2dc1a39b4..1d5faf86e 100644 --- a/verso/src/verso/Verso/Parser.lean +++ b/verso/src/verso/Verso/Parser.lean @@ -18,7 +18,6 @@ public section namespace Verso.Parser open Verso.SyntaxUtils -open Lean.Doc.Syntax open Lean Parser export Lean.Doc.Parser (skipFn ignoreFn) @@ -306,21 +305,6 @@ export Lean.Doc.Parser ( /-- One or more inline elements. With `allowNewlines`, they may continue onto the following lines. -/ def textLine (allowNewlines := true) : ParserFn := many1Fn (inlineFn { allowNewlines }) -/-- An inline element. -/ -def inline (ctxt : InlineCtxt) : ParserFn := inlineFn ctxt - -/-- A block element. -/ -def block (ctxt : BlockCtxt) : ParserFn := blockFn ctxt - -/-- Zero or more block elements. -/ -def blocks (ctxt : BlockCtxt) : ParserFn := blocksFn ctxt - -/-- One or more block elements. -/ -def blocks1 (ctxt : BlockCtxt) : ParserFn := blocks1Fn ctxt - -/-- A metadata block. -/ -def metadataBlock : ParserFn := metadataBlockFn - /-- Some number of blank lines followed by zero or more blocks. @@ -349,6 +333,6 @@ public def stringToInlines [Monad m] [MonadFileMap m] [MonadError m] [MonadEnv m open Lean Elab Term in public def stringToBlocks [Monad m] [MonadFileMap m] [MonadError m] [MonadEnv m] [MonadQuotation m] (s : StrLit) : m (Array Syntax) := withRef s do - return (← parseMarkupStrLit (blocks {}) s).getArgs + return (← parseMarkupStrLit (blocksFn {}) s).getArgs end Verso.Doc.Concrete diff --git a/verso/src/verso/Verso/SyntaxUtils.lean b/verso/src/verso/Verso/SyntaxUtils.lean index bfac2f2ee..904037348 100644 --- a/verso/src/verso/Verso/SyntaxUtils.lean +++ b/verso/src/verso/Verso/SyntaxUtils.lean @@ -15,6 +15,7 @@ namespace Verso.Parser -- TODO: make public upstream and delete these open Lean Doc Parser +open Lean.Doc public def nl := satisfyFn (· == '\n') "newline" @@ -155,7 +156,7 @@ public def strLitInputContext [Monad m] [MonadFileMap m] [MonadError m] (str : S Given a string literal, constructs a Lean string that can be parsed by the Lean parser, yielding correct source positions for items in the string literal. -/ -public def parserInputString [Monad m] [MonadFileMap m] [Verso.VersoLiteral k] +public def parserInputString [Monad m] [MonadFileMap m] [VersoLiteral k] (str : TSyntax k) : m String := do let text ← getFileMap @@ -173,7 +174,7 @@ public def parserInputString [Monad m] [MonadFileMap m] [Verso.VersoLiteral k] let strOriginal? : Option String := do let ⟨start, stop⟩ ← str.raw.getRange? start.extract text.source stop - code := code ++ strOriginal?.getD (Verso.decode str) + code := code ++ strOriginal?.getD (decode str) return code @@ -336,9 +337,9 @@ actual string contents. When the literal's source text differs from its contents sequences, the decoded contents are parsed and the resulting positions are mapped back to the source. -/ -public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Verso.VersoLiteral k] (p : ParserFn) (input : TSyntax k) : m Syntax := do +public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [VersoLiteral k] (p : ParserFn) (input : TSyntax k) : m Syntax := do let text ← getFileMap - let inputText := Verso.decode input + let inputText := decode input if let some startPos := input.raw.getPos? then let endPos := input.raw.getTailPos?.getD startPos let stopPos := if endPos > text.source.rawEndPos then text.source.rawEndPos else endPos @@ -384,7 +385,7 @@ Parses an original string literal as part of a syntax category. The provided string literal is used only for source positions; the `FileMap` is used to acquire the actual string contents. -/ -public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Verso.VersoLiteral k] (catName : Name) (input : TSyntax k) : m Syntax := +public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [VersoLiteral k] (catName : Name) (input : TSyntax k) : m Syntax := parseStrLitWith (andthenFn whitespace (categoryParserFnImpl catName)) input /-- diff --git a/verso/test-projects/website/DemoSite/About.lean b/verso/test-projects/website/DemoSite/About.lean index e4cdc43f2..52f1d8b39 100644 --- a/verso/test-projects/website/DemoSite/About.lean +++ b/verso/test-projects/website/DemoSite/About.lean @@ -10,9 +10,9 @@ open Verso Genre Blog section open Verso Doc Elab ArgParse open Lean +open Lean.Doc (DescListView DescItemView TextView ImageView) open Verso Output Html open Template -open scoped Lean.Doc.Syntax set_option pp.rawOnError true @@ -54,20 +54,20 @@ def galleryImpl : DirectiveExpanderOf Unit let #[stx] := stxs | logErrorAt (mkNullNode stxs) "Expected one block" return (← `(sorry)) - let some dl := Lean.Doc.DescListView.of stx + let some dl := DescListView.of stx | throwErrorAt stx "Expected definition list" let items ← dl.items.mapM getItem ``(Block.other (Blog.BlockExt.component $(quote `gallery) Json.null) #[$(items),*]) where - getItem (item : Lean.Doc.DescItemView) : DocElabM Term := do + getItem (item : DescItemView) : DocElabM Term := do let #[inl] := item.term.filter (fun i => - match Lean.Doc.TextView.of i with + match TextView.of i with | some t => t.getVersoText.any (not ∘ Char.isWhitespace) | none => true) | throwErrorAt (mkNullNode (item.term.map (·.raw))) "Expected one inline" - let some img := Lean.Doc.ImageView.of inl + let some img := ImageView.of inl | throwErrorAt inl "Expected an image" - let .url _ _ url _ := img.target + let .url (url := url) .. := img.target | throwErrorAt inl "Expected an image with a URL" let some desc := item.desc[0]? | throwErrorAt item.stx "Expected a description" From 290c2c4036a5ab304a296c54d7cbf0670f5d30f5 Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 16:53:20 +0200 Subject: [PATCH 06/15] cleanup verso Literal class --- .../src/verso-illuminate/VersoIlluminate.lean | 2 +- .../verso-manual/VersoManual/InlineLean.lean | 12 +++++------ .../VersoManual/InlineLean/LongLines.lean | 2 +- verso/src/verso/Verso/ExpectString.lean | 8 ++++---- verso/src/verso/Verso/Literal.lean | 20 +++++++++---------- verso/src/verso/Verso/SyntaxUtils.lean | 10 +++++----- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/verso/src/verso-illuminate/VersoIlluminate.lean b/verso/src/verso-illuminate/VersoIlluminate.lean index 76e517004..cb410a4aa 100644 --- a/verso/src/verso-illuminate/VersoIlluminate.lean +++ b/verso/src/verso-illuminate/VersoIlluminate.lean @@ -146,7 +146,7 @@ section variables for the Manual genre). It defaults to the identity. Genre-specific code-block expanders call this to do the shared evaluation work and then emit their own `GenreDiagram.diagramBlock` term. -/ -public def elabAndStoreDiagram [VersoLiteral k] (str : TSyntax k) +public def elabAndStoreDiagram [Verso.Literal k] (str : TSyntax k) (scope : {α : Type} → TermElabM α → TermElabM α := fun act => act) : DocElabM (String × Float) := do let stx ← parseStrLitAsCategory `term str diff --git a/verso/src/verso-manual/VersoManual/InlineLean.lean b/verso/src/verso-manual/VersoManual/InlineLean.lean index 1a7860127..ed641265b 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean.lean @@ -200,7 +200,7 @@ private meta def quoteHighlightViaSerialization (hls : Highlighted) : DocElabM T De-indents and returns (syntax of) a Block representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanBlock [VersoLiteral k] (shouldShow : Bool) +private meta def toHighlightedLeanBlock [Literal k] (shouldShow : Bool) (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Block.concat #[]) @@ -214,18 +214,18 @@ private meta def toHighlightedLeanBlock [VersoLiteral k] (shouldShow : Bool) let range := range.map (← getFileMap).utf8RangeToLspRange ``(Block.other (Block.lean $(← quoteHighlightViaSerialization hls) (some $(quote (← getFileName))) $(quote range)) - #[Block.code $(quote (decode str))]) + #[Block.code $(quote (Literal.decode str))]) /-- Returns (syntax of) an Inline representation containing highlighted Lean code. The argument `hls` must be a highlighting of the parsed string `str`. -/ -private meta def toHighlightedLeanInline [VersoLiteral k] (shouldShow : Bool) +private meta def toHighlightedLeanInline [Literal k] (shouldShow : Bool) (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Inline.concat #[]) - ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote (decode str))]) + ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(← quoteHighlightViaSerialization hls)) #[Inline.code $(quote (Literal.decode str))]) /-- @@ -248,14 +248,14 @@ private meta partial def disableUnusedVarLinterInInfoTree : InfoTree → InfoTre .node info (children.map disableUnusedVarLinterInInfoTree) | .hole id => .hole id -meta def elabCommands [VersoLiteral k] (config : LeanBlockConfig) (str : TSyntax k) +meta def elabCommands [Literal k] (config : LeanBlockConfig) (str : TSyntax k) (toHighlightedLeanContent : (shouldShow : Bool) → (hls : Highlighted) → (str : TSyntax k) → DocElabM Term) (minCommands : Option Nat := none) (maxCommands : Option Nat := none) : DocElabM Term := withoutAsync <| do - PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (decode str))) + PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (Literal.decode str))) (kind := Lsp.SymbolKind.file) (detail? := some ("Lean code" ++ config.outlineMeta)) diff --git a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean index 92429f938..72dd734fa 100644 --- a/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean +++ b/verso/src/verso-manual/VersoManual/InlineLean/LongLines.lean @@ -40,7 +40,7 @@ def warnLongLines [Monad m] [MonadLog m] [AddMessageContext m] [MonadOptions m] let lines := code.getVersoCodeBlockLines for h : i in [0:lines.size] do let line := lines[i] - let width := line.getVersoCodeBlockLine.trimAsciiEnd.length + let width := line.getVersoCodeBlockLine.trimAsciiEnd.positions.length if width > maxCodeColumns then let note := MessageData.note m!"Example code is shown on mobile devices and other narrow contexts. \ diff --git a/verso/src/verso/Verso/ExpectString.lean b/verso/src/verso/Verso/ExpectString.lean index c4c32c427..97ded1f4e 100644 --- a/verso/src/verso/Verso/ExpectString.lean +++ b/verso/src/verso/Verso/ExpectString.lean @@ -47,10 +47,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectStringOrDiff [VersoLiteral k] (expected : TSyntax k) (actual : String) +def expectStringOrDiff [Literal k] (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m (Option MessageData) := do - let expectedLines := (decode expected).splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (Literal.decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do @@ -73,10 +73,10 @@ are good candidates for {name}`preEq`. Errors are logged, not thrown; the returned {name}`Bool` indicates whether an error was logged. -/ -def expectString [VersoLiteral k] (what : String) (expected : TSyntax k) (actual : String) +def expectString [Literal k] (what : String) (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m Bool := do - let expectedLines := (decode expected).splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (Literal.decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do diff --git a/verso/src/verso/Verso/Literal.lean b/verso/src/verso/Verso/Literal.lean index 46d030917..41ac3248b 100644 --- a/verso/src/verso/Verso/Literal.lean +++ b/verso/src/verso/Verso/Literal.lean @@ -19,42 +19,40 @@ A syntax kind whose tokens denote a string. Each kind writes its string with delimiters and escapes of its own, so code that reads a literal's text, or replaces it with new text, works at any of them. -/ -class VersoLiteral (k : SyntaxNodeKinds) where +class Literal (k : SyntaxNodeKinds) where /-- The text that `stx` denotes. -/ decode (stx : TSyntax k) : String /-- A token at `src`'s position that denotes `value`. -/ encode (src : Syntax) (value : String) : TSyntax k -export VersoLiteral (decode encode) - -instance : VersoLiteral strLitKind where +instance : Literal strLitKind where decode stx := stx.getString encode src value := Syntax.mkStrLit value (info := src.getHeadInfo) -instance : VersoLiteral ``Doc.Parser.versoText where +instance : Literal ``Doc.Parser.versoText where decode stx := stx.getVersoText encode src value := Doc.mkVersoTextFrom src value -instance : VersoLiteral ``Doc.Parser.versoRef where +instance : Literal ``Doc.Parser.versoRef where decode stx := stx.getVersoRefName encode src value := Doc.mkVersoRefNameFrom src value -instance : VersoLiteral ``Doc.Parser.versoLinkUrl where +instance : Literal ``Doc.Parser.versoLinkUrl where decode stx := stx.getVersoLinkUrl encode src value := Doc.mkVersoLinkUrlFrom src value -instance : VersoLiteral ``Doc.Parser.versoLinkRefUrl where +instance : Literal ``Doc.Parser.versoLinkRefUrl where decode stx := stx.getVersoLinkRefUrl encode src value := Doc.mkVersoLinkRefUrlFrom src value -instance : VersoLiteral ``Doc.Parser.versoImageAlt where +instance : Literal ``Doc.Parser.versoImageAlt where decode stx := stx.getVersoImageAlt encode src value := Doc.mkVersoImageAltFrom src value -instance : VersoLiteral ``Doc.Parser.versoCode where +instance : Literal ``Doc.Parser.versoCode where decode stx := stx.getVersoCode encode src value := Doc.mkVersoCodeFrom src value -instance : VersoLiteral ``Doc.Parser.versoCodeBlock where +instance : Literal ``Doc.Parser.versoCodeBlock where decode stx := stx.getVersoCodeBlock encode src value := Doc.mkVersoCodeBlockFrom src value diff --git a/verso/src/verso/Verso/SyntaxUtils.lean b/verso/src/verso/Verso/SyntaxUtils.lean index 904037348..57964f28f 100644 --- a/verso/src/verso/Verso/SyntaxUtils.lean +++ b/verso/src/verso/Verso/SyntaxUtils.lean @@ -156,7 +156,7 @@ public def strLitInputContext [Monad m] [MonadFileMap m] [MonadError m] (str : S Given a string literal, constructs a Lean string that can be parsed by the Lean parser, yielding correct source positions for items in the string literal. -/ -public def parserInputString [Monad m] [MonadFileMap m] [VersoLiteral k] +public def parserInputString [Monad m] [MonadFileMap m] [Literal k] (str : TSyntax k) : m String := do let text ← getFileMap @@ -174,7 +174,7 @@ public def parserInputString [Monad m] [MonadFileMap m] [VersoLiteral k] let strOriginal? : Option String := do let ⟨start, stop⟩ ← str.raw.getRange? start.extract text.source stop - code := code ++ strOriginal?.getD (decode str) + code := code ++ strOriginal?.getD (Literal.decode str) return code @@ -337,9 +337,9 @@ actual string contents. When the literal's source text differs from its contents sequences, the decoded contents are parsed and the resulting positions are mapped back to the source. -/ -public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [VersoLiteral k] (p : ParserFn) (input : TSyntax k) : m Syntax := do +public def parseStrLitWith [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Literal k] (p : ParserFn) (input : TSyntax k) : m Syntax := do let text ← getFileMap - let inputText := decode input + let inputText := Literal.decode input if let some startPos := input.raw.getPos? then let endPos := input.raw.getTailPos?.getD startPos let stopPos := if endPos > text.source.rawEndPos then text.source.rawEndPos else endPos @@ -385,7 +385,7 @@ Parses an original string literal as part of a syntax category. The provided string literal is used only for source positions; the `FileMap` is used to acquire the actual string contents. -/ -public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [VersoLiteral k] (catName : Name) (input : TSyntax k) : m Syntax := +public def parseStrLitAsCategory [Monad m] [MonadLog m] [MonadEnv m] [MonadOptions m] [MonadError m] [AddMessageContext m] [Literal k] (catName : Name) (input : TSyntax k) : m Syntax := parseStrLitWith (andthenFn whitespace (categoryParserFnImpl catName)) input /-- From 9263d6836f223b01792e73773c5c710419b29bd4 Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 16:54:28 +0200 Subject: [PATCH 07/15] fix: adapt Mathlib to new Verso parser --- mathlib4/Mathlib/Tactic/Linter/DocString.lean | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathlib4/Mathlib/Tactic/Linter/DocString.lean b/mathlib4/Mathlib/Tactic/Linter/DocString.lean index 2fcbff3df..d0144c862 100644 --- a/mathlib4/Mathlib/Tactic/Linter/DocString.lean +++ b/mathlib4/Mathlib/Tactic/Linter/DocString.lean @@ -92,7 +92,7 @@ def checkVersoSyntax (docComment : String) (fileName : Option String := none) : openDecls := (← getOpenDecls) } let s := mkParserState docComment - let s := Doc.Parser.document.run ictx pmctx (getTokenTable env) s + let s := (Doc.Parser.documentFn {}).run ictx pmctx (getTokenTable env) s return s.allErrors /-- From 79de323e2329061b4518db43f9b4828d53021e38 Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Wed, 9 Sep 2026 15:00:05 +0000 Subject: [PATCH 08/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index a8fcba0d6..6b93c2d94 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4-pr-releases:pr-release-15064-62f3bbf +leanprover/lean4-pr-releases:pr-release-15064-2a13699 From bd963cf0bfbbe41a22e59417a62bc8b599808c42 Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 17:11:33 +0200 Subject: [PATCH 09/15] fix: adapt the reference manual to the Verso parser rewrite Every element the manual reads is a view: `CodeView`, `CodeBlockView` and `RoleView` replace the quotations against the old encoding, which compiled but no longer matched. `Manual.namedCodeBlock` covers the common case of a code block matching one name. State that held a `StrLit` now holds the content token it came from, and the category names `inline` and `block` are the parser's own. --- reference-manual/Manual/Meta.lean | 26 ++++--- reference-manual/Manual/Meta/Attribute.lean | 16 ++-- reference-manual/Manual/Meta/Basic.lean | 25 ++++-- reference-manual/Manual/Meta/ConfigFile.lean | 6 +- reference-manual/Manual/Meta/CustomStyle.lean | 2 +- reference-manual/Manual/Meta/ElanCmd.lean | 14 ++-- reference-manual/Manual/Meta/ElanOpt.lean | 13 ++-- reference-manual/Manual/Meta/Env.lean | 6 +- .../Manual/Meta/ErrorExplanation/Example.lean | 13 +++- reference-manual/Manual/Meta/Example.lean | 43 ++++++----- .../Manual/Meta/ExpectString.lean | 5 +- reference-manual/Manual/Meta/Figure.lean | 2 +- reference-manual/Manual/Meta/LakeCmd.lean | 14 ++-- reference-manual/Manual/Meta/LakeLean.lean | 11 +-- .../Manual/Meta/LakeManifest.lean | 8 +- reference-manual/Manual/Meta/LakeOpt.lean | 12 +-- reference-manual/Manual/Meta/LakeSession.lean | 30 ++++---- reference-manual/Manual/Meta/LakeToml.lean | 18 ++--- .../Manual/Meta/LakeToml/Toml.lean | 10 +-- reference-manual/Manual/Meta/Lean.lean | 12 +-- reference-manual/Manual/Meta/LexedText.lean | 8 +- reference-manual/Manual/Meta/Markdown.lean | 6 +- .../Manual/Meta/ModuleExample.lean | 76 ++++++++++--------- .../Manual/Meta/Monotonicity.lean | 1 - reference-manual/Manual/Meta/Namespace.lean | 6 +- .../Manual/Meta/SpliceContents.lean | 8 +- reference-manual/Manual/Meta/Syntax.lean | 54 +++++++------ reference-manual/Manual/Meta/Tactics.lean | 56 +++++++------- 28 files changed, 262 insertions(+), 239 deletions(-) diff --git a/reference-manual/Manual/Meta.lean b/reference-manual/Manual/Meta.lean index 6887555ff..806b66805 100644 --- a/reference-manual/Manual/Meta.lean +++ b/reference-manual/Manual/Meta.lean @@ -48,8 +48,8 @@ open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open SubVerso.Highlighting Highlighted open Lean Elab open Lean.Elab.Tactic.GuardMsgs +open Lean.Doc (CodeView CodeBlockView) -open scoped Lean.Doc.Syntax namespace Manual @@ -71,10 +71,15 @@ def commentDirective : DirectiveExpander -- These are part commands rather than block expanders so that it can be used in contexts where -- block content doesn't fit, like right after an include. However, the blocks are still needed -- for contexts where part commands aren't run. -@[part_command Lean.Doc.Syntax.codeblock, part_command Lean.Doc.Syntax.directive] +@[part_command Lean.Doc.Parser.Block.codeblock, part_command Lean.Doc.Parser.Block.directive] def commentBlock : PartCommand - | `(block| ::: $commentId $_* { $_* } ) - | `(block| ``` $commentId $_* | $_ ``` ) => do + | v => do + let some commentId := + (match v with + | .directive d => some d.name + | .codeblock c => c.name? + | _ => none) + | throwUnsupportedSyntax try let n ← realizeGlobalConstNoOverloadWithInfo commentId if n == ``comment then @@ -82,7 +87,6 @@ def commentBlock : PartCommand else throwUnsupportedSyntax catch | _ => throwUnsupportedSyntax - | _ => throwUnsupportedSyntax @@ -351,9 +355,9 @@ def ctype : RoleExpander ArgParse.done.run args let #[x] := contents | throwError "Expected exactly one parameter" - let `(inline|code($t)) := x + let some { content := t, .. } := CodeView.of x | throwError "Expected exactly one code item" - pure #[← ``(Inline.code $(quote t.getString))] + pure #[← ``(Inline.code $(quote t.getVersoCode))] def Inline.ckw : Inline where name := `Manual.ckw @@ -367,9 +371,9 @@ def ckw : RoleExpander ArgParse.done.run args let #[x] := contents | throwError "Expected exactly one parameter" - let `(inline|code($t)) := x + let some { content := t, .. } := CodeView.of x | throwError "Expected exactly one code item" - pure #[← ``(Inline.code $(quote t.getString))] + pure #[← ``(Inline.code $(quote t.getVersoCode))] @[inline_extension ckw] def ckw.descr : InlineDescr where @@ -392,10 +396,10 @@ def ffi : DirectiveExpander else let firstBlock := blocks[0] let moreBlocks := blocks.extract 1 blocks.size - let `(block|``` | $contents ```) := firstBlock + let some { name? := none, content := contents, .. } := CodeBlockView.of firstBlock | throwErrorAt firstBlock "Expected code block" let body ← moreBlocks.mapM elabBlock - pure #[← `(Block.other {Block.ffi with data := ToJson.toJson ($(quote config.name), $(quote config.kind), $(quote contents.getString))} #[$body,*])] + pure #[← `(Block.other {Block.ffi with data := ToJson.toJson ($(quote config.name), $(quote config.kind), $(quote contents.getVersoCodeBlock))} #[$body,*])] @[block_extension ffi] def ffi.descr : BlockDescr where diff --git a/reference-manual/Manual/Meta/Attribute.lean b/reference-manual/Manual/Meta/Attribute.lean index f93a65ee8..714fdf54c 100644 --- a/reference-manual/Manual/Meta/Attribute.lean +++ b/reference-manual/Manual/Meta/Attribute.lean @@ -18,13 +18,13 @@ open Verso.ArgParse open Verso.Code (highlightingJs) open Verso.Code.Highlighted.WebAssets -open scoped Lean.Doc.Syntax open Lean Elab Parser open Lean.Widget (TaggedText) open SubVerso.Highlighting open Verso.Code +open Lean.Doc (CodeView) namespace Manual @@ -64,7 +64,7 @@ def attr : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $a:str )) := arg + let some { content := a, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the attribute" let altStr ← parserInputString a @@ -72,14 +72,14 @@ def attr : RoleExpander | .error e => -- Attributes whose syntax requires arguments (e.g. `export`) don't parse from their bare name. -- When the name is a leading token, refer to that syntax and link to its docs. - match attrSyntaxKind? (← getEnv) a.getString with + match attrSyntaxKind? (← getEnv) a.getVersoCode with | some kind => let kindDoc ← findDocString? (← getEnv) kind pure #[← `(Verso.Doc.Inline.other { Inline.keywordOf with data := ToJson.toJson (α := String × Option Name × Name × Option String) - ($(quote a.getString), $(quote (some `attr)), $(quote kind), $(quote kindDoc)) - } #[Verso.Doc.Inline.code $(quote a.getString)])] + ($(quote a.getVersoCode), $(quote (some `attr)), $(quote kind), $(quote kindDoc)) + } #[Verso.Doc.Inline.code $(quote a.getVersoCode)])] | none => throwErrorAt a e | .ok stx => let attrName ← @@ -93,7 +93,7 @@ def attr : RoleExpander match getAttributeImpl (← getEnv) attrName with | .error e => throwErrorAt a e | .ok {descr, name, ref, ..} => do - let attrTok := a.getString + let attrTok := a.getVersoCode let hl : Highlighted := attrToken ref descr attrTok try -- Attempt to add info to the document source for go-to-def and the like, but this doesn't @@ -139,7 +139,7 @@ def attrs : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $a:str )) := arg + let some { content := a, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the attribute application syntax" let altStr ← parserInputString a @@ -183,7 +183,7 @@ def attrs : RoleExpander catch _ => pure () hl := .token ⟨.keyword ``Term.attributes none none, "@["⟩ ++ hl ++ .token ⟨.keyword ``Term.attributes none none, "]"⟩ - pure #[← `(Verso.Doc.Inline.other {Inline.attrs with data := ToJson.toJson $(quote hl)} #[Verso.Doc.Inline.code $(quote a.getString)])] + pure #[← `(Verso.Doc.Inline.other {Inline.attrs with data := ToJson.toJson $(quote hl)} #[Verso.Doc.Inline.code $(quote a.getVersoCode)])] where -- TODO: This will eventually generate the right cross-reference, but VersoManual needs to have a diff --git a/reference-manual/Manual/Meta/Basic.lean b/reference-manual/Manual/Meta/Basic.lean index a52a4524d..3e3f4f67e 100644 --- a/reference-manual/Manual/Meta/Basic.lean +++ b/reference-manual/Manual/Meta/Basic.lean @@ -9,8 +9,10 @@ public import Lean.Syntax public import Lean.Environment public import Lean.Parser.Types public import Lean.Elab.Command +public import Lean.DocString.View import Lean.Parser +public import Verso.Literal import Verso.Parser import Verso.Doc.ArgParse import SubVerso.Highlighting @@ -19,8 +21,9 @@ open Lean namespace Manual -public def parserInputString [Monad m] [MonadFileMap m] - (str : TSyntax `str) : +open Verso in +public def parserInputString [Monad m] [MonadFileMap m] [Literal k] + (str : TSyntax k) : m String := do let text ← getFileMap let preString := String.Pos.Raw.extract text.source 0 (str.raw.getPos?.getD 0) @@ -34,7 +37,7 @@ public def parserInputString [Monad m] [MonadFileMap m] let strOriginal? : Option String := do let ⟨start, stop⟩ ← str.raw.getRange? start.extract text.source stop - code := code ++ strOriginal?.getD str.getString + code := code ++ strOriginal?.getD (Literal.decode str) return code public structure SyntaxError where @@ -156,20 +159,30 @@ public def commandWithoutAsync : (act : CommandElabM α) → CommandElabM α := public def withoutAsync [Monad m] [MonadWithOptions m] : (act : m α) → m α := withOptions (Elab.async.set · false) -open scoped Lean.Doc.Syntax in +open Lean.Doc (CodeBlockView VersoCodeBlock) in +/-- +The contents of `blk`, if it is a code block that names `name`. +-/ +public def namedCodeBlock (name : Name) (blk : TSyntax ``Lean.Doc.Parser.block) : + Option VersoCodeBlock := + match CodeBlockView.of blk with + | some { name? := some n, content, .. } => if n.getId == name then some content else none + | _ => none + +open Lean.Doc (CodeView VersoCode) in /-- If the array of inlines contains a single code element, it is returned. Otherwise, an error is logged and `none` is returned. -/ public def oneCodeStr? [Monad m] [MonadError m] [MonadLog m] [AddMessageContext m] [MonadOptions m] - (inlines : Array (TSyntax `inline)) : m (Option StrLit) := do + (inlines : TSyntaxArray ``Lean.Doc.Parser.inline) : m (Option VersoCode) := do let #[code] := inlines | if inlines.size == 0 then Lean.logError "Expected a code element" else logErrorAt (mkNullNode inlines) "Expected one code element" return none - let `(inline|code($code)) := code + let some { content := code, .. } := CodeView.of code | logErrorAt code "Expected a code element" return none return some code diff --git a/reference-manual/Manual/Meta/ConfigFile.lean b/reference-manual/Manual/Meta/ConfigFile.lean index 3e1f51977..7b482f8d6 100644 --- a/reference-manual/Manual/Meta/ConfigFile.lean +++ b/reference-manual/Manual/Meta/ConfigFile.lean @@ -65,17 +65,17 @@ inline_extension Inline.configFile (filename : String) where toTeX := none open Verso.Doc.Elab -open Lean.Doc.Syntax open Lean +open Lean.Doc (CodeView) @[role] def configFile : RoleExpanderOf Unit | (), inlines => do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $cmdName:str )) := arg + let some { content := cmdName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the config file's name" - let filename := cmdName.getString + let filename := cmdName.getVersoCode `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.configFile $(quote filename)) #[.code $(quote filename)]) diff --git a/reference-manual/Manual/Meta/CustomStyle.lean b/reference-manual/Manual/Meta/CustomStyle.lean index 599420d67..adf16d887 100644 --- a/reference-manual/Manual/Meta/CustomStyle.lean +++ b/reference-manual/Manual/Meta/CustomStyle.lean @@ -33,4 +33,4 @@ block_extension Block.customCSS (css : String) where @[code_block] def customCSS : CodeBlockExpanderOf Unit | (), str => - `(Block.other (Block.customCSS $(quote str.getString)) #[]) + `(Block.other (Block.customCSS $(quote str.getVersoCodeBlock)) #[]) diff --git a/reference-manual/Manual/Meta/ElanCmd.lean b/reference-manual/Manual/Meta/ElanCmd.lean index d586d17af..099b06908 100644 --- a/reference-manual/Manual/Meta/ElanCmd.lean +++ b/reference-manual/Manual/Meta/ElanCmd.lean @@ -9,7 +9,7 @@ import Manual.Meta.LakeCmd -- TODO: generalize the common parts into a library t open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open Lean Elab open SubVerso.Highlighting Highlighted -open scoped Lean.Doc.Syntax +open Lean.Doc (CodeView) namespace Manual @@ -204,9 +204,9 @@ def elanMeta : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $mName:str )) := arg + let some { content := mName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the metavariable" - let mName := mName.getString + let mName := mName.getVersoCode pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other {Manual.Inline.elanMeta with data := Json.arr #[$(quote mName), .null]} #[Inline.code $(quote mName)])] @@ -240,9 +240,9 @@ def elanInline : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $cmdName:str )) := arg + let some { content := cmdName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the Elan command name" - let name := cmdName.getString + let name := cmdName.getVersoCode pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other {Manual.Inline.elan with data := $(quote name)} #[Inline.code $(quote name)])] @@ -289,10 +289,10 @@ def elanArgs : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $spec:str )) := arg + let some { content := spec, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the Elan command name" - match Parser.runParserCategory (← getEnv) `lake_cmd_spec spec.getString (← getFileName) with + match Parser.runParserCategory (← getEnv) `lake_cmd_spec spec.getVersoCode (← getFileName) with | .error e => throwErrorAt spec e | .ok stx => match CommandSpec.ofSyntax stx with diff --git a/reference-manual/Manual/Meta/ElanOpt.lean b/reference-manual/Manual/Meta/ElanOpt.lean index 5aed8e74b..f03d91895 100644 --- a/reference-manual/Manual/Meta/ElanOpt.lean +++ b/reference-manual/Manual/Meta/ElanOpt.lean @@ -20,8 +20,9 @@ import Manual.Meta.Basic open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets -open Lean.Doc.Syntax open Lean Elab +open Lean.Doc (CodeView) + namespace Manual inductive ElanOptKind where @@ -84,9 +85,9 @@ def elanOptDef : RoleExpander let {kind} ← ElanOptDefOpts.parse.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $name:str )) := arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the option or flag" - let origName := name.getString + let origName := name.getVersoCode let name := origName.takeWhile fun c => c == '-' || c.isAlphanum let name := name.copy let valMeta := origName.drop name.length |>.dropWhile fun (c : Char) => !c.isAlphanum @@ -143,12 +144,12 @@ def elanOpt : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $name:str )) := arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the option or flag" - let optName := name.getString.takeWhile fun c => c == '-' || c.isAlphanum + let optName := name.getVersoCode.takeWhile fun c => c == '-' || c.isAlphanum let optName := optName.copy - pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.elanOpt $(quote optName) $(quote name.getString)) #[Inline.code $(quote name.getString)])] + pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.elanOpt $(quote optName) $(quote name.getVersoCode)) #[Inline.code $(quote name.getVersoCode)])] @[inline_extension elanOpt] def elanOpt.descr : InlineDescr where diff --git a/reference-manual/Manual/Meta/Env.lean b/reference-manual/Manual/Meta/Env.lean index 23c5cccce..8bd671eb8 100644 --- a/reference-manual/Manual/Meta/Env.lean +++ b/reference-manual/Manual/Meta/Env.lean @@ -19,11 +19,11 @@ import SubVerso.Examples import Manual.Meta.Basic -open Lean.Doc.Syntax open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open SubVerso.Highlighting Highlighted open Lean Elab open Lean.Elab.Tactic.GuardMsgs +open Lean.Doc (CodeView) namespace Manual @@ -38,9 +38,9 @@ def envVar : RoleExpander let isDef ← parseOpts.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $varName:str )) := arg + let some { content := varName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the environment variable" - let v := varName.getString + let v := varName.getVersoCode pure #[← `(.other {Manual.Inline.envVar with data := Json.arr #[.str $(quote v), .bool $(quote isDef)] } #[Inline.code $(quote v)])] where diff --git a/reference-manual/Manual/Meta/ErrorExplanation/Example.lean b/reference-manual/Manual/Meta/ErrorExplanation/Example.lean index be8f1f400..322bd74c1 100644 --- a/reference-manual/Manual/Meta/ErrorExplanation/Example.lean +++ b/reference-manual/Manual/Meta/ErrorExplanation/Example.lean @@ -9,6 +9,8 @@ import Manual.Meta.Example open Lean open Verso (reportError) +open Lean.Doc (CodeBlockView VersoCodeBlock) +open Manual (namedCodeBlock) set_option doc.verso true /- @@ -194,12 +196,12 @@ def errorExample : Verso.Doc.Elab.DirectiveExpanderOf ErrorExampleConfig | { title }, contents => do let brokenStx :: restStx := contents.toList | throwError m!"The error example had no contents" - let `(Lean.Doc.Syntax.codeblock|``` broken| $brokenTxt ```) := brokenStx + let some brokenTxt := namedCodeBlock `broken brokenStx | throwErrorAt brokenStx m!"First element in errorExample must be a `broken` codeblock containing the broken minimal working example" let errorStx :: restStx := restStx | throwError m!"The error example did not contain a second element" - let `(Lean.Doc.Syntax.codeblock|``` output| $errorTxt ```) := errorStx + let some errorTxt := namedCodeBlock `output errorStx | throwErrorAt errorStx m!"Second element in errorExample must be an `output` codeblock containing the generated error message" let brokenBlock ← brokenTxt |> Verso.Genre.Manual.InlineLean.lean @@ -234,12 +236,15 @@ def errorExample : Verso.Doc.Elab.DirectiveExpanderOf ErrorExampleConfig Doc.Block.other (Manual.Block.tabbedErrorReproduction $(quote tabbedContentHeaders.toArray)) #[$tabbedContentBlocks.toArray,*], $narrativeBlocks.toArray,*]) where - partitionFixed (blocks: List (TSyntax `block)) : Verso.Doc.Elab.DocElabM (List (Syntax × Option StrLit × TSyntax `str) × List (TSyntax `block)) := do + partitionFixed (blocks : List (TSyntax ``Lean.Doc.Parser.block)) : + Verso.Doc.Elab.DocElabM + (List (Syntax × Option StrLit × VersoCodeBlock) × List (TSyntax ``Lean.Doc.Parser.block)) := do match blocks with | [] => pure ([], []) | block :: rest => - let `(Lean.Doc.Syntax.codeblock|``` fixed $args*| $fixedTxt ```) := block + let some { name? := some name, args, content := fixedTxt, .. } := CodeBlockView.of block | return ([], blocks) + unless name.getId == `fixed do return ([], blocks) let parsedArgs ← Verso.Doc.Elab.parseArgs args let arg? : Option _ ← match parsedArgs.toList with | [] => pure none diff --git a/reference-manual/Manual/Meta/Example.lean b/reference-manual/Manual/Meta/Example.lean index 94770bbfa..fca0c8b3c 100644 --- a/reference-manual/Manual/Meta/Example.lean +++ b/reference-manual/Manual/Meta/Example.lean @@ -12,9 +12,9 @@ import Lean.Elab.InfoTree.Types open Verso Doc Elab open Verso.Genre Manual open Verso.ArgParse -open Lean.Doc.Syntax open Lean Elab +open Lean.Doc (CodeBlockView) namespace Manual @@ -28,7 +28,7 @@ def Block.example (descriptionString : String) (name : Option String) (opened : abbrev ExampleBlockJson := String × Option String × Bool × Option Tag × Option String structure ExampleConfig where - description : TSyntaxArray `inline + description : TSyntaxArray ``Lean.Doc.Parser.inline /-- Name for refs -/ tag : Option String := none keep : Bool := false @@ -61,29 +61,30 @@ def prioritizedElab [Monad m] (prioritize : α → m Bool) (act : α → m β) out := out.qsort (fun (i, _) (j, _) => i < j) return out.map (·.2) -def isLeanBlock : TSyntax `block → CoreM Bool - | `(block|```$nameStx:ident $_args*|$_contents:str```) => do - let name ← realizeGlobalConstNoOverload nameStx - return name == ``Verso.Genre.Manual.InlineLean.lean - | _ => pure false +def isLeanBlock (blk : TSyntax ``Lean.Doc.Parser.block) : CoreM Bool := do + let some { name? := some nameStx, .. } := CodeBlockView.of blk + | return false + let name ← realizeGlobalConstNoOverload nameStx + return name == ``Verso.Genre.Manual.InlineLean.lean structure LeanBlockContent where content : Option String shouldElab : Bool -def getLeanBlockContents? : TSyntax `block → DocElabM (LeanBlockContent) - | `(block|```$nameStx:ident $args*|$contents:str```) => do - let name ← realizeGlobalConstNoOverload nameStx - if name == ``Verso.Genre.Manual.imports then - return { content := some contents.getString, shouldElab := false } - if name != ``Verso.Genre.Manual.InlineLean.lean then - return { content := none, shouldElab := false } - let args ← Verso.Doc.Elab.parseArgs args - let args ← parseThe InlineLean.LeanBlockConfig args - if !args.keep || args.error then - return { content := none, shouldElab := true } - pure <| { content := some contents.getString, shouldElab := true } - | _ => pure { content := none, shouldElab := false } +def getLeanBlockContents? (blk : TSyntax ``Lean.Doc.Parser.block) : + DocElabM LeanBlockContent := do + let some { name? := some nameStx, args, content := contents, .. } := CodeBlockView.of blk + | return { content := none, shouldElab := false } + let name ← realizeGlobalConstNoOverload nameStx + if name == ``Verso.Genre.Manual.imports then + return { content := some contents.getVersoCodeBlock, shouldElab := false } + if name != ``Verso.Genre.Manual.InlineLean.lean then + return { content := none, shouldElab := false } + let args ← Verso.Doc.Elab.parseArgs args + let args ← parseThe InlineLean.LeanBlockConfig args + if !args.keep || args.error then + return { content := none, shouldElab := true } + pure <| { content := some contents.getVersoCodeBlock, shouldElab := true } /-- Elaborates all Lean blocks first, enabling local forward references @@ -151,7 +152,7 @@ def «example» : DirectiveExpanderOf ExampleConfig (kind := Lsp.SymbolKind.interface) (detail? := some "Example") - let accumulate (b : TSyntax `block) : StateT (List String) DocElabM Bool := do + let accumulate (b : TSyntax ``Lean.Doc.Parser.block) : StateT (List String) DocElabM Bool := do let {content, shouldElab} ← getLeanBlockContents? b if let some x := content then modify (· ++ [x]) diff --git a/reference-manual/Manual/Meta/ExpectString.lean b/reference-manual/Manual/Meta/ExpectString.lean index 48a352a21..fc2a39615 100644 --- a/reference-manual/Manual/Meta/ExpectString.lean +++ b/reference-manual/Manual/Meta/ExpectString.lean @@ -11,6 +11,7 @@ import Verso open Lean Elab open Verso Doc +open Verso (Literal) namespace Manual @@ -34,10 +35,10 @@ modulo `preEq`. The parameter `what` is used in the error message header, in a c Errors are logged, not thrown; the returned `Bool` indicates whether an error was logged. -/ -def expectString (what : String) (expected : StrLit) (actual : String) +def expectString [Literal k] (what : String) (expected : TSyntax k) (actual : String) (preEq : String → String := id) (useLine : String → Bool := fun _ => true) : m Bool := do - let expectedLines := expected.getString.splitOn "\n" |>.filter useLine |>.toArray + let expectedLines := (Literal.decode expected).splitOn "\n" |>.filter useLine |>.toArray let actualLines := actual.splitOn "\n" |>.filter useLine |>.toArray unless expectedLines.map preEq == actualLines.map preEq do diff --git a/reference-manual/Manual/Meta/Figure.lean b/reference-manual/Manual/Meta/Figure.lean index 9a360e76e..89f055e5d 100644 --- a/reference-manual/Manual/Meta/Figure.lean +++ b/reference-manual/Manual/Meta/Figure.lean @@ -24,7 +24,7 @@ def Block.figure (captionString : String) (name : Option String) : Block where data := ToJson.toJson (captionString, name, (none : Option Tag)) structure FigureConfig where - caption : TSyntaxArray `inline + caption : TSyntaxArray ``Lean.Doc.Parser.inline /-- Name for refs -/ tag : Option String := none diff --git a/reference-manual/Manual/Meta/LakeCmd.lean b/reference-manual/Manual/Meta/LakeCmd.lean index c2e510bf7..aaf2f1557 100644 --- a/reference-manual/Manual/Meta/LakeCmd.lean +++ b/reference-manual/Manual/Meta/LakeCmd.lean @@ -22,9 +22,9 @@ import Manual.Meta.Basic open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open Lean Elab open SubVerso.Highlighting Highlighted -open Lean.Doc.Syntax open Lean.Elab.Tactic.GuardMsgs +open Lean.Doc (CodeView) namespace Manual @@ -316,9 +316,9 @@ def lakeMeta : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $mName:str )) := arg + let some { content := mName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the metavariable" - let mName := mName.getString + let mName := mName.getVersoCode pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other {Manual.Inline.lakeMeta with data := Json.arr #[$(quote mName), .null]} #[Inline.code $(quote mName)])] @@ -350,9 +350,9 @@ def lakeInline : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $cmdName:str )) := arg + let some { content := cmdName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the Lake command name" - let name := cmdName.getString + let name := cmdName.getVersoCode pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other {Manual.Inline.lake with data := $(quote name)} #[Inline.code $(quote name)])] @@ -399,10 +399,10 @@ def lakeArgs : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $spec:str )) := arg + let some { content := spec, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the Lake command name" - match Parser.runParserCategory (← getEnv) `lake_cmd_spec spec.getString (← getFileName) with + match Parser.runParserCategory (← getEnv) `lake_cmd_spec spec.getVersoCode (← getFileName) with | .error e => throwErrorAt spec e | .ok stx => match CommandSpec.ofSyntax stx with diff --git a/reference-manual/Manual/Meta/LakeLean.lean b/reference-manual/Manual/Meta/LakeLean.lean index a1f417879..e8446ebea 100644 --- a/reference-manual/Manual/Meta/LakeLean.lean +++ b/reference-manual/Manual/Meta/LakeLean.lean @@ -31,7 +31,6 @@ SubVerso highlighting of the source (for display). open Verso ArgParse Doc Elab Genre.Manual open Verso.Log open Lean Elab -open scoped Lean.Doc.Syntax namespace Manual @@ -55,19 +54,15 @@ executable. The configuration is elaborated, the result is checked against the ` def lakeLean : DirectiveExpander | args, contents => do let opts ← LakeLeanOpts.parse.run args - let (expected, contents) := contents.partition fun - | `(block| ``` expected | $_ ```) => true - | _ => false - let leanBlocks := contents.filterMap fun - | `(block| ``` lean $_* | $leanStr ```) => some leanStr - | _ => none + let (expected, contents) := contents.partition (namedCodeBlock `expected · |>.isSome) + let leanBlocks := contents.filterMap (namedCodeBlock `lean ·) if h : expected.size ≠ 1 then throwError "Expected exactly 1 'expected' code block, got {expected.size}" else if h : leanBlocks.size ≠ 1 then throwError "Expected exactly 1 'lean' code block, got {leanBlocks.size}" else - let `(block| ```expected | $expectedStr ```) := expected[0] + let some expectedStr := namedCodeBlock `expected expected[0] | throwErrorAt expected[0] "Expected an 'expected' code block with no arguments" let leanStr := leanBlocks[0] diff --git a/reference-manual/Manual/Meta/LakeManifest.lean b/reference-manual/Manual/Meta/LakeManifest.lean index 43c861cce..dfd90a9a3 100644 --- a/reference-manual/Manual/Meta/LakeManifest.lean +++ b/reference-manual/Manual/Meta/LakeManifest.lean @@ -47,8 +47,8 @@ def jsonHl : Highlighter where tokenClass := fun s => some (toString s.getKind) -private def parseJson (str : StrLit) : DocElabM (Json × LexedText) := do - let s := str.getString +private def parseJson [Verso.Literal k] (str : TSyntax k) : DocElabM (Json × LexedText) := do + let s := Verso.Literal.decode str let json ← match Json.parse s with | .ok json => pure json @@ -109,7 +109,7 @@ def lakeManifest : CodeBlockExpanderOf Unit match Lake.Manifest.decodeEntries json with | .ok _ => pure () | .error e => throwError m!"Block is not a valid manifest: {e}" - ``(Verso.Doc.Block.other (Block.json $(quote toks)) #[Verso.Doc.Block.code $str]) + ``(Verso.Doc.Block.other (Block.json $(quote toks)) #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) /-- Check that contents of the block is a valid package overrides file. @@ -121,4 +121,4 @@ def lakePackageOverrides : CodeBlockExpanderOf Unit match fromJson? json with | .ok (_ : Lake.Manifest) => pure () | .error e => throwError m!"Block is not a valid package overrides file: {e}" - ``(Verso.Doc.Block.other (Block.json $(quote toks)) #[Verso.Doc.Block.code $str]) + ``(Verso.Doc.Block.other (Block.json $(quote toks)) #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) diff --git a/reference-manual/Manual/Meta/LakeOpt.lean b/reference-manual/Manual/Meta/LakeOpt.lean index 0f46458c9..244aa125d 100644 --- a/reference-manual/Manual/Meta/LakeOpt.lean +++ b/reference-manual/Manual/Meta/LakeOpt.lean @@ -16,8 +16,8 @@ import Verso.Code import Manual.Meta.Basic open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets -open Lean.Doc.Syntax open Lean Elab +open Lean.Doc (CodeView) namespace Manual @@ -81,9 +81,9 @@ def lakeOptDef : RoleExpander let {kind} ← LakeOptDefOpts.parse.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $name:str )) := arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the option or flag" - let origName := name.getString + let origName := name.getVersoCode let name := origName.takeWhile fun c => c == '-' || c.isAlphanum let name := name.copy let valMeta := origName.drop name.length |>.dropWhile fun (c : Char) => !c.isAlphanum @@ -140,12 +140,12 @@ def lakeOpt : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $name:str )) := arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the option or flag" - let optName := name.getString.takeWhile fun c => c == '-' || c.isAlphanum + let optName := name.getVersoCode.takeWhile fun c => c == '-' || c.isAlphanum let optName := optName.copy - pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.lakeOpt $(quote optName) $(quote name.getString)) #[Inline.code $(quote name.getString)])] + pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.lakeOpt $(quote optName) $(quote name.getVersoCode)) #[Inline.code $(quote name.getVersoCode)])] @[inline_extension lakeOpt] def lakeOpt.descr : InlineDescr where diff --git a/reference-manual/Manual/Meta/LakeSession.lean b/reference-manual/Manual/Meta/LakeSession.lean index 09507fe03..611dddf0b 100644 --- a/reference-manual/Manual/Meta/LakeSession.lean +++ b/reference-manual/Manual/Meta/LakeSession.lean @@ -43,8 +43,8 @@ open Verso ArgParse Doc Elab Genre.Manual open Verso.Doc.Elab open Verso.Log open Lean Elab -open scoped Lean.Doc.Syntax open SubVerso.Highlighting (Highlighted) +open Lean.Doc (CodeBlockView VersoCodeBlock) namespace Manual @@ -91,29 +91,31 @@ private def isBlank (s : String) : Bool := s.all Char.isWhitespace /-- The classification of a block inside a `lakeSession` directive. -/ inductive SessionItem where /-- A `toml` block, becoming `lakefile.toml`. The syntax is kept for rendering. -/ - | tomlConfig (contents : StrLit) (block : Syntax) + | tomlConfig (contents : VersoCodeBlock) (block : Syntax) /-- A `lean +lakefile` block, becoming `lakefile.lean`. The syntax is kept for rendering. -/ - | leanConfig (contents : StrLit) (block : Syntax) + | leanConfig (contents : VersoCodeBlock) (block : Syntax) /-- A `lean (file := …)` source-file block. -/ - | source (cfg : SourceFileConfig) (contents : StrLit) + | source (cfg : SourceFileConfig) (contents : VersoCodeBlock) /-- A `lakeCmd "…"` block, with its expected output and the block syntax (for error reporting). -/ - | command (cfg : LakeCmdConfig) (output : StrLit) (blame : Syntax) + | command (cfg : LakeCmdConfig) (output : VersoCodeBlock) (blame : Syntax) /-- Any other block, rendered unchanged. -/ | passthrough (block : Syntax) /-- Classify a block within a `lakeSession`. -/ -def classifySessionBlock (block : Syntax) : DocElabM SessionItem := do - match block with - | `(block| ``` toml $_* | $contents ```) => return .tomlConfig contents block - | `(block| ``` lakeCmd $args* | $output ```) => +def classifySessionBlock (block : TSyntax ``Lean.Doc.Parser.block) : DocElabM SessionItem := do + let some { name? := some name, args, content, .. } := CodeBlockView.of block + | return .passthrough block + match name.getId with + | `toml => return .tomlConfig content block + | `lakeCmd => let cfg ← LakeCmdConfig.parse.run (← parseArgs args) - return .command cfg output block - | `(block| ``` lean $args* | $contents ```) => + return .command cfg content block + | `lean => -- A `lean` block is the Lean-format configuration when marked `+lakefile`, a project source -- file when it carries a `file` argument, and otherwise an ordinary rendered example. match ← (try some <$> leanBlockArgs.run (← parseArgs args) catch _ => pure none) with - | some (_, true) => return .leanConfig contents block - | some (some file, false) => return .source ⟨file⟩ contents + | some (_, true) => return .leanConfig content block + | some (some file, false) => return .source ⟨file⟩ content | _ => return .passthrough block | _ => return .passthrough block @@ -280,7 +282,7 @@ def lakeSession : DirectiveExpander return #[← ``(Verso.Doc.Block.empty)] where /-- Run a single command in `dir` and check its exit code and output. -/ - runCommand (dir : System.FilePath) (cfg : LakeCmdConfig) (output : StrLit) (blame : Syntax) : + runCommand (dir : System.FilePath) (cfg : LakeCmdConfig) (output : VersoCodeBlock) (blame : Syntax) : DocElabM Unit := do let parts := cfg.command.splitOn " " |>.filter (!·.isEmpty) let some cmd := parts.head? diff --git a/reference-manual/Manual/Meta/LakeToml.lean b/reference-manual/Manual/Meta/LakeToml.lean index de5b188dc..5fe075b51 100644 --- a/reference-manual/Manual/Meta/LakeToml.lean +++ b/reference-manual/Manual/Meta/LakeToml.lean @@ -29,8 +29,8 @@ import Lake.Load.Toml open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open Lean Elab open SubVerso.Highlighting Highlighted -open scoped Lean.Doc.Syntax open Lean.Elab.Tactic.GuardMsgs +open Lean.Doc (CodeView) set_option guard_msgs.diff true @@ -791,22 +791,18 @@ def LakeTomlOpts.parse [Monad m] [MonadInfoTree m] [MonadLiftT CoreM m] [MonadEn def lakeToml : DirectiveExpander | args, contents => do let opts ← LakeTomlOpts.parse.run args - let (expected, contents) := contents.partition fun - | `(block| ``` expected | $_ ```) => true - | _ => false - let toml := contents.filterMap fun - | `(block| ``` toml $_* | $tomlStr ```) => some tomlStr - | _ => none + let (expected, contents) := contents.partition (namedCodeBlock `expected · |>.isSome) + let toml := contents.filterMap (namedCodeBlock `toml ·) if h : expected.size ≠ 1 then throwError "Expected exactly 1 'expected' code block, got {expected.size}" else - let `(block| ```expected | $expectedStr ```) := expected[0] + let some expectedStr := namedCodeBlock `expected expected[0] | throwErrorAt expected[0] "Expected an 'expected' code block with no arguments" if h : toml.size ≠ 1 then throwError "Expected exactly 1 toml code block, got {toml.size}" else let tomlStr := toml[0] - let tomlInput := tomlStr.getString ++ "\n" + let tomlInput := tomlStr.getVersoCodeBlock ++ "\n" let v ← match opts.field, opts.type with | `_root_, ``Lake.PackageConfig => match (← checkTomlPackage ((← parserInputString tomlStr) ++ "\n")) with @@ -839,9 +835,9 @@ def tomlFieldInline : RoleExpander let table ← (ArgParse.positional `table .resolvedName).run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $name:str )) := arg + let some { content := name, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the field name" - let name := name.getString + let name := name.getVersoCode pure #[← `(show Verso.Doc.Inline Verso.Genre.Manual from .other (Manual.Inline.tomlField $(quote table) $(quote name.toName)) #[Inline.code $(quote name)])] diff --git a/reference-manual/Manual/Meta/LakeToml/Toml.lean b/reference-manual/Manual/Meta/LakeToml/Toml.lean index 1e192025f..11395b6ab 100644 --- a/reference-manual/Manual/Meta/LakeToml/Toml.lean +++ b/reference-manual/Manual/Meta/LakeToml/Toml.lean @@ -16,9 +16,9 @@ open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets Multi open SubVerso.Highlighting Highlighted open Lean Elab -open scoped Lean.Doc.Syntax open Lean.Elab.Tactic.GuardMsgs +open Lean.Doc (CodeView) namespace Manual @@ -388,7 +388,7 @@ def Toml.tableLink (xref : Genre.Manual.TraverseState) (table : Name) : Option S return dest.link open Lean.Parser in -def tomlContent (str : StrLit) : DocElabM Toml.Highlighted := do +def tomlContent [Verso.Literal k] (str : TSyntax k) : DocElabM Toml.Highlighted := do let scope : Command.Scope := {header := ""} let inputCtx := Parser.mkInputContext (← parserInputString str) (← getFileName) let pmctx : Parser.ParserModuleContext := @@ -453,7 +453,7 @@ open Lean.Parser in def toml : CodeBlockExpanderOf TomlParams | { link }, str => do let hl ← tomlContent str - ``(Block.other (Block.toml $(quote hl) $(quote link)) #[Block.code $(quote str.getString)]) + ``(Block.other (Block.toml $(quote hl) $(quote link)) #[Block.code $(quote str.getVersoCodeBlock)]) open Lean.Parser in @[role_expander toml] @@ -463,12 +463,12 @@ def tomlInline : RoleExpander let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $str:str )) := arg + let some { content := str, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with TOML code" let hl ← tomlContent str - pure #[← ``(Inline.other (Inline.toml $(quote hl)) #[Inline.code $(quote str.getString)])] + pure #[← ``(Inline.other (Inline.toml $(quote hl)) #[Inline.code $(quote str.getVersoCode)])] @[block_extension Block.toml] diff --git a/reference-manual/Manual/Meta/Lean.lean b/reference-manual/Manual/Meta/Lean.lean index 644eacb70..704eccf9f 100644 --- a/reference-manual/Manual/Meta/Lean.lean +++ b/reference-manual/Manual/Meta/Lean.lean @@ -8,7 +8,7 @@ import VersoManual import Lean.Elab.InfoTree.Types import SubVerso.Highlighting.Code -open scoped Lean.Doc.Syntax +open Lean.Doc (CodeView RoleView) open Verso Doc Elab open Lean Elab @@ -26,7 +26,7 @@ def typed : RoleExpander let config ← LeanInlineConfig.parse.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $term:str )) := arg + let some { content := term, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let altStr ← parserInputString term @@ -85,9 +85,9 @@ def typed : RoleExpander pushInfoTree tree - if let `(inline|role{%$s $f $_*}%$e[$_*]) ← getRef then - Hover.addCustomHover (mkNullNode #[s, e]) type - Hover.addCustomHover f type + if let some { braceOpen, name, braceClose, .. } := RoleView.of ⟨← getRef⟩ then + Hover.addCustomHover (mkNullNode #[braceOpen, braceClose]) type + Hover.addCustomHover name type if config.error then if newMsgs.hasErrors then @@ -107,7 +107,7 @@ def typed : RoleExpander if config.show then - pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(quote hls)) #[Inline.code $(quote term.getString)])] + pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(quote hls)) #[Inline.code $(quote term.getVersoCode)])] else pure #[] where diff --git a/reference-manual/Manual/Meta/LexedText.lean b/reference-manual/Manual/Meta/LexedText.lean index cc5065461..0945abf36 100644 --- a/reference-manual/Manual/Meta/LexedText.lean +++ b/reference-manual/Manual/Meta/LexedText.lean @@ -10,7 +10,7 @@ import VersoManual -- TODO generalize upstream - this is based on the one in the blog genre. namespace Manual open Verso -open Lean.Doc.Syntax +open Lean.Doc (CodeView) abbrev LexedText.Highlighted := Array (Option String × String) @@ -137,7 +137,7 @@ def lexedText := () @[code_block] def C : CodeBlockExpanderOf Unit | (), str => do - let codeStr := str.getString + let codeStr := str.getVersoCodeBlock let toks ← LexedText.highlight hlC codeStr ``(Block.other (Block.c $(quote toks)) #[Block.code $(quote codeStr)]) @@ -170,8 +170,8 @@ def cInline : RoleExpanderOf Unit | (), contents => do let #[x] := contents | throwError "Expected exactly one parameter" - let `(inline|code($str)) := x + let some { content := str, .. } := CodeView.of x | throwError "Expected exactly one code item" - let codeStr := str.getString + let codeStr := str.getVersoCode let toks ← LexedText.highlight hlC codeStr ``(Inline.other (Inline.c $(quote toks)) #[Inline.code $(quote codeStr)]) diff --git a/reference-manual/Manual/Meta/Markdown.lean b/reference-manual/Manual/Meta/Markdown.lean index 723c23957..baa9ff6e4 100644 --- a/reference-manual/Manual/Meta/Markdown.lean +++ b/reference-manual/Manual/Meta/Markdown.lean @@ -44,15 +44,15 @@ def closeEnclosingSections (headerMapping : Markdown.HeaderMapping) : PartElabM for _ in headerMapping do closeEnclosingSection -@[part_command Lean.Doc.Syntax.codeblock] +@[part_command Lean.Doc.Parser.Block.codeblock] def markdown : PartCommand - | `(Lean.Doc.Syntax.codeblock| ``` $markdown:ident $args*| $txt ``` ) => do + | .codeblock { name? := some markdown, args, content := txt, .. } => do let x ← Lean.Elab.realizeGlobalConstNoOverloadWithInfo markdown if x != by exact decl_name% then Elab.throwUnsupportedSyntax for arg in args do let h ← MessageData.hint m!"Remove it" #[""] (ref? := arg) logErrorAt arg m!"No arguments expected{h}" - let some ast := MD4Lean.parse txt.getString + let some ast := MD4Lean.parse txt.getVersoCodeBlock | throwError "Failed to parse body of markdown code block" let mut currentHeaderLevels : Markdown.HeaderMapping := {} for block in ast.blocks do diff --git a/reference-manual/Manual/Meta/ModuleExample.lean b/reference-manual/Manual/Meta/ModuleExample.lean index f1302fb85..3a8296d03 100644 --- a/reference-manual/Manual/Meta/ModuleExample.lean +++ b/reference-manual/Manual/Meta/ModuleExample.lean @@ -22,6 +22,7 @@ open Verso.Doc.Elab open Verso.ArgParse open Verso.Log open Lean +open Lean.Doc (CodeBlockView RoleView VersoCode VersoCodeBlock mkVersoCodeBlockFrom) namespace Manual @@ -93,7 +94,7 @@ def lineStx [Monad m] [MonadFileMap m] (l : Nat) : m Syntax := do def leanModule : CodeBlockExpanderOf ModuleConfig | { name, moduleName, error, «show» }, str => do let line := (← getFileMap).utf8PosToLspPos str.raw.getPos! |>.line - let leanCode := line.fold (fun _ _ s => s.push '\n') "" ++ str.getString ++ "\n" + let leanCode := line.fold (fun _ _ s => s.push '\n') "" ++ str.getVersoCodeBlock ++ "\n" let hl ← IO.FS.withTempDir fun dirname => do let u := toString (← IO.monoMsNow) let dirname := dirname / u @@ -187,22 +188,24 @@ instance : FromArgs ModulesConfig m where fromArgs := ModulesConfig.mk <$> .flag `server true <*> .many (.named' `moduleRoot false) <*> .flag `error false end -open Lean.Doc.Syntax in -partial def getBlocks (block : Syntax) : StateT (NameMap (ModuleConfig × StrLit × Syntax)) DocElabM Syntax := do - if block.getKind == ``Lean.Doc.Syntax.codeblock then - if let `(Lean.Doc.Syntax.codeblock|```$x:ident $args* | $s:str ```) := block then - try - let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x - if x' == ``leanModule then - let n ← mkFreshUserName `code - let blame := mkNullNode <| #[x] ++ args - let argVals ← parseArgs args - let cfg ← fromArgs.run argVals - modify (·.insert n (cfg, s, blame)) - let x := mkIdentFrom block n - return ← `(Lean.Doc.Syntax.codeblock|```identRef $x:ident | $(quote "") ```) - catch - | _ => pure () +partial def getBlocks (block : Syntax) : + StateT (NameMap (ModuleConfig × VersoCodeBlock × Syntax)) DocElabM Syntax := do + if let some { openFence, name? := some x, args, content := s, closeFence, .. } := + CodeBlockView.of ⟨block⟩ then + try + let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x + if x' == ``leanModule then + let n ← mkFreshUserName `code + let blame := mkNullNode <| #[x.raw] ++ args.map (·.raw) + let argVals ← parseArgs args + let cfg ← fromArgs.run argVals + modify (·.insert n (cfg, s, blame)) + let x := mkIdentFrom block n + return ← `(Lean.Doc.Parser.Block.codeblock| + $openFence:codeBlockFence identRef $x:ident + $(mkVersoCodeBlockFrom s ""):versoCodeBlock $closeFence:codeBlockFence) + catch + | _ => pure () match block with | .node i k xs => do @@ -210,23 +213,22 @@ partial def getBlocks (block : Syntax) : StateT (NameMap (ModuleConfig × StrLit return Syntax.node i k args | _ => return block -open Lean.Doc.Syntax in -partial def getQuotes (stx : Syntax) : StateT (NameMap StrLit) DocElabM Syntax := do - if stx.getKind == ``Lean.Doc.Syntax.role then - if let `(Lean.Doc.Syntax.role|role{$x:ident $args*}[$inls*]) := stx then - try - let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x - if x' == ``Verso.Genre.Manual.InlineLean.name then - unless args.isEmpty do logErrorAt (mkNullNode args) m!"No arguments expected here" - let some code ← oneCodeStr? inls - | return ((← `(.empty)) : Syntax) - - let n ← mkFreshUserName `name - modify (·.insert n code) - let x := mkIdentFrom stx n - return ((← `(Lean.Doc.Syntax.role|role{identRef $x:ident}[])) : Syntax) - catch - | _ => pure () +partial def getQuotes (stx : Syntax) : StateT (NameMap VersoCode) DocElabM Syntax := do + if let some { name := x, args, content := inls, .. } := RoleView.of ⟨stx⟩ then + try + let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x + if x' == ``Verso.Genre.Manual.InlineLean.name then + unless args.isEmpty do + logErrorAt (mkNullNode (args.map (·.raw))) m!"No arguments expected here" + let some code ← oneCodeStr? inls + | return ((← `(.empty)) : Syntax) + + let n ← mkFreshUserName `name + modify (·.insert n code) + let x := mkIdentFrom stx n + return ((← `(Lean.Doc.Parser.Inline.role|{identRef $x:ident}[])) : Syntax) + catch + | _ => pure () match stx with | .node i k xs => do @@ -367,13 +369,13 @@ def leanModules : DirectiveExpanderOf ModulesConfig let (blocks, quotes) ← blocks.mapM getQuotes |>.run {} for (x, q) in quotes do - if let some tok := allHl.matchingName? q.getString then + if let some tok := allHl.matchingName? q.getVersoCode then addLets := addLets >=> fun stx => do let hl : SubVerso.Highlighting.Highlighted := .token tok let hl : Term := quote hl - let name ← `(Verso.Doc.Inline.other {Verso.Genre.Manual.InlineLean.Inline.name with data := ToJson.toJson $hl} #[Verso.Doc.Inline.code $(quote q.getString)]) + let name ← `(Verso.Doc.Inline.other {Verso.Genre.Manual.InlineLean.Inline.name with data := ToJson.toJson $hl} #[Verso.Doc.Inline.code $(quote q.getVersoCode)]) `(let $(mkIdent x) := $name; $stx) - else logErrorAt q m!"Not found: {q.getString.quote}" + else logErrorAt q m!"Not found: {q.getVersoCode.quote}" let body ← blocks.mapM (elabBlock <| ⟨·⟩) let body ← `(Verso.Doc.Block.concat #[$body,*]) addLets body diff --git a/reference-manual/Manual/Meta/Monotonicity.lean b/reference-manual/Manual/Meta/Monotonicity.lean index 06fafdb81..5f284f9a1 100644 --- a/reference-manual/Manual/Meta/Monotonicity.lean +++ b/reference-manual/Manual/Meta/Monotonicity.lean @@ -11,7 +11,6 @@ import Manual.Meta.Basic import Manual.Meta.CustomStyle import Manual.Meta.Instances -open scoped Lean.Doc.Syntax open Verso Doc Elab Manual open Verso.Genre.Manual diff --git a/reference-manual/Manual/Meta/Namespace.lean b/reference-manual/Manual/Meta/Namespace.lean index b32f2c63e..ad0e6754a 100644 --- a/reference-manual/Manual/Meta/Namespace.lean +++ b/reference-manual/Manual/Meta/Namespace.lean @@ -8,7 +8,7 @@ import VersoManual import Lean.Elab.InfoTree.Types import SubVerso.Highlighting.Code -open scoped Lean.Doc.Syntax +open Lean.Doc (CodeView) open Verso Doc Elab open Lean Elab @@ -19,11 +19,11 @@ open SubVerso.Highlighting @[role] def «namespace» : RoleExpanderOf Unit | (), #[arg] => do - let `(inline|code($s)) := arg + let some { content := s, .. } := CodeView.of arg | throwErrorAt arg "Expected code" -- TODO validate that namespace exists? Or is that too strict? -- TODO namespace domain for documentation - ``(Inline.code $(quote s.getString)) + ``(Inline.code $(quote s.getVersoCode)) | _, more => if h : more.size > 0 then throwErrorAt more[0] "Expected code literal with the namespace" diff --git a/reference-manual/Manual/Meta/SpliceContents.lean b/reference-manual/Manual/Meta/SpliceContents.lean index f5a6d9a2d..a8fc4b1a2 100644 --- a/reference-manual/Manual/Meta/SpliceContents.lean +++ b/reference-manual/Manual/Meta/SpliceContents.lean @@ -14,7 +14,6 @@ namespace Manual open Verso ArgParse Doc Elab Genre.Manual Html Code Highlighted.WebAssets open Lean Elab -open Lean.Doc.Syntax structure SpliceContentsConfig where moduleName : Ident @@ -22,10 +21,11 @@ structure SpliceContentsConfig where def SpliceContentsConfig.parse [Monad m] [MonadInfoTree m] [MonadLiftT CoreM m] [MonadEnv m] [MonadError m] : ArgParse m SpliceContentsConfig := SpliceContentsConfig.mk <$> .positional `moduleName .ident -@[part_command Lean.Doc.Syntax.command] +@[part_command Lean.Doc.Parser.Block.command] def spliceContents : PartCommand - | `(block|command{spliceContents $args*}) => do - let {moduleName} ← SpliceContentsConfig.parse.run (← parseArgs args) + | .command v => do + unless v.name.getId == `spliceContents do Lean.Elab.throwUnsupportedSyntax + let {moduleName} ← SpliceContentsConfig.parse.run (← parseArgs v.args) let moduleIdent ← mkIdentFrom moduleName <$> realizeGlobalConstNoOverloadWithInfo (mkIdentFrom moduleName (docName moduleName.getId)) diff --git a/reference-manual/Manual/Meta/Syntax.lean b/reference-manual/Manual/Meta/Syntax.lean index f3d8c0d22..53744fb7c 100644 --- a/reference-manual/Manual/Meta/Syntax.lean +++ b/reference-manual/Manual/Meta/Syntax.lean @@ -17,11 +17,11 @@ open Verso.Genre Manual open Verso.ArgParse open Verso.Code (highlightingJs) open Verso.Code.Highlighted.WebAssets -open Lean.Doc.Syntax open Verso.Genre.Manual.InlineLean.Scopes (getScopes) open Lean Elab Parser +open Lean.Doc (CodeView CodeBlockView) open Lean.Widget (TaggedText) namespace Manual @@ -35,7 +35,7 @@ def evalPrio : RoleExpander ArgParse.done.run args let #[inl] := inlines | throwError "Expected a single code argument" - let `(inline|code( $s:str )) := inl + let some { content := s, .. } := CodeView.of inl | throwErrorAt inl "Expected code literal with the priority" let altStr ← parserInputString s match runParser (← getEnv) (← getOptions) (andthen ⟨{}, whitespace⟩ priorityParser) altStr (← getFileName) with @@ -45,7 +45,7 @@ def evalPrio : RoleExpander | .error es => for (pos, msg) in es do log (severity := .error) (mkErrorStringWithPos "" pos msg) - throwError s!"Failed to parse priority from '{s.getString}'" + throwError s!"Failed to parse priority from '{s.getVersoCode}'" @[role_expander evalPrec] def evalPrec : RoleExpander @@ -53,7 +53,7 @@ def evalPrec : RoleExpander ArgParse.done.run args let #[inl] := inlines | throwError "Expected a single code argument" - let `(inline|code( $s:str )) := inl + let some { content := s, .. } := CodeView.of inl | throwErrorAt inl "Expected code literal with the precedence" let altStr ← parserInputString s match runParser (← getEnv) (← getOptions) (andthen ⟨{}, whitespace⟩ (categoryParser `prec 1024)) altStr (← getFileName) with @@ -63,7 +63,7 @@ def evalPrec : RoleExpander | .error es => for (pos, msg) in es do log (severity := .error) (mkErrorStringWithPos "" pos msg) - throwError s!"Failed to parse precedence from '{s.getString}'" + throwError s!"Failed to parse precedence from '{s.getVersoCode}'" def Block.syntax : Block where name := `Manual.syntax @@ -81,7 +81,7 @@ structure FreeSyntaxConfig where name : Name «open» : Bool := true label : Option String := none - title : TSyntaxArray `inline + title : TSyntaxArray ``Lean.Doc.Parser.inline def FreeSyntaxConfig.getLabel (config : FreeSyntaxConfig) : String := config.label.getD <| @@ -109,7 +109,7 @@ def keywordOf : RoleExpander let ⟨kind, parser⟩ ← KeywordOfConfig.parse.run args let #[inl] := inlines | throwError "Expected a single code argument" - let `(inline|code( $kw:str )) := inl + let some { content := kw, .. } := CodeView.of inl | throwErrorAt inl "Expected code literal with the keyword" let kindName := kind.getId let parserName ← parser.mapM (realizeGlobalConstNoOverloadWithInfo ·) @@ -120,7 +120,7 @@ def keywordOf : RoleExpander if kindName == k then catName := some cat; break if let some _ := catName then break let kindDoc ← findDocString? (← getEnv) kindName - return #[← `(Inline.other {Inline.keywordOf with data := ToJson.toJson (α := (String × Option Name × Name × Option String)) $(quote (kw.getString, catName, parserName.getD kindName, kindDoc))} #[Inline.code $kw])] + return #[← `(Inline.other {Inline.keywordOf with data := ToJson.toJson (α := (String × Option Name × Name × Option String)) $(quote (kw.getVersoCode, catName, parserName.getD kindName, kindDoc))} #[Inline.code $(quote kw.getVersoCode)])] @[inline_extension keywordOf] def keywordOf.descr : InlineDescr := withHighlighting { @@ -217,10 +217,10 @@ def keyword : RoleExpander let () ← ArgParse.done.run args let #[inl] := inlines | throwError "Expected a single code argument" - let `(inline|code( $kw:str )) := inl + let some { content := kw, .. } := CodeView.of inl | throwErrorAt inl "Expected code literal with the keyword" - return #[← `(Inline.other {Inline.keyword with data := Lean.Json.str $(quote kw.getString)} #[Inline.code $kw])] + return #[← `(Inline.other {Inline.keyword with data := Lean.Json.str $(quote kw.getVersoCode)} #[Inline.code $(quote kw.getVersoCode)])] @[inline_extension keyword] def keyword.descr : InlineDescr where @@ -1104,12 +1104,14 @@ def «syntax» : DirectiveExpander pure #[← `(Block.other {Block.syntax with data := ToJson.toJson (α := Option String × Name × String × Option Tag × Array Name) ($(quote titleString), $(quote config.name), $(quote config.getLabel), none, $(quote config.aliases.toArray))} #[Block.para #[$(title),*], $content,*])] where - isGrammar? : Syntax → Option (Syntax × Array Syntax × StrLit) - | `(block|``` $nameStx:ident $argsStx* | $contents ```) => - if nameStx.getId == `grammar then some (nameStx, argsStx, contents) else none - | _ => none - - elabGrammar nameStx config isFirst (argsStx : Array Syntax) (str : TSyntax `str) := do + isGrammar? (blk : TSyntax ``Lean.Doc.Parser.block) : + Option (Syntax × Array Syntax × Lean.Doc.VersoCodeBlock) := + match CodeBlockView.of blk with + | some { name? := some nameStx, args, content, .. } => + if nameStx.getId == `grammar then some (nameStx, args.map (·.raw), content) else none + | _ => none + + elabGrammar nameStx config isFirst (argsStx : Array Syntax) (str : Lean.Doc.VersoCodeBlock) := do let args ← parseArgs <| argsStx.map (⟨·⟩) let {of, prec} ← GrammarConfig.parse.run args let config : SyntaxConfig := @@ -1174,12 +1176,14 @@ def freeSyntax : DirectiveExpander content := content.push <| ← elabBlock b pure #[← `(Block.other {Block.syntax with data := ToJson.toJson (α := Option String × Name × String × Option Tag × Array Name) ($(quote titleString), $(quote config.name), $(quote config.getLabel), none, #[])} #[Block.para #[$(title),*], $content,*])] where - isGrammar? : Syntax → Option (Syntax × Array Syntax × StrLit) - | `(block|```$nameStx:ident $argsStx* | $contents:str ```) => - if nameStx.getId == `grammar then some (nameStx, argsStx, contents) else none - | _ => none - - elabGrammar nameStx config isFirst (argsStx : Array Syntax) (str : TSyntax `str) := do + isGrammar? (blk : TSyntax ``Lean.Doc.Parser.block) : + Option (Syntax × Array Syntax × Lean.Doc.VersoCodeBlock) := + match CodeBlockView.of blk with + | some { name? := some nameStx, args, content, .. } => + if nameStx.getId == `grammar then some (nameStx, args.map (·.raw), content) else none + | _ => none + + elabGrammar nameStx config isFirst (argsStx : Array Syntax) (str : Lean.Doc.VersoCodeBlock) := do let args ← parseArgs <| argsStx.map (⟨·⟩) let () ← ArgParse.done.run args let altStr ← parserInputString str @@ -1528,13 +1532,13 @@ def syntaxKind : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $syntaxKindName:str )) := arg + let some { content := syntaxKindName, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the syntax kind name" - let kName := syntaxKindName.getString.toName + let kName := syntaxKindName.getVersoCode.toName let id : Ident := mkIdentFrom syntaxKindName kName let k ← try realizeGlobalConstNoOverloadWithInfo id catch _ => pure kName let doc? ← findDocString? (← getEnv) k - return #[← `(Inline.other {Inline.syntaxKind with data := ToJson.toJson (α := Name × String × Option String) ($(quote k), $(quote syntaxKindName.getString), $(quote doc?))} #[Inline.code $(quote k.toString)])] + return #[← `(Inline.other {Inline.syntaxKind with data := ToJson.toJson (α := Name × String × Option String) ($(quote k), $(quote syntaxKindName.getVersoCode), $(quote doc?))} #[Inline.code $(quote k.toString)])] @[inline_extension syntaxKind] diff --git a/reference-manual/Manual/Meta/Tactics.lean b/reference-manual/Manual/Meta/Tactics.lean index 06689902a..f77590f63 100644 --- a/reference-manual/Manual/Meta/Tactics.lean +++ b/reference-manual/Manual/Meta/Tactics.lean @@ -43,7 +43,7 @@ private partial def disableUnusedVarLinterInInfoTree : InfoTree → InfoTree open Verso.Genre.Manual.InlineLean.Scopes (runWithOpenDecls runWithVariables) open SubVerso.Highlighting open SubVerso.Examples.Messages -open Lean.Doc.Syntax +open Lean.Doc (CodeView VersoCode VersoCodeBlock) structure TacticOutputConfig where «show» : Bool := true @@ -64,7 +64,7 @@ def TacticOutputConfig.parser [Monad m] [MonadInfoTree m] [MonadLiftT CoreM m] [ (many (.named `expandTrace .name false)) -def checkTacticExample (goal : Term) (proofPrefix : Syntax) (tactic : Syntax) (pre : TSyntax `str) (post : TSyntax `str) : TermElabM Unit := do +def checkTacticExample (goal : Term) (proofPrefix : Syntax) (tactic : Syntax) (pre post : VersoCodeBlock) : TermElabM Unit := do let statement ← elabType goal let mv ← Meta.mkFreshExprMVar (some statement) let Expr.mvar mvarId := mv @@ -82,8 +82,8 @@ def checkTacticExample (goal : Term) (proofPrefix : Syntax) (tactic : Syntax) (p let st1 := goalsToMessageData remainingGoals --logInfoAt proofPrefix st1 let goodPre ← (← addMessageContext st1).toString - if pre.getString != goodPre then - logErrorAt pre m!"Mismatch. Expected {indentD goodPre}\n but got {indentD pre.getString}" + if pre.getVersoCodeBlock != goodPre then + logErrorAt pre m!"Mismatch. Expected {indentD goodPre}\n but got {indentD pre.getVersoCodeBlock}" -- Run the example let remainingGoals' ← Tactic.run mvarId do withoutTacticIncrementality true <| @@ -93,14 +93,14 @@ def checkTacticExample (goal : Term) (proofPrefix : Syntax) (tactic : Syntax) (p let st2 := goalsToMessageData remainingGoals' --logInfoAt tactic st2 let goodPost ← (← addMessageContext st2).toString - if post.getString != goodPost then - logErrorAt post m!"Mismatch. Expected {indentD goodPost}\n but got {indentD post.getString}" + if post.getVersoCodeBlock != goodPost then + logErrorAt post m!"Mismatch. Expected {indentD goodPost}\n but got {indentD post.getVersoCodeBlock}" open Lean.Elab.Tactic.GuardMsgs in def checkTacticExample' (goal : Expr) (proofPrefix : Syntax) (tactic : Syntax) - (pre : TSyntax `str) (post : TSyntax `str) - (output : Option (TSyntax `str × TacticOutputConfig)) : + (pre post : VersoCodeBlock) + (output : Option (VersoCodeBlock × TacticOutputConfig)) : TermElabM (Array (Highlighted.Goal Highlighted) × Array (Highlighted.Goal Highlighted) × @@ -132,9 +132,9 @@ def checkTacticExample' let st1 := goalsToMessageData remainingGoals --logInfoAt proofPrefix st1 let goodPre ← (← addMessageContext st1).toString - if pre.getString.trimAscii != goodPre.trimAscii then + if pre.getVersoCodeBlock.trimAscii != goodPre.trimAscii then Verso.Doc.Suggestion.saveSuggestion pre ((goodPre.take 30).copy ++ "…") (goodPre ++ "\n") - logErrorAt pre m!"Mismatch. Expected {indentD goodPre}\n but got {indentD pre.getString}" + logErrorAt pre m!"Mismatch. Expected {indentD goodPre}\n but got {indentD pre.getVersoCodeBlock}" let ci : ContextInfo := { env := ← getEnv, fileMap := ← getFileMap, ngen := ← getNGen, @@ -160,9 +160,9 @@ def checkTacticExample' --logInfoAt tactic st2 let goodPost ← (← addMessageContext st2).toString - if post.getString.trimAscii != goodPost.trimAscii then + if post.getVersoCodeBlock.trimAscii != goodPost.trimAscii then Verso.Doc.Suggestion.saveSuggestion post ((goodPost.take 30).copy ++ "…") (goodPost ++ "\n") - logErrorAt post m!"Mismatch. Expected {indentD goodPost}\n but got {indentD post.getString}" + logErrorAt post m!"Mismatch. Expected {indentD goodPost}\n but got {indentD post.getVersoCodeBlock}" let ci : ContextInfo := { ci with mctx := ← getMCtx @@ -178,14 +178,14 @@ def checkTacticExample' let txt := withNewline <| head ++ (← msg.data.toString) pure (msg.severity, txt) for (sev, txt) in processed do - if mostlyEqual config.whitespace wantedOut.getString txt then + if mostlyEqual config.whitespace wantedOut.getVersoCodeBlock txt then if let some s := config.severity then if s != sev then throwErrorAt wantedOut s!"Expected severity {sevStr s}, but got {sevStr sev}" return sev for (_, m) in processed do Verso.Doc.Suggestion.saveSuggestion wantedOut ((m.take 30).copy ++ "…") m - throwErrorAt wantedOut "Didn't match - expected one of: {indentD (toMessageData <| processed.map (·.2))}\nbut got:{indentD (toMessageData wantedOut.getString)}" + throwErrorAt wantedOut "Didn't match - expected one of: {indentD (toMessageData <| processed.map (·.2))}\nbut got:{indentD (toMessageData wantedOut.getVersoCodeBlock)}" else pure .information return (hlPre, hlPost, hlTac, outSev) @@ -231,13 +231,13 @@ open Command structure TacticExampleContext where goal : Option Expr := none setup : Option Syntax := none - pre : Option (TSyntax `str) := none + pre : Option VersoCodeBlock := none preName : Ident tactic : Option Syntax := none tacticName : Ident - post : Option (TSyntax `str) := none + post : Option VersoCodeBlock := none postName : Ident - output : Option (TSyntax `str × TacticOutputConfig) := none + output : Option (VersoCodeBlock × TacticOutputConfig) := none outputSeverityName : Ident initialize tacticExampleCtx : Lean.EnvExtension (Option TacticExampleContext) ← @@ -280,7 +280,7 @@ def saveTactic [Monad m] [MonadEnv m] [MonadError m] (tactic : Syntax) : m Ident modifyEnv fun env => tacticExampleCtx.setState env (some {st with tactic := tactic}) return st.tacticName -def savePre [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (pre : TSyntax `str) : m Ident := do +def savePre [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (pre : VersoCodeBlock) : m Ident := do match tacticExampleCtx.getState (← getEnv) with | none => throwError "Can't set pre-state - not in a tactic example" | some st => @@ -291,7 +291,7 @@ def savePre [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [Add logErrorAt (← getRef) "Pre-state already specified" return st.preName -def saveOutput [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (output : TSyntax `str) (options : TacticOutputConfig) : m Ident := do +def saveOutput [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (output : VersoCodeBlock) (options : TacticOutputConfig) : m Ident := do match tacticExampleCtx.getState (← getEnv) with | none => throwError "Can't set expected output - not in a tactic example" | some st => @@ -303,7 +303,7 @@ def saveOutput [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [ return st.outputSeverityName -def savePost [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (post : TSyntax `str) : m Ident := do +def savePost [Monad m] [MonadEnv m] [MonadLog m] [MonadRef m] [MonadError m] [AddMessageContext m] [MonadOptions m] (post : VersoCodeBlock) : m Ident := do match tacticExampleCtx.getState (← getEnv) with | none => throwError "Can't set post-state - not in a tactic example" | some st => @@ -362,7 +362,7 @@ def goal : RoleExpander let config ← TacticGoalConfig.parse.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $term:str )) := arg + let some { content := term, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let altStr ← parserInputString term @@ -388,7 +388,7 @@ def goal : RoleExpander if config.show then -- Just emit a normal Lean node - no need to do anything special with the rendered result - pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(quote hls)) #[Inline.code $(quote term.getString)])] + pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $(quote hls)) #[Inline.code $(quote term.getVersoCode)])] else pure #[] where @@ -441,7 +441,7 @@ def tacticOutput : CodeBlockExpander let outputSeverityName ← saveOutput str opts if opts.show then - return #[← `(Block.other {Verso.Genre.Manual.InlineLean.Block.leanOutput with data := ToJson.toJson (Highlighted.Message.ofSeverityString $outputSeverityName $(quote str.getString), $(quote opts.summarize), ($(quote opts.expandTraces) : List Lean.Name))} #[Block.code $(quote str.getString)])] + return #[← `(Block.other {Verso.Genre.Manual.InlineLean.Block.leanOutput with data := ToJson.toJson (Highlighted.Message.ofSeverityString $outputSeverityName $(quote str.getVersoCodeBlock), $(quote opts.summarize), ($(quote opts.expandTraces) : List Lean.Name))} #[Block.code $(quote str.getVersoCodeBlock)])] else return #[] @@ -462,7 +462,7 @@ def tacticStep : CodeBlockExpander throwErrorAt str "Failed to parse tactic step" | .ok stx => let hlTac ← saveTactic stx - pure #[← ``(Block.other (Verso.Genre.Manual.InlineLean.Block.lean $hlTac) #[Block.code $(quote str.getString)])] + pure #[← ``(Block.other (Verso.Genre.Manual.InlineLean.Block.lean $hlTac) #[Block.code $(quote str.getVersoCodeBlock)])] open Lean.Parser in @[role_expander tacticStep] @@ -471,7 +471,7 @@ def tacticStepInline : RoleExpander let () ← ArgParse.done.run args let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $tacStr:str )) := arg + let some { content := tacStr, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let altStr ← parserInputString tacStr @@ -485,7 +485,7 @@ def tacticStepInline : RoleExpander | .ok stx => let hlTac ← saveTactic stx - pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $hlTac) #[Inline.code $(quote tacStr.getString)])] + pure #[← ``(Inline.other (Verso.Genre.Manual.InlineLean.Inline.lean $hlTac) #[Inline.code $(quote tacStr.getVersoCode)])] def Block.proofState : Block where name := `Manual.proofState @@ -660,7 +660,7 @@ def pre : CodeBlockExpander -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - pure #[← `(Block.other {Block.proofState with data := ToJson.toJson (α := Option String × Array (Highlighted.Goal Highlighted)) ($(quote opts.tag), $(hlPre))} #[Block.code $(quote str.getString)])] + pure #[← `(Block.other {Block.proofState with data := ToJson.toJson (α := Option String × Array (Highlighted.Goal Highlighted)) ($(quote opts.tag), $(hlPre))} #[Block.code $(quote str.getVersoCodeBlock)])] else pure #[] @@ -673,6 +673,6 @@ def post : CodeBlockExpander -- The quote step here is to prevent the editor from showing document AST internals when the -- cursor is on the code block if opts.show then - pure #[← `(Block.other {Block.proofState with data := ToJson.toJson (α := Option String × Array (Highlighted.Goal Highlighted)) ($(quote opts.tag), $(hlPost))} #[Block.code $(quote str.getString)])] + pure #[← `(Block.other {Block.proofState with data := ToJson.toJson (α := Option String × Array (Highlighted.Goal Highlighted)) ($(quote opts.tag), $(hlPost))} #[Block.code $(quote str.getVersoCodeBlock)])] else pure #[] From 0308120068300f2d9164144853854b99077d50ba Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 17:11:37 +0200 Subject: [PATCH 10/15] test: update Mathlib docstring linter expectations The rewritten parser reports the position where a construct opened rather than the end of input, and names the delimiter it expects. --- mathlib4/MathlibTest/Linter/DocString.lean | 30 +++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/mathlib4/MathlibTest/Linter/DocString.lean b/mathlib4/MathlibTest/Linter/DocString.lean index 85894c3de..90c403778 100644 --- a/mathlib4/MathlibTest/Linter/DocString.lean +++ b/mathlib4/MathlibTest/Linter/DocString.lean @@ -164,7 +164,7 @@ example : Nat := 0 -- Errors on underscores, but not when they appear in a URL: /-- -warning: expected '_' without preceding space +warning: unterminated emphasis; expected '_' Note: This linter can be disabled with `set_option linter.style.docStringVerso false` -/ @@ -181,7 +181,19 @@ warning: expected identifier Note: This linter can be disabled with `set_option linter.style.docStringVerso false` --- -warning: unexpected end of input; expected '![', '$$', '$', '*', '[', '[^', '_', '`' or '{' +warning: expected identifier + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: unexpected ' '; expected '![', '$$', '$', '*', '[', '[^', '_', '`' or '{' + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: expected identifier + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: unexpected ' '; expected '![', '$$', '$', '*', '[', '[^', '_', '`' or '{' Note: This linter can be disabled with `set_option linter.style.docStringVerso false` -/ @@ -198,7 +210,19 @@ warning: expected identifier Note: This linter can be disabled with `set_option linter.style.docStringVerso false` --- -warning: unexpected end of input; expected '![', '$$', '$', '*', '[', '[^', '_', '`' or '{' +warning: expected identifier + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: unexpected ' '; expected '![', '$$', '*', '[', '[^', '_', '`', '{' or backticks to open inline code + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: expected identifier + +Note: This linter can be disabled with `set_option linter.style.docStringVerso false` +--- +warning: unexpected ' '; expected '![', '$$', '*', '[', '[^', '_', '`', '{' or backticks to open inline code Note: This linter can be disabled with `set_option linter.style.docStringVerso false` -/ From 7ec7425ee824622447d5f0e74054c34d8eddc0ed Mon Sep 17 00:00:00 2001 From: David Thrane Christiansen Date: Wed, 9 Sep 2026 17:11:38 +0200 Subject: [PATCH 11/15] fix: adapt verso-slides and verso-web-components to the Verso parser rewrite The expanders read views, and the long-line warning moves to the code block expander that renders wide enough for it to matter. --- verso-slides/VersoSlides/Animate.lean | 8 +- verso-slides/VersoSlides/Diagram.lean | 8 +- verso-slides/VersoSlides/Directives.lean | 38 ++++----- verso-slides/VersoSlides/InlineLean.lean | 50 +++++------ verso-slides/VersoSlides/LibModule.lean | 4 +- verso-slides/VersoSlides/ModuleExample.lean | 85 ++++++++++--------- verso-slides/VersoSlides/OtherLanguages.lean | 3 +- .../VersoWeb/Components/Gallery.lean | 39 ++++----- verso-web-components/VersoWeb/Features.lean | 42 ++++----- 9 files changed, 137 insertions(+), 140 deletions(-) diff --git a/verso-slides/VersoSlides/Animate.lean b/verso-slides/VersoSlides/Animate.lean index 9f4464b15..4704aa356 100644 --- a/verso-slides/VersoSlides/Animate.lean +++ b/verso-slides/VersoSlides/Animate.lean @@ -16,7 +16,7 @@ open Lean Elab open Verso.SyntaxUtils (parserInputString) open Verso.Genre.Manual.InlineLean.Scopes (runWithOpenDecls runWithVariables) open Verso (withoutAsync) -open Lean.Doc.Syntax +open Lean.Doc (VersoCodeBlock) namespace VersoSlides @@ -70,7 +70,7 @@ end private initialize animContainerCounter : IO.Ref Nat ← IO.mkRef 0 open Lean.Widget Lean.Elab.Term Lean.Meta Illuminate in -private meta unsafe def animateExpanderUnsafe (config : AnimateConfig) (str : StrLit) : +private meta unsafe def animateExpanderUnsafe (config : AnimateConfig) (str : VersoCodeBlock) : DocElabM Term := withoutAsync do let altStr ← parserInputString str @@ -131,11 +131,11 @@ private meta unsafe def animateExpanderUnsafe (config : AnimateConfig) (str : St let autoplay := config.autoplay ``(Verso.Doc.Block.other (VersoSlides.BlockExt.animate $(quote containerId) $(quote animDataJson) $(quote cssWidth) $(quote bg) $(quote fragmentIndices) $(quote autoplay)) - #[Verso.Doc.Block.code $(quote str.getString)]) + #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) open Lean.Widget Lean.Elab.Term Lean.Meta Illuminate in @[implemented_by animateExpanderUnsafe] -private opaque animateExpanderImpl (config : AnimateConfig) (str : StrLit) : DocElabM Term +private opaque animateExpanderImpl (config : AnimateConfig) (str : VersoCodeBlock) : DocElabM Term @[code_block] def «animate» : CodeBlockExpanderOf AnimateConfig diff --git a/verso-slides/VersoSlides/Diagram.lean b/verso-slides/VersoSlides/Diagram.lean index 1b88a8da5..a79f15f5b 100644 --- a/verso-slides/VersoSlides/Diagram.lean +++ b/verso-slides/VersoSlides/Diagram.lean @@ -15,7 +15,7 @@ open Lean Elab open Verso.SyntaxUtils (parserInputString) open Verso.Genre.Manual.InlineLean.Scopes (runWithOpenDecls runWithVariables) open Verso (withoutAsync) -open Lean.Doc.Syntax +open Lean.Doc (VersoCodeBlock) namespace VersoSlides @@ -54,7 +54,7 @@ def svgViewBoxWidth (svg : String) : Float := go.getD 640.0 open Lean.Widget Lean.Elab.Term Lean.Meta Illuminate in -private meta unsafe def diagramExpanderUnsafe (config : DiagramConfig) (str : StrLit) : +private meta unsafe def diagramExpanderUnsafe (config : DiagramConfig) (str : VersoCodeBlock) : DocElabM Term := withoutAsync do let altStr ← parserInputString str @@ -107,11 +107,11 @@ private meta unsafe def diagramExpanderUnsafe (config : DiagramConfig) (str : St let bg := config.background ``(Verso.Doc.Block.other (VersoSlides.BlockExt.diagram $(quote svgStr) $(quote cssWidth) $(quote bg)) - #[Verso.Doc.Block.code $(quote str.getString)]) + #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) open Lean.Widget Lean.Elab.Term Lean.Meta Illuminate in @[implemented_by diagramExpanderUnsafe] -private opaque diagramExpanderImpl (config : DiagramConfig) (str : StrLit) : DocElabM Term +private opaque diagramExpanderImpl (config : DiagramConfig) (str : VersoCodeBlock) : DocElabM Term @[code_block] def diagram : CodeBlockExpanderOf DiagramConfig diff --git a/verso-slides/VersoSlides/Directives.lean b/verso-slides/VersoSlides/Directives.lean index e22e22853..098c0f06a 100644 --- a/verso-slides/VersoSlides/Directives.lean +++ b/verso-slides/VersoSlides/Directives.lean @@ -17,7 +17,7 @@ public meta import Verso.Doc.Elab.Inline open Verso Doc Elab ArgParse open Lean Elab Widget -open Lean.Doc.Syntax +open Lean.Doc (CodeView ImageView InlineView UnorderedListView) register_option verso.slides.warnOnImage : Bool := { defValue := true @@ -421,10 +421,10 @@ public meta def image : RoleExpanderOf ImageArgs | args, stxs => do let mut altParts : Array String := #[] for stx in stxs do - match stx with - | `(inline| $strLit:str) => - altParts := altParts.push strLit.getString.trimAscii.copy - | `(inline| line! $_) => continue + match InlineView.of stx with + | some (.text t) => + altParts := altParts.push t.getVersoText.trimAscii.copy + | some (.linebreak _) => continue | _ => logErrorAt stx "image alt text must be plain text, not formatted content" let alt : String := " ".intercalate altParts.toList @@ -452,11 +452,11 @@ Intercepts the Markdown-like `![alt](url)` syntax and warns that the `{image}` r instead, since it supports width, height, and class, and uses local path resolution. Controlled by the `verso.slides.warnOnImage` option. After warning, delegates to the default handler. -/ -@[inline_expander Lean.Doc.Syntax.image] +@[inline_expander Lean.Doc.Parser.Inline.image] public meta def warnOnMarkdownImage : InlineExpander - | `(inline| image( $alt:str ) ( $url )) => do + | .image { alt, target := .url (url := url) .., .. } => do if (← getOptions).getBool `verso.slides.warnOnImage true then - let suggestion := "{image " ++ url.getString.quote ++ "}[" ++ alt.getString ++ "]" + let suggestion := "{image " ++ url.getVersoLinkUrl.quote ++ "}[" ++ alt.getVersoImageAlt ++ "]" let msg := m!"This image syntax is missing features that are useful for slides, such as width and height." let h ← (m!"Use the `{.ofConstName ``image}` role instead of `![alt](url)` for slides. " ++ @@ -533,15 +533,15 @@ public meta def table : DirectiveExpanderOf TableArgs | args, contents => do let #[oneBlock] := contents | throwError "Expected a single unordered list" - let `(block|ul{$items*}) := oneBlock + let some outer := UnorderedListView.of oneBlock | throwErrorAt oneBlock "Expected a single unordered list" - let preRows ← items.mapM getLi + let preRows := outer.items.map (·.contents) let rows ← preRows.mapM fun blks => do - let #[oneInRow] := blks.filter (·.raw.isOfKind ``Lean.Doc.Syntax.ul) + let #[oneInRow] := blks.filter (·.raw.isOfKind ``Lean.Doc.Parser.Block.ul) | throwError "Each row should have exactly one list in it" - let `(block|ul{ $cellItems*}) := oneInRow + let some inner := UnorderedListView.of oneInRow | throwErrorAt oneInRow "Each row should have exactly one list in it" - cellItems.mapM getLi + pure (inner.items.map (·.contents)) if h : rows.size = 0 then throwErrorAt oneBlock "Expected at least one row" else @@ -561,10 +561,6 @@ public meta def table : DirectiveExpanderOf TableArgs ``(Block.other (VersoSlides.BlockExt.table $(quote columns) $(quote style)) #[Block.ul #[$[Verso.Doc.ListItem.mk #[$blocks,*]],*]]) -where - getLi : Syntax → DocElabM (TSyntaxArray `block) - | `(list_item| * $content*) => pure content - | other => throwErrorAt other "Expected list item" /-- Custom CSS block. The content is collected during traversal and injected @@ -580,7 +576,7 @@ Usage: @[code_block] public meta def css : CodeBlockExpanderOf Unit | (), str => - ``(Verso.Doc.Block.other (BlockExt.css $(quote str.getString)) #[]) + ``(Verso.Doc.Block.other (BlockExt.css $(quote str.getVersoCodeBlock)) #[]) /-- Custom HTML block. The content is inserted directly into the slide body @@ -597,7 +593,7 @@ Usage: public meta def html : CodeBlockExpanderOf Unit | (), str => do -- The `false` parameter treats the text as unescaped raw HTML data. - let html := Verso.Output.Html.text false str.getString + let html := Verso.Output.Html.text false str.getVersoCodeBlock ``(Verso.Doc.Block.other (BlockExt.ofHtml $(quote html)) #[]) /-- @@ -613,7 +609,7 @@ public meta def htmlRole : RoleExpanderOf Unit | (), inlines => do let #[arg] := inlines | throwError "Expected a single inline code argument" - let `(inline|code( $htmlStr:str )) := arg + let some { content := htmlStr, .. } := CodeView.of arg | throwErrorAt arg "Expected inline code" - let html := Verso.Output.Html.text false htmlStr.getString + let html := Verso.Output.Html.text false htmlStr.getVersoCode ``(Verso.Doc.Inline.other (VersoSlides.InlineExt.ofHtml $(quote html)) #[]) diff --git a/verso-slides/VersoSlides/InlineLean.lean b/verso-slides/VersoSlides/InlineLean.lean index 63053e3c2..9848710b5 100644 --- a/verso-slides/VersoSlides/InlineLean.lean +++ b/verso-slides/VersoSlides/InlineLean.lean @@ -18,7 +18,8 @@ open Verso.SyntaxUtils (strLitInputContext parseStrLitAsCategory) open Verso.Genre.Manual.InlineLean (reportMessages firstToken? saveOutputs) open Verso.Genre.Manual.InlineLean.Scopes (getScopes setScopes runWithOpenDecls runWithVariables) open Verso (withoutAsync) -open Lean.Doc.Syntax +open Verso.Genre.Manual (warnLongLines) +open Lean.Doc (CodeView RoleView) register_option verso.slides.panel : Bool := { defValue := true @@ -91,8 +92,8 @@ instance : FromArgs SlidesLeanBlockConfig DocElabM where .flag `stretch true /-- Callback for `elabCommands`: produces a `Block.other (BlockExt.slideCode ...)` term. -/ -private def toSlidesHighlightedBlock (panel stretch shouldShow : Bool) (hls : Highlighted) - (str : StrLit) : DocElabM Term := do +private def toSlidesHighlightedBlock [Literal k] (panel stretch shouldShow : Bool) + (hls : Highlighted) (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Verso.Doc.Block.concat #[]) @@ -105,20 +106,20 @@ private def toSlidesHighlightedBlock (panel stretch shouldShow : Bool) (hls : Hi match fragmentize hls.trim with | .ok sc => let exported := scToExport sc - ``(Verso.Doc.Block.other (VersoSlides.BlockExt.slideCode $(quote exported) $(quote panel) $(quote stretch)) #[Verso.Doc.Block.code $(quote str.getString)]) + ``(Verso.Doc.Block.other (VersoSlides.BlockExt.slideCode $(quote exported) $(quote panel) $(quote stretch)) #[Verso.Doc.Block.code $(quote (Literal.decode str))]) | .error msg => throwErrorAt str.raw msg /-- Callback for `elabCommands`: produces an `Inline.other (InlineExt.slideCode ...)` term. -/ -private def toSlidesHighlightedInline (shouldShow : Bool) (hls : Highlighted) (str : StrLit) : - DocElabM Term := do +private def toSlidesHighlightedInline [Literal k] (shouldShow : Bool) (hls : Highlighted) + (str : TSyntax k) : DocElabM Term := do if !shouldShow then return ← ``(Verso.Doc.Inline.concat #[]) match fragmentize hls.trim with | .ok sc => let exported := scToExport sc - ``(Verso.Doc.Inline.other (VersoSlides.InlineExt.slideCode $(quote exported)) #[Verso.Doc.Inline.code $(quote str.getString)]) + ``(Verso.Doc.Inline.other (VersoSlides.InlineExt.slideCode $(quote exported)) #[Verso.Doc.Inline.code $(quote (Literal.decode str))]) | .error msg => throwErrorAt str.raw msg @@ -132,18 +133,17 @@ private def abbrevFirstLine (width : Nat) (str : String) : String := Fork of `Verso.Genre.Manual.InlineLean.elabCommands` that passes `collectFormat := true` to `highlightIncludingUnparsed`, enabling format data collection for reflowable rendering. -/ -def elabCommandsWithFormat (config : LeanBlockConfig) (str : StrLit) - (toHighlightedLeanContent : (shouldShow : Bool) → (hls : Highlighted) → (str: StrLit) → DocElabM Term) +def elabCommandsWithFormat [Literal k] (config : LeanBlockConfig) (str : TSyntax k) + (toHighlightedLeanContent : + (shouldShow : Bool) → (hls : Highlighted) → (str : TSyntax k) → DocElabM Term) (minCommands : Option Nat := none) (maxCommands : Option Nat := none) : DocElabM Term := withoutAsync <| do - PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 str.getString)) + PointOfInterest.save (← getRef) ((config.name.map (·.toString)).getD (abbrevFirstLine 20 (Literal.decode str))) (kind := Lsp.SymbolKind.file) (detail? := some ("Lean code" ++ config.outlineMeta)) - let col? := (← getRef).getPos? |>.map (← getFileMap).utf8PosToLspPos |>.map (·.character) - let origScopes ← if config.fresh then pure [{header := ""}] else getScopes let origScopes := origScopes.modifyHead fun sc => @@ -237,9 +237,6 @@ def elabCommandsWithFormat (config : LeanBlockConfig) (str : StrLit) saveOutputs name msgs reportMessages (if config.error then some true else none) str cmdState.messages - - if config.show then - Verso.Genre.Manual.warnLongLines col? str where runCommand (act : Command.CommandElabM Unit) (stx : Syntax) (cctx : Command.Context) (cmdState : Command.State) : @@ -262,7 +259,10 @@ where /-- Elaborated Lean code block for slides (with format data collection). -/ @[code_block] def lean : CodeBlockExpanderOf SlidesLeanBlockConfig - | config, str => elabCommandsWithFormat config.toLeanBlockConfig str (toSlidesHighlightedBlock config.panel config.stretch) + | config, str => do + -- Only a code block is rendered wide enough for its line lengths to matter. + if config.show then warnLongLines str + elabCommandsWithFormat config.toLeanBlockConfig str (toSlidesHighlightedBlock config.panel config.stretch) /-- Inline elaborated Lean command for slides (with format data collection). -/ @[role] @@ -279,7 +279,7 @@ def leanInline : RoleExpanderOf LeanInlineConfig | config, inlines => withoutAsync do let #[arg] := inlines | throwError "Expected exactly one argument" - let `(inline|code( $term:str )) := arg + let some { content := term, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" let leveller := if let some us := config.universes then @@ -333,9 +333,9 @@ def leanInline : RoleExpanderOf LeanInlineConfig pushInfoTree tree - if let `(inline|role{%$s $f $_*}%$e[$_*]) ← getRef then - Verso.Hover.addCustomHover (mkNullNode #[s, e]) type - Verso.Hover.addCustomHover f type + if let some v := RoleView.of ⟨← getRef⟩ then + Verso.Hover.addCustomHover (mkNullNode #[v.braceOpen, v.braceClose]) type + Verso.Hover.addCustomHover v.name type reportMessages (if config.error then some true else none) term newMsgs @@ -389,9 +389,9 @@ Usage: `{name}[List.map]` or `{name List.map'}[map']` @[role] def name : RoleExpanderOf NameConfig | cfg, #[arg] => do - let `(inline|code( $nameStx:str )) := arg + let some { content := nameStx, .. } := CodeView.of arg | throwErrorAt arg "Expected code literal with the example name" - let exampleName := nameStx.getString.toName + let exampleName := nameStx.getVersoCode.toName let identStx := mkIdentFrom arg (cfg.full.getD exampleName) (canonical := true) try @@ -400,13 +400,13 @@ def name : RoleExpanderOf NameConfig withInfoTreeContext (mkInfoTree := pure ∘ InfoTree.node (.ofCommandInfo {elaborator := `VersoSlides.name, stx := identStx})) do realizeGlobalConstNoOverloadWithInfo identStx - let hl : Highlighted ← constTok resolvedName nameStx.getString + let hl : Highlighted ← constTok resolvedName nameStx.getVersoCode let exported := hlToExport hl - ``(Verso.Doc.Inline.other (VersoSlides.InlineExt.name $(quote exported)) #[Verso.Doc.Inline.code $(quote nameStx.getString)]) + ``(Verso.Doc.Inline.other (VersoSlides.InlineExt.name $(quote exported)) #[Verso.Doc.Inline.code $(quote nameStx.getVersoCode)]) catch e => logErrorAt identStx e.toMessageData - ``(Verso.Doc.Inline.code $(quote nameStx.getString)) + ``(Verso.Doc.Inline.code $(quote nameStx.getVersoCode)) | _, more => if h : more.size > 0 then throwErrorAt more[0] "Unexpected contents" diff --git a/verso-slides/VersoSlides/LibModule.lean b/verso-slides/VersoSlides/LibModule.lean index d59a69c5a..e7f0c4623 100644 --- a/verso-slides/VersoSlides/LibModule.lean +++ b/verso-slides/VersoSlides/LibModule.lean @@ -472,7 +472,7 @@ def leanLibCode : CodeBlockExpanderOf LibModuleConfig let lineHint? : Option String ← match cfg.startLine, cfg.endLine with | some sl, some el => - match findBodyLineRange str.getString items with + match findBodyLineRange str.getVersoCodeBlock items with | some (newSl, newEl, newContents) => if newSl != sl || newEl != el then let newArgs := [ @@ -504,6 +504,6 @@ def leanLibCode : CodeBlockExpanderOf LibModuleConfig let exported := scToExport sc ``(Verso.Doc.Block.other (VersoSlides.BlockExt.slideCode $(quote exported) $(quote cfg.panel) $(quote cfg.stretch)) - #[Verso.Doc.Block.code $(quote str.getString)]) + #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) | .error msg => throwErrorAt str.raw msg diff --git a/verso-slides/VersoSlides/ModuleExample.lean b/verso-slides/VersoSlides/ModuleExample.lean index 76a9b706a..1f331029e 100644 --- a/verso-slides/VersoSlides/ModuleExample.lean +++ b/verso-slides/VersoSlides/ModuleExample.lean @@ -26,6 +26,7 @@ open Verso.Doc.Elab open Verso.ArgParse open Verso.Log open Lean +open Lean.Doc (CodeBlockView RoleView VersoCode VersoCodeBlock mkVersoCodeBlockFrom) namespace VersoSlides @@ -112,12 +113,11 @@ def lineStx [Monad m] [MonadFileMap m] (l : Nat) : m Syntax := do let r := ⟨text.lineStart (l + 1), text.lineStart (l + 2)⟩ return .ofRange r -open Lean.Doc.Syntax in @[code_block] def leanModule : CodeBlockExpanderOf ModuleConfig | { name, moduleName, error, «show», panel, stretch, lakefile }, str => do let line := (← getFileMap).utf8PosToLspPos str.raw.getPos! |>.line - let leanCode := line.fold (fun _ _ s => s.push '\n') "" ++ str.getString ++ "\n" + let leanCode := line.fold (fun _ _ s => s.push '\n') "" ++ str.getVersoCodeBlock ++ "\n" let hl ← IO.FS.withTempDir fun dirname => do let u := toString (← IO.monoMsNow) let dirname := dirname / u @@ -215,7 +215,7 @@ def leanModule : CodeBlockExpanderOf ModuleConfig | .ok sc => let exported := scToExport sc ``(Verso.Doc.Block.other (VersoSlides.BlockExt.slideCode $(quote exported) $(quote panel) $(quote stretch)) - #[Verso.Doc.Block.code $(quote str.getString)]) + #[Verso.Doc.Block.code $(quote str.getVersoCodeBlock)]) | .error msg => throwErrorAt str.raw msg else @@ -249,22 +249,24 @@ instance : FromArgs ModulesConfig m where fromArgs := ModulesConfig.mk <$> .flag `server true <*> .many (.named' `moduleRoot false) <*> .flag `error false end -open Lean.Doc.Syntax in -partial def getBlocks (block : Syntax) : StateT (NameMap (ModuleConfig × StrLit × Syntax)) DocElabM Syntax := do - if block.getKind == ``Lean.Doc.Syntax.codeblock then - if let `(Lean.Doc.Syntax.codeblock|```$x:ident $args* | $s:str ```) := block then - try - let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x - if x' == ``leanModule then - let n ← mkFreshUserName `code - let blame := mkNullNode <| #[x] ++ args - let argVals ← parseArgs args - let cfg ← fromArgs.run argVals - modify (·.insert n (cfg, s, blame)) - let x := mkIdentFrom block n - return ← `(Lean.Doc.Syntax.codeblock|```identRef $x:ident | $(quote "") ```) - catch - | _ => pure () +partial def getBlocks (block : Syntax) : + StateT (NameMap (ModuleConfig × VersoCodeBlock × Syntax)) DocElabM Syntax := do + if let some { openFence, name? := some x, args, content := s, closeFence, .. } := + CodeBlockView.of ⟨block⟩ then + try + let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x + if x' == ``leanModule then + let n ← mkFreshUserName `code + let blame := mkNullNode <| #[x.raw] ++ args.map (·.raw) + let argVals ← parseArgs args + let cfg ← fromArgs.run argVals + modify (·.insert n (cfg, s, blame)) + let x := mkIdentFrom block n + return ← `(Lean.Doc.Parser.Block.codeblock| + $openFence:codeBlockFence identRef $x:ident + $(mkVersoCodeBlockFrom s ""):versoCodeBlock $closeFence:codeBlockFence) + catch + | _ => pure () match block with | .node i k xs => do @@ -272,24 +274,23 @@ partial def getBlocks (block : Syntax) : StateT (NameMap (ModuleConfig × StrLit return Syntax.node i k args | _ => return block -open Lean.Doc.Syntax in open Verso.Doc (oneCodeStr?) in -partial def getQuotes (stx : Syntax) : StateT (NameMap StrLit) DocElabM Syntax := do - if stx.getKind == ``Lean.Doc.Syntax.role then - if let `(Lean.Doc.Syntax.role|role{$x:ident $args*}[$inls*]) := stx then - try - let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x - if x' == ``VersoSlides.name then - unless args.isEmpty do logErrorAt (mkNullNode args) m!"No arguments expected here" - let some code ← oneCodeStr? inls - | return ((← `(.empty)) : Syntax) - - let n ← mkFreshUserName `name - modify (·.insert n code) - let x := mkIdentFrom stx n - return ((← `(Lean.Doc.Syntax.role|role{identRef $x:ident}[])) : Syntax) - catch - | _ => pure () +partial def getQuotes (stx : Syntax) : StateT (NameMap VersoCode) DocElabM Syntax := do + if let some { name := x, args, content := inls, .. } := RoleView.of ⟨stx⟩ then + try + let x' ← Elab.realizeGlobalConstNoOverloadWithInfo x + if x' == ``VersoSlides.name then + unless args.isEmpty do + logErrorAt (mkNullNode (args.map (·.raw))) m!"No arguments expected here" + let some code ← oneCodeStr? inls + | return ((← `(.empty)) : Syntax) + + let n ← mkFreshUserName `name + modify (·.insert n code) + let x := mkIdentFrom stx n + return ((← `(Lean.Doc.Parser.Inline.role|{identRef $x:ident}[])) : Syntax) + catch + | _ => pure () match stx with | .node i k xs => do @@ -348,7 +349,7 @@ def leanModules : DirectiveExpanderOf ModulesConfig let mut mods := #[] for (x, modConfig, s, blame) in codeBlocks do let line := (← getFileMap).utf8PosToLspPos s.raw.getPos! |>.line - let leanCode := line.fold (fun _ _ str => str.push '\n') "" ++ s.getString ++ "\n" + let leanCode := line.fold (fun _ _ str => str.push '\n') "" ++ s.getVersoCodeBlock ++ "\n" if modConfig.lakefile then -- Write as lakefile.lean IO.FS.writeFile (dirname / "lakefile.lean") leanCode @@ -454,7 +455,7 @@ def leanModules : DirectiveExpanderOf ModulesConfig let exported := scToExport sc ``((Verso.Doc.Block.other (VersoSlides.BlockExt.slideCode $(quote exported) $(quote modConfig.panel) $(quote modConfig.stretch)) - #[Verso.Doc.Block.code $(quote s.getString)] : Verso.Doc.Block Slides)) + #[Verso.Doc.Block.code $(quote s.getVersoCodeBlock)] : Verso.Doc.Block Slides)) | .error msg => throwErrorAt blame msg else @@ -469,18 +470,18 @@ def leanModules : DirectiveExpanderOf ModulesConfig let (blocks, quotes) ← blocks.mapM getQuotes |>.run {} for (x, q) in quotes do - if let some tok := allHl.matchingName? q.getString then + if let some tok := allHl.matchingName? q.getVersoCode then addLets := addLets >=> fun stx => do let hl : Highlighted := .token tok match fragmentize hl with | .ok sc => let exported := scToExport sc - let name ← ``((Verso.Doc.Inline.other (VersoSlides.InlineExt.slideCode $(quote exported)) #[Verso.Doc.Inline.code $(quote q.getString)] : Verso.Doc.Inline Slides)) + let name ← ``((Verso.Doc.Inline.other (VersoSlides.InlineExt.slideCode $(quote exported)) #[Verso.Doc.Inline.code $(quote q.getVersoCode)] : Verso.Doc.Inline Slides)) `(let $(mkIdent x) := $name; $stx) | .error _ => - let name ← ``((Verso.Doc.Inline.code $(quote q.getString) : Verso.Doc.Inline Slides)) + let name ← ``((Verso.Doc.Inline.code $(quote q.getVersoCode) : Verso.Doc.Inline Slides)) `(let $(mkIdent x) := $name; $stx) - else logErrorAt q m!"Not found: {q.getString.quote}" + else logErrorAt q m!"Not found: {q.getVersoCode.quote}" let body ← blocks.mapM (elabBlock <| ⟨·⟩) let body ← ``((Verso.Doc.Block.concat #[$body,*] : Verso.Doc.Block Slides)) addLets body diff --git a/verso-slides/VersoSlides/OtherLanguages.lean b/verso-slides/VersoSlides/OtherLanguages.lean index 6c32107a4..de4af5b81 100644 --- a/verso-slides/VersoSlides/OtherLanguages.lean +++ b/verso-slides/VersoSlides/OtherLanguages.lean @@ -31,7 +31,6 @@ The language name can be an identifier (`rust`, `python`) or a string open Lean Elab open Verso Doc Elab -open Lean.Doc.Syntax namespace VersoSlides @@ -57,6 +56,6 @@ Uses `reveal.js`'s built-in syntax highlighting for code. @[code_block] def code : CodeBlockExpanderOf CodeConfig | config, str => - ``(Verso.Doc.Block.other (BlockExt.otherLanguage $(quote config.language) $(quote str.getString)) #[]) + ``(Verso.Doc.Block.other (BlockExt.otherLanguage $(quote config.language) $(quote str.getVersoCodeBlock)) #[]) end VersoSlides diff --git a/verso-web-components/VersoWeb/Components/Gallery.lean b/verso-web-components/VersoWeb/Components/Gallery.lean index e75ab76e7..609ced595 100644 --- a/verso-web-components/VersoWeb/Components/Gallery.lean +++ b/verso-web-components/VersoWeb/Components/Gallery.lean @@ -7,7 +7,7 @@ import VersoWeb.Components.Icon import Lean.Elab open Lean Elab Term -open Lean.Doc.Syntax +open Lean.Doc (DescListView DescItemView ImageView TextView) open Verso Genre Blog ArgParse Doc Elab open Output Html Traverse @@ -76,31 +76,28 @@ block_component galleryItem (name url title : String) where /-- Translates an item of a description list into a gallery item component. -/ -private def getItem (name : String) : Syntax → DocElabM (TSyntax `term) - | `(desc_item|: $dts* => $dds*) => do - let #[img] := dts.filter fun - | `(inline|$s:str) => s.getString.any (fun c : Char => !c.isWhitespace) - | _ => true - | throwErrorAt (mkNullNode dts.raw) "Expected a single image, got {dts}" - let `(inline|image($title)$dest) := img - | throwErrorAt img "Expected an image, got {img}" - let title := title.getString - let `(link_target|( $url:str )) := dest - | throwErrorAt dest "Expected URL, got {dest}" - - ``(galleryItem $(quote name) $(quote url.getString) $(quote title) #[$[$(← dds.mapM elabBlock)],*]) - - | other => throwErrorAt other "Failed to parse description list item" +private def getItem (name : String) (item : DescItemView) : DocElabM (TSyntax `term) := do + let terms := mkNullNode (item.term.map (·.raw)) + let #[img] := item.term.filter fun inl => + match TextView.of inl with + | some t => t.getVersoText.any (fun c : Char => !c.isWhitespace) + | none => true + | throwErrorAt terms m!"Expected a single image, got {terms}" + let some image := ImageView.of img + | throwErrorAt img m!"Expected an image, got {img}" + let .url (url := url) .. := image.target + | throwErrorAt img m!"Expected an image with a URL, got {img}" + + ``(galleryItem $(quote name) $(quote url.getVersoLinkUrl) $(quote image.getAlt) #[$[$(← item.desc.mapM elabBlock)],*]) @[directive_expander gallery, inherit_doc gallery] def galleryDir : DirectiveExpander | args, #[block] => do let name ← ArgParse.run (.positional `name .string) args - match block with - | `(block|dl{$item*})=> - let items ← item.mapM (getItem name) - pure #[← ``(gallery #[$items,*])] - | other => throwErrorAt other "Expected a description list" + let some { items, .. } := DescListView.of block + | throwErrorAt block "Expected a description list" + let items ← items.mapM (getItem name) + pure #[← ``(gallery #[$items,*])] | _, more => do if h : more.size > 1 then throwErrorAt more[1] "Expected only a single block" diff --git a/verso-web-components/VersoWeb/Features.lean b/verso-web-components/VersoWeb/Features.lean index 458b53045..37c4fede0 100644 --- a/verso-web-components/VersoWeb/Features.lean +++ b/verso-web-components/VersoWeb/Features.lean @@ -11,12 +11,13 @@ namespace Verso.Web open Verso.Genre.Blog open Verso Doc Elab open Lean Elab -open Lean.Doc.Syntax +open Lean.Doc (CodeBlockView CodeView TextView) open Verso.ArgParse open Verso.Output (Html) -private def codeblockContents : Lean.Syntax → Option String - | `(block|``` | $contents ```) => some contents.getString +private def codeblockContents (stx : TSyntax ``Lean.Doc.Parser.block) : Option String := + match CodeBlockView.of stx with + | some { name? := none, content, .. } => some content.getVersoCodeBlock | _ => none @[directive_expander diff] @@ -43,16 +44,18 @@ def diffs : DirectiveExpander pure #[← ``(Block.other (BlockExt.htmlDiv "diff-view") #[$blockStx,*])] | _, _ => throwUnsupportedSyntax where - doBlock : Syntax → DocElabM (String × Array (TSyntax `term)) - | `(block|```|$contents```) => do - let lines ← contents.getString.dropEndWhile (· == '\n') |>.copy.splitOn "\n" |>.toArray.mapM fun l => `(Block.code $(quote l)) - pure ("plain", lines) - | `(block|```$nameStx|$contents```) => do - let cls := nameStx.getId.toString - if cls ∉ ["ins", "del"] then throwErrorAt nameStx "Expected 'ins' or 'del'" - let lines ← contents.getString.dropEndWhile (· == '\n') |>.copy.splitOn "\n" |>.toArray.mapM fun l => `(Block.code $(quote l)) - pure (cls, lines) - | blk => dbg_trace blk; throwErrorAt blk "Expected code block (unnamed, or with 'ins' or 'del')" + doBlock (blk : TSyntax ``Lean.Doc.Parser.block) : DocElabM (String × Array (TSyntax `term)) := do + let some { name?, content, .. } := CodeBlockView.of blk + | throwErrorAt blk "Expected code block (unnamed, or with 'ins' or 'del')" + let cls ← + match name? with + | none => pure "plain" + | some nameStx => + let cls := nameStx.getId.toString + if cls ∉ ["ins", "del"] then throwErrorAt nameStx "Expected 'ins' or 'del'" + pure cls + let lines ← content.getVersoCodeBlock.dropEndWhile (· == '\n') |>.copy.splitOn "\n" |>.toArray.mapM fun l => `(Block.code $(quote l)) + pure (cls, lines) -- Stolen from Lean.Parser.Module @@ -242,10 +245,11 @@ open Verso.Output Html def kbd : RoleExpander | args, items => do ArgParse.done.run args - let strs ← items.filterMapM fun - | `(inline|code( $s:str )) => pure (some s.getString) - | `(inline|$s:str) => pure none - | other => logErrorAt other m!"Expected a code element, got {other}" *> pure none + let strs ← items.filterMapM fun inl => + match CodeView.of inl, TextView.of inl with + | some { content := s, .. }, _ => pure (some s.getVersoCode) + | none, some _ => pure none + | none, none => logErrorAt inl m!"Expected a code element, got {inl}" *> pure none if h : strs.size = 0 then throwError "Expected one or more inline code literals" else let basic := String.intercalate "+" strs.toList @@ -262,9 +266,9 @@ def color : RoleExpander ArgParse.done.run args let #[str] := items | throwError "Expected exactly one inline code element" - let `(inline|code( $s:str )) := str + let some { content := s, .. } := CodeView.of str | throwErrorAt str "Expected an inline code element" - let s := s.getString + let s := s.getVersoCode let html : Html := {{{{s}}}} return #[← ``(Inline.other (InlineExt.blob $(quote html)) #[Inline.code $(quote s)])] From d972677e0b6d1fb7bbfe6c9967b07e7004961250 Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Thu, 10 Sep 2026 07:45:25 +0000 Subject: [PATCH 12/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index 6b93c2d94..ecb239f64 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4-pr-releases:pr-release-15064-2a13699 +leanprover/lean4-pr-releases:pr-release-15064-0db7882 From 9fd6577a92e387182db4da88cbeba33e403f8fad Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Fri, 11 Sep 2026 12:31:27 +0000 Subject: [PATCH 13/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index ecb239f64..35f849111 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4-pr-releases:pr-release-15064-0db7882 +leanprover/lean4-pr-releases:pr-release-15064-9f40692 From b8ddcacd973e4d6529f2e34d7c6d17b8a59a2cd6 Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Fri, 11 Sep 2026 14:42:31 +0000 Subject: [PATCH 14/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index 35f849111..dd3095106 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4-pr-releases:pr-release-15064-9f40692 +leanprover/lean4-pr-releases:pr-release-15064-5132248 From 7b1bb7f1f0abac8fb417ce1d04bfa49bbfe939dd Mon Sep 17 00:00:00 2001 From: "downstream-lean4[bot]" Date: Mon, 14 Sep 2026 08:06:26 +0000 Subject: [PATCH 15/15] downstream: follow upstream PR --- lean-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean-toolchain b/lean-toolchain index c0afdab9f..45409ef75 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4:nightly-2026-09-12 +leanprover/lean4-pr-releases:pr-release-15064-88bbfc6