Markdown Parser Rework

quartz.config.ts

@@ -1,5 +1,9 @@
 import { QuartzConfig } from "./quartz/cfg"
 import * as Plugin from "./quartz/plugins"
+import * as Text from "./quartz/plugins/transformers/text"
+import * as Markdown from "./quartz/plugins/transformers/markdown"
+import * as Html from "./quartz/plugins/transformers/html"
+import * as Resources from "./quartz/plugins/transformers/resources"
 
 /**
  * Quartz 4.0 Configuration
@@ -54,25 +58,51 @@ const config: QuartzConfig = {
     },
   },
   plugins: {
-    transformers: [
-      Plugin.FrontMatter(),
-      Plugin.CreatedModifiedDate({
-        priority: ["frontmatter", "filesystem"],
-      }),
-      Plugin.SyntaxHighlighting({
-        theme: {
-          light: "github-light",
-          dark: "github-dark",
-        },
-        keepBackground: false,
-      }),
-      Plugin.ObsidianFlavoredMarkdown({ enableInHtmlEmbed: false }),
-      Plugin.GitHubFlavoredMarkdown(),
-      Plugin.TableOfContents(),
-      Plugin.CrawlLinks({ markdownLinkResolution: "shortest" }),
-      Plugin.Description(),
-      Plugin.Latex({ renderEngine: "katex" }),
-    ],
+    transformers: {
+      textTransformers: [
+        Text.ObsidianFlavoredMarkdownComments(),
+        Text.ObsidianFlavoredMarkdownCallouts(),
+        Text.ObsidianFlavoredMarkdownWikilinks(),
+      ],
+      markdownTransformers: [
+        Markdown.FrontMatter(),
+        Markdown.CreatedModifiedDate({
+          priority: ["frontmatter", "filesystem"],
+        }),
+        Markdown.ObsidianFlavoredMarkdownWikilinks(),
+        Markdown.ObsidianFlavoredMarkdownHighlight(),
+        Markdown.ObsidianFlavoredMarkdownArrow(),
+        Markdown.ObsidianFlavoredMarkdownTags(),
+        Markdown.ObsidianFlavoredMarkdownVideoEmbed(),
+        Markdown.ObsidianFlavoredMarkdownCallouts(),
+        Markdown.ObsidianFlavoredMarkdownMermaid(),
+        Markdown.GitHubFlavoredMarkdownRemark(),
+        Markdown.TableOfContents(),
+        Markdown.Latex(),
+      ],
+      htmlTransformers: [
+        Html.SyntaxHighlighting({
+          theme: {
+            light: "github-light",
+            dark: "github-dark",
+          },
+          keepBackground: false,
+        }),
+        Html.ObsidianFlavoredMarkdownBlockReferences(),
+        Html.ObsidianFlavoredMarkdownYouTubeEmbed(),
+        Html.ObsidianFlavoredMarkdownCheckbox(),
+        Html.GitHubFlavoredMarkdownLinkHeadings(),
+        Html.CrawlLinks({ markdownLinkResolution: "shortest" }),
+        Html.Description(),
+        Html.Latex({ renderEngine: "katex" }),
+      ],
+      externalResources: [
+        Resources.ObsidianFlavoredMarkdownCheckbox(),
+        Resources.ObsidianFlavoredMarkdownCallouts(),
+        Resources.ObsidianFlavoredMarkdownMermaid(),
+        Resources.Latex({ renderEngine: "katex" }),
+      ],
+    },
     filters: [Plugin.RemoveDrafts()],
     emitters: [
       Plugin.AliasRedirects(),

quartz/build.ts

@@ -54,11 +54,21 @@ async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) {
   const output = argv.output
 
   const pluginCount = Object.values(cfg.plugins).flat().length
-  const pluginNames = (key: "transformers" | "filters" | "emitters") =>
-    cfg.plugins[key].map((plugin) => plugin.name)
+  const pluginNames = (key: "filters" | "emitters") => cfg.plugins[key].map((plugin) => plugin.name)
   if (argv.verbose) {
     console.log(`Loaded ${pluginCount} plugins`)
-    console.log(`  Transformers: ${pluginNames("transformers").join(", ")}`)
+    console.log(
+      `  Text Transformers: ${cfg.plugins["transformers"].textTransformers.map((plugin) => plugin.name).join(", ")}`,
+    )
+    console.log(
+      `  Markdown Transformers: ${cfg.plugins["transformers"].markdownTransformers.map((plugin) => plugin.name).join(", ")}`,
+    )
+    console.log(
+      `  Html Transformers: ${cfg.plugins["transformers"].htmlTransformers.map((plugin) => plugin.name).join(", ")}`,
+    )
+    console.log(
+      `  External Resources: ${cfg.plugins["transformers"].externalResources.map((plugin) => plugin.name).join(", ")}`,
+    )
     console.log(`  Filters: ${pluginNames("filters").join(", ")}`)
     console.log(`  Emitters: ${pluginNames("emitters").join(", ")}`)
   }

quartz/plugins/index.ts

@@ -8,8 +8,8 @@ export function getStaticResourcesFromPlugins(ctx: BuildCtx) {
     js: [],
   }
 
-  for (const transformer of ctx.cfg.plugins.transformers) {
-    const res = transformer.externalResources ? transformer.externalResources(ctx) : {}
+  for (const transformer of ctx.cfg.plugins.transformers.externalResources) {
+    const res = transformer ? transformer.transformation(ctx) : {}
     if (res?.js) {
       staticResources.js.push(...res.js)
     }
@@ -38,7 +38,10 @@ export function getStaticResourcesFromPlugins(ctx: BuildCtx) {
   return staticResources
 }
 
-export * from "./transformers"
+export * as Text from "./transformers/text"
+export * as Markdown from "./transformers/markdown"
+export * as Html from "./transformers/html"
+export * as Resources from "./transformers/resources"
 export * from "./filters"
 export * from "./emitters"
 

quartz/plugins/transformers/citations.ts → quartz/plugins/transformers/html/citations.ts

@@ -1,7 +1,7 @@
 import rehypeCitation from "rehype-citation"
 import { PluggableList } from "unified"
 import { visit } from "unist-util-visit"
-import { QuartzTransformerPlugin } from "../types"
+import { HtmlTransformerPlugin } from "../../types"
 
 export interface Options {
   bibliographyFile: string
@@ -17,11 +17,11 @@ const defaultOptions: Options = {
   csl: "apa",
 }
 
-export const Citations: QuartzTransformerPlugin<Partial<Options>> = (userOpts) => {
+export const Citations: HtmlTransformerPlugin<Partial<Options>> = (userOpts) => {
   const opts = { ...defaultOptions, ...userOpts }
   return {
     name: "Citations",
-    htmlPlugins(ctx) {
+    transformation(ctx) {
       const plugins: PluggableList = []
 
       // Add rehype-citation to the list of plugins

quartz/plugins/transformers/description.ts → quartz/plugins/transformers/html/description.ts

@@ -1,7 +1,7 @@
 import { Root as HTMLRoot } from "hast"
 import { toString } from "hast-util-to-string"
-import { QuartzTransformerPlugin } from "../types"
-import { escapeHTML } from "../../util/escape"
+import { HtmlTransformerPlugin } from "../../types"
+import { escapeHTML } from "../../../util/escape"
 
 export interface Options {
   descriptionLength: number
@@ -18,11 +18,11 @@ const urlRegex = new RegExp(
   "g",
 )
 
-export const Description: QuartzTransformerPlugin<Partial<Options>> = (userOpts) => {
+export const Description: HtmlTransformerPlugin<Partial<Options>> = (userOpts) => {
   const opts = { ...defaultOptions, ...userOpts }
   return {
     name: "Description",
-    htmlPlugins() {
+    transformation() {
       return [
         () => {
           return async (tree: HTMLRoot, file) => {

quartz/plugins/transformers/gfm.ts → quartz/plugins/transformers/html/gfmLinkHeadings.ts

@@ -1,27 +1,22 @@
-import remarkGfm from "remark-gfm"
-import smartypants from "remark-smartypants"
-import { QuartzTransformerPlugin } from "../types"
+import { HtmlTransformerPlugin } from "../../types"
 import rehypeSlug from "rehype-slug"
 import rehypeAutolinkHeadings from "rehype-autolink-headings"
 
 export interface Options {
-  enableSmartyPants: boolean
   linkHeadings: boolean
 }
 
 const defaultOptions: Options = {
-  enableSmartyPants: true,
   linkHeadings: true,
 }
 
-export const GitHubFlavoredMarkdown: QuartzTransformerPlugin<Partial<Options>> = (userOpts) => {
+export const GitHubFlavoredMarkdownLinkHeadings: HtmlTransformerPlugin<Partial<Options>> = (
+  userOpts,
+) => {
   const opts = { ...defaultOptions, ...userOpts }
   return {
-    name: "GitHubFlavoredMarkdown",
-    markdownPlugins() {
-      return opts.enableSmartyPants ? [remarkGfm, smartypants] : [remarkGfm]
-    },
-    htmlPlugins() {
+    name: "GitHubFlavoredMarkdownLinkHeadings",
+    transformation() {
       if (opts.linkHeadings) {
         return [
           rehypeSlug,

quartz/plugins/transformers/html/index.ts

@@ -0,0 +1,9 @@
+export { Citations } from "./citations"
+export { CrawlLinks } from "./links"
+export { Description } from "./description"
+export { GitHubFlavoredMarkdownLinkHeadings } from "./gfmLinkHeadings"
+export { Latex } from "./latex"
+export { SyntaxHighlighting } from "./syntax"
+export { ObsidianFlavoredMarkdownBlockReferences } from "./ofmBlockReferences"
+export { ObsidianFlavoredMarkdownCheckbox } from "./ofmCheckbox"
+export { ObsidianFlavoredMarkdownYouTubeEmbed } from "./ofmYoutubeEmbed"

quartz/plugins/transformers/html/latex.ts

@@ -0,0 +1,27 @@
+import rehypeKatex from "rehype-katex"
+import rehypeMathjax from "rehype-mathjax/svg"
+import { HtmlTransformerPlugin } from "../../types"
+
+interface Options {
+  renderEngine: "katex" | "mathjax"
+  customMacros: MacroType
+}
+
+interface MacroType {
+  [key: string]: string
+}
+
+export const Latex: HtmlTransformerPlugin<Partial<Options>> = (opts) => {
+  const engine = opts?.renderEngine ?? "katex"
+  const macros = opts?.customMacros ?? {}
+  return {
+    name: "Latex",
+    transformation() {
+      if (engine === "katex") {
+        return [[rehypeKatex, { output: "html", macros }]]
+      } else {
+        return [[rehypeMathjax, { macros }]]
+      }
+    },
+  }
+}

quartz/plugins/transformers/links.ts → quartz/plugins/transformers/html/links.ts

@@ -1,4 +1,4 @@
-import { QuartzTransformerPlugin } from "../types"
+import { HtmlTransformerPlugin, QuartzTransformerPlugin } from "../../types"
 import {
   FullSlug,
   RelativeURL,
@@ -8,7 +8,7 @@ import {
   simplifySlug,
   splitAnchor,
   transformLink,
-} from "../../util/path"
+} from "../../../util/path"
 import path from "path"
 import { visit } from "unist-util-visit"
 import isAbsoluteUrl from "is-absolute-url"
@@ -32,11 +32,11 @@ const defaultOptions: Options = {
   externalLinkIcon: true,
 }
 
-export const CrawlLinks: QuartzTransformerPlugin<Partial<Options>> = (userOpts) => {
+export const CrawlLinks: HtmlTransformerPlugin<Partial<Options>> = (userOpts) => {
   const opts = { ...defaultOptions, ...userOpts }
   return {
     name: "LinkProcessing",
-    htmlPlugins(ctx) {
+    transformation(ctx) {
       return [
         () => {
           return (tree: Root, file) => {

quartz/plugins/transformers/html/ofmBlockReferences.ts

@@ -0,0 +1,99 @@
+import { HtmlTransformerPlugin } from "../../types"
+import { Element, Literal, Root as HtmlRoot } from "hast"
+import rehypeRaw from "rehype-raw"
+import { visit } from "unist-util-visit"
+import { PluggableList } from "unified"
+
+const blockReferenceRegex = new RegExp(/\^([-_A-Za-z0-9]+)$/g)
+
+export const ObsidianFlavoredMarkdownBlockReferences: HtmlTransformerPlugin = () => {
+  return {
+    name: "ObsidianFlavoredMarkdownBlockReferences",
+    transformation() {
+      const plugins: PluggableList = [rehypeRaw]
+
+      plugins.push(() => {
+        const inlineTagTypes = new Set(["p", "li"])
+        const blockTagTypes = new Set(["blockquote"])
+        return (tree: HtmlRoot, file) => {
+          file.data.blocks = {}
+
+          visit(tree, "element", (node, index, parent) => {
+            if (blockTagTypes.has(node.tagName)) {
+              const nextChild = parent?.children.at(index! + 2) as Element
+              if (nextChild && nextChild.tagName === "p") {
+                const text = nextChild.children.at(0) as Literal
+                if (text && text.value && text.type === "text") {
+                  const matches = text.value.match(blockReferenceRegex)
+                  if (matches && matches.length >= 1) {
+                    parent!.children.splice(index! + 2, 1)
+                    const block = matches[0].slice(1)
+
+                    if (!Object.keys(file.data.blocks!).includes(block)) {
+                      node.properties = {
+                        ...node.properties,
+                        id: block,
+                      }
+                      file.data.blocks![block] = node
+                    }
+                  }
+                }
+              }
+            } else if (inlineTagTypes.has(node.tagName)) {
+              const last = node.children.at(-1) as Literal
+              if (last && last.value && typeof last.value === "string") {
+                const matches = last.value.match(blockReferenceRegex)
+                if (matches && matches.length >= 1) {
+                  last.value = last.value.slice(0, -matches[0].length)
+                  const block = matches[0].slice(1)
+
+                  if (last.value === "") {
+                    // this is an inline block ref but the actual block
+                    // is the previous element above it
+                    let idx = (index ?? 1) - 1
+                    while (idx >= 0) {
+                      const element = parent?.children.at(idx)
+                      if (!element) break
+                      if (element.type !== "element") {
+                        idx -= 1
+                      } else {
+                        if (!Object.keys(file.data.blocks!).includes(block)) {
+                          element.properties = {
+                            ...element.properties,
+                            id: block,
+                          }
+                          file.data.blocks![block] = element
+                        }
+                        return
+                      }
+                    }
+                  } else {
+                    // normal paragraph transclude
+                    if (!Object.keys(file.data.blocks!).includes(block)) {
+                      node.properties = {
+                        ...node.properties,
+                        id: block,
+                      }
+                      file.data.blocks![block] = node
+                    }
+                  }
+                }
+              }
+            }
+          })
+
+          file.data.htmlAst = tree
+        }
+      })
+
+      return plugins
+    },
+  }
+}
+
+declare module "vfile" {
+  interface DataMap {
+    blocks: Record<string, Element>
+    htmlAst: HtmlRoot
+  }
+}