|
| 1 | +"""Sphinx extension that writes a Markdown copy of each page next to its HTML. |
| 2 | +
|
| 3 | +Uses the Markdown translator from sphinx-markdown-builder, but runs it from |
| 4 | +the HTML builder's ``write_doc`` instead of as a separate |
| 5 | +``sphinx-build -b markdown``: conf.py regenerates every source file, so a |
| 6 | +second build would re-read all ~20k documents. Converting the doctree the |
| 7 | +HTML builder just rendered also avoids resolving it a second time, and runs |
| 8 | +in the worker processes when writing in parallel. |
| 9 | +
|
| 10 | +Internal links point at the sibling ``.md`` files. |
| 11 | +""" |
| 12 | + |
| 13 | +import json |
| 14 | +import posixpath |
| 15 | +import re |
| 16 | +from urllib.parse import unquote, urlsplit, urlunsplit |
| 17 | + |
| 18 | +from docutils import nodes |
| 19 | +from sphinx_markdown_builder import MarkdownBuilder |
| 20 | +from sphinx_markdown_builder.escape import escape_markdown_chars |
| 21 | + |
| 22 | + |
| 23 | +def _skip_node(self, node): |
| 24 | + raise nodes.SkipNode |
| 25 | + |
| 26 | + |
| 27 | +def _visit_literal(self, node): |
| 28 | + # Use a longer delimiter so backticks inside code samples remain literal. |
| 29 | + # DataBrew's ":col1": "`Column A`". becomes ``":col1": "`Column A`".``. |
| 30 | + text = node.astext() |
| 31 | + longest_run = max((len(run) for run in re.findall(r'`+', text)), default=0) |
| 32 | + delimiter = '`' * (longest_run + 1) |
| 33 | + # Padding separates edge backticks from the delimiter, and preserves |
| 34 | + # surrounding spaces that Markdown would otherwise strip. |
| 35 | + if ( |
| 36 | + text.startswith('`') |
| 37 | + or text.endswith('`') |
| 38 | + or (text.startswith(' ') and text.endswith(' ') and text.strip()) |
| 39 | + ): |
| 40 | + text = f' {text} ' |
| 41 | + self.add(f'{delimiter}{text}{delimiter}') |
| 42 | + raise nodes.SkipNode |
| 43 | + |
| 44 | + |
| 45 | +def _visit_problematic(self, node): |
| 46 | + # Markup that docutils could not parse. The translator puts it in its own |
| 47 | + # code block, which splits the surrounding paragraph or list item, so |
| 48 | + # write it inline like the HTML does. |
| 49 | + self.add(escape_markdown_chars(node.astext())) |
| 50 | + raise nodes.SkipNode |
| 51 | + |
| 52 | + |
| 53 | +def _create_markdown_builder(app): |
| 54 | + # Builder.__init__ adds the builder's name and format to app.tags, which |
| 55 | + # would make ``.. only:: markdown`` content show up in the HTML output. |
| 56 | + tags = set(app.tags) |
| 57 | + builder = MarkdownBuilder(app, app.env) |
| 58 | + for tag in set(app.tags) - tags: |
| 59 | + app.tags.remove(tag) |
| 60 | + builder.init() |
| 61 | + builder.prepare_writing(set()) |
| 62 | + return builder |
| 63 | + |
| 64 | + |
| 65 | +def _link_to_markdown(doctree, docname, found_docs): |
| 66 | + # Rewrite links to generated HTML documents to their Markdown counterparts. |
| 67 | + for node in doctree.findall(nodes.reference): |
| 68 | + try: |
| 69 | + uri = urlsplit(node.get('refuri', '')) |
| 70 | + except ValueError: |
| 71 | + continue |
| 72 | + if ( |
| 73 | + uri.scheme |
| 74 | + or uri.netloc |
| 75 | + or uri.path.startswith('/') |
| 76 | + or not uri.path.endswith('.html') |
| 77 | + ): |
| 78 | + continue |
| 79 | + # Explicit relative links lack Sphinx's "internal" flag. Only |
| 80 | + # rewrite them when their target is a generated document. |
| 81 | + target = posixpath.normpath( |
| 82 | + posixpath.join(posixpath.dirname(docname), unquote(uri.path)) |
| 83 | + )[:-5] |
| 84 | + if node.get('internal') or target in found_docs: |
| 85 | + node['refuri'] = urlunsplit(uri._replace(path=uri.path[:-5] + '.md')) |
| 86 | + |
| 87 | + |
| 88 | +def _remove_breadcrumbs(doctree): |
| 89 | + # Generated pages start with a "[ aws . s3api ]" breadcrumb paragraph |
| 90 | + # (see doc_breadcrumbs in awscli/clidocs.py), which is navigation noise |
| 91 | + # in Markdown. It is the first paragraph, before the title section. |
| 92 | + for node in doctree.children: |
| 93 | + if isinstance(node, nodes.section): |
| 94 | + return |
| 95 | + if isinstance(node, nodes.paragraph): |
| 96 | + text = node.astext() |
| 97 | + if text.startswith('[ aws') and text.endswith(' ]'): |
| 98 | + doctree.remove(node) |
| 99 | + return |
| 100 | + |
| 101 | + |
| 102 | +def _guess_code_language(code): |
| 103 | + code = code.lstrip() |
| 104 | + if code.startswith('aws '): |
| 105 | + return 'bash' |
| 106 | + # "JSON Syntax" templates are not valid JSON, e.g. "true|false" or "..." |
| 107 | + if code.startswith('{'): |
| 108 | + return 'json' |
| 109 | + try: |
| 110 | + json.loads(code) |
| 111 | + except ValueError: |
| 112 | + return '' |
| 113 | + return 'json' |
| 114 | + |
| 115 | + |
| 116 | +def _set_code_languages(doctree): |
| 117 | + # Code blocks without an explicit language get Sphinx's "default", which |
| 118 | + # is not a language Markdown renderers know, so guess one instead. |
| 119 | + for node in doctree.findall(nodes.literal_block): |
| 120 | + if node.get('language') == 'default': |
| 121 | + node['language'] = _guess_code_language(node.astext()) |
| 122 | + |
| 123 | + |
| 124 | +def _install(app): |
| 125 | + if app.builder.name != 'html': |
| 126 | + return |
| 127 | + markdown_builder = _create_markdown_builder(app) |
| 128 | + write_html = app.builder.write_doc |
| 129 | + |
| 130 | + def write_doc(docname, doctree): |
| 131 | + write_html(docname, doctree) |
| 132 | + # The doctree is discarded after this, so it can be modified in place |
| 133 | + _remove_breadcrumbs(doctree) |
| 134 | + _link_to_markdown(doctree, docname, app.env.found_docs) |
| 135 | + _set_code_languages(doctree) |
| 136 | + markdown_builder.write_doc(docname, doctree) |
| 137 | + |
| 138 | + app.builder.write_doc = write_doc |
| 139 | + |
| 140 | + |
| 141 | +def setup(app): |
| 142 | + app.setup_extension('sphinx_markdown_builder') |
| 143 | + # The translator does not handle meta nodes and would warn for every |
| 144 | + # page. The description is already in the HTML, so drop it. |
| 145 | + app.add_node(nodes.meta, override=True, markdown=(_skip_node, None)) |
| 146 | + app.add_node(nodes.literal, override=True, markdown=(_visit_literal, None)) |
| 147 | + app.add_node( |
| 148 | + nodes.problematic, override=True, markdown=(_visit_problematic, None) |
| 149 | + ) |
| 150 | + app.connect('builder-inited', _install) |
| 151 | + return { |
| 152 | + 'version': '1.0', |
| 153 | + 'parallel_read_safe': True, |
| 154 | + 'parallel_write_safe': True, |
| 155 | + } |
0 commit comments