diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index 8a47c44cc1..868b0dac03 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -1,6 +1,8 @@ +import argparse import os import re from pathlib import Path +from typing import Literal, Optional import nbformat from nbconvert.exporters import MarkdownExporter @@ -8,23 +10,45 @@ class EscapePreprocessor(Preprocessor): + def __init__(self, rewrite_links: bool = True, **kwargs) -> None: + super().__init__(**kwargs) + self.rewrite_links = rewrite_links + def preprocess_cell(self, cell, resources, cell_index): if cell.cell_type == "markdown": - # rewrite markdown links to html links (excluding image links) - cell.source = re.sub( - r"(?\1', - cell.source, - ) + if self.rewrite_links: + # We'll need to adjust the logic for this to keep markdown format + # but link to markdown files rather than ipynb files. + cell.source = re.sub( + r"(?\1', + cell.source, + ) + else: + # Keep format but replace the .ipynb extension with .md + cell.source = re.sub( + r"(? tags cell.source = re.sub( r' Path: + mode: Literal["markdown", "exec"] = "markdown", +) -> str: with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) - body, _ = exporter.from_notebook_node(nb) + nb.metadata.mode = mode + if mode == "markdown": + body, _ = exporter.from_notebook_node(nb) + else: + body, _ = md_executable.from_notebook_node(nb) return body + + +HERE = Path(__file__).parent +DOCS = HERE.parent / "docs" + + +# Convert notebooks to markdown +def _convert_notebooks( + *, output_dir: Optional[Path] = None, replace: bool = False +) -> None: + """Converting notebooks.""" + if not output_dir and not replace: + raise ValueError("Either --output_dir or --replace must be specified") + + output_dir_path = DOCS if replace else Path(output_dir) + for notebook in DOCS.rglob("*.ipynb"): + markdown = convert_notebook(notebook, mode="exec") + markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md") + markdown_path.parent.mkdir(parents=True, exist_ok=True) + with open(markdown_path, "w") as f: + f.write(markdown) + if replace: + notebook.unlink(missing_ok=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert notebooks to markdown") + parser.add_argument( + "--output_dir", default=None, help="Directory to output markdown files", + ) + parser.add_argument( + "--replace", + action="store_true", + help="Replace original notebooks with markdown files", + ) + args = parser.parse_args() + _convert_notebooks(replace=args.replace, output_dir=args.output_dir) diff --git a/docs/_scripts/notebook_convert_templates/md_executable/conf.json b/docs/_scripts/notebook_convert_templates/md_executable/conf.json new file mode 100644 index 0000000000..7adab7c92a --- /dev/null +++ b/docs/_scripts/notebook_convert_templates/md_executable/conf.json @@ -0,0 +1,5 @@ +{ + "mimetypes": { + "text/markdown": true + } +} \ No newline at end of file diff --git a/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 new file mode 100644 index 0000000000..fee77ae6af --- /dev/null +++ b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 @@ -0,0 +1,42 @@ +{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#} +{% extends 'markdown/index.md.j2' %} + +{% block input %} + +``` +{%- if 'magics_language' in cell.metadata -%} + {{ cell.metadata.magics_language}} +{%- elif 'name' in cell.metadata.get('language_info', {}) -%} + {%- if cell.metadata['language_info']['name'] == "python" -%} + {{ cell.metadata.language_info.name }} exec="1" source="below" result="ini" + {%- endif -%} +{%- elif 'name' in nb.metadata.get('language_info', {}) -%} + {{ nb.metadata.language_info.name }} exec="1" source="below" result="ini" +{%- endif %} +{{ cell.source}} +``` + +{% endblock input %} + +{%- block traceback_line -%} +{%- endblock traceback_line -%} + +{%- block stream -%} +{%- endblock stream -%} + +{%- block data_text scoped -%} +{%- endblock data_text -%} + +{%- block data_html scoped -%} +```html +{{ output.data['text/html'] | safe }} +``` +{%- endblock data_html -%} + +{%- block data_jpg scoped -%} +![](data:image/jpg;base64,{{ output.data['image/jpeg'] }}) +{%- endblock data_jpg -%} + +{%- block data_png scoped -%} +![](data:image/png;base64,{{ output.data['image/png'] }}) +{%- endblock data_png -%} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index be663e709f..1cd8e22342 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -56,6 +56,7 @@ plugins: - search: separator: '[\s\u200b\-_,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])' - autorefs + - markdown-exec - mkdocstrings: handlers: python: diff --git a/docs/tests/__init__.py b/docs/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/tests/test_notebook_convert.py b/docs/tests/test_notebook_convert.py new file mode 100644 index 0000000000..cbfbfb6e9b --- /dev/null +++ b/docs/tests/test_notebook_convert.py @@ -0,0 +1,46 @@ +import nbformat + +from _scripts.notebook_convert import exporter + + +def _remove_consecutive_new_lines(s) -> str: + """Remove consecutive new lines from a string.""" + return "\n".join([line for line in s.split("\n") if line.strip()]) + + +def test_convert_notebook(): + # Test the convert_notebook function + # Create a new, minimal notebook programmatically + nb = nbformat.v4.new_notebook() + nb.metadata.kernelspec = { + "name": "python3", + "language": "python", + "display_name": "Python 3", + } + nb.metadata.language_info = { + "name": "python", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3, + }, + } + + # Add a markdown cell with a link to an .ipynb file + md_cell_source = "This is a [link](example_notebook.ipynb) in markdown." + nb.cells.append(nbformat.v4.new_markdown_cell(md_cell_source)) + + # Add a code cell with a noqa comment + code_cell_source = "print('hello')" + nb.cells.append(nbformat.v4.new_code_cell(code_cell_source)) + nb.metadata.mode = "exec" + + body, _ = exporter.from_notebook_node(nb) + assert ( + _remove_consecutive_new_lines(body) + == """\ +This is a [link](example_notebook.ipynb) in markdown. +```python exec="1" source="below" result="ini" +print('hello') +```""" + )