From 60ed400a47e111d0e64637070e309ced2ef4baab Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 16:29:34 -0500 Subject: [PATCH 01/11] qxqx --- docs/_scripts/notebook_convert.py | 26 ++++++++++++------- .../mdoutput/index.md.j2 | 19 ++++++++++++++ docs/mkdocs.yml | 1 + 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index 8a47c44cc1..3128b7248f 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -10,12 +10,15 @@ class EscapePreprocessor(Preprocessor): def preprocess_cell(self, cell, resources, cell_index): if cell.cell_type == "markdown": - # rewrite markdown links to html links (excluding image links) - cell.source = re.sub( - r"(?\1', - cell.source, - ) + # Temporarily disable this. + # We'll re-enable, but keeping markdown format and replacing ipynb links + # with links to corresponding markdown files. + # # rewrite markdown links to html links (excluding image links) + # cell.source = re.sub( + # r"(?\1', + # cell.source, + # ) # Fix image paths in tags cell.source = re.sub( r' Date: Mon, 3 Feb 2025 16:29:37 -0500 Subject: [PATCH 02/11] qxqx --- docs/tests/test_notebook_convert.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/tests/test_notebook_convert.py diff --git a/docs/tests/test_notebook_convert.py b/docs/tests/test_notebook_convert.py new file mode 100644 index 0000000000..67fc66c402 --- /dev/null +++ b/docs/tests/test_notebook_convert.py @@ -0,0 +1,39 @@ +import nbformat + +from _scripts.notebook_convert import exporter + + +def _remove_consecutive_new_lines(s) -> str: + """Remove consecutive new lines from a string.""" + return "\n".join([line for line in s.split("\n") if line.strip()]) + + +def test_convert_notebook(): + # Test the convert_notebook function + # Create a new, minimal notebook programmatically + nb = nbformat.v4.new_notebook() + nb.metadata.kernelspec = { + "name": "python3", + "language": "python", + "display_name": "Python 3", + } + nb.metadata.language_info = { + "name": "python", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3, + }, + } + + # Add a markdown cell with a link to an .ipynb file + md_cell_source = "This is a [link](example_notebook.ipynb) in markdown." + nb.cells.append(nbformat.v4.new_markdown_cell(md_cell_source)) + + # Add a code cell with a noqa comment + code_cell_source = "print('hello') # noqa: F401" + nb.cells.append(nbformat.v4.new_code_cell(code_cell_source)) + nb.metadata.mode = "exec" + + body, _ = exporter.from_notebook_node(nb) + assert body == """""" From c43474f29b4c0bd11e3a077b5c40d20412c04624 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 16:29:40 -0500 Subject: [PATCH 03/11] x --- docs/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/tests/__init__.py diff --git a/docs/tests/__init__.py b/docs/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 6ae3052b4b7df28e97d6b0197f801fe203c40d2a Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 16:36:47 -0500 Subject: [PATCH 04/11] qx --- docs/tests/test_notebook_convert.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/tests/test_notebook_convert.py b/docs/tests/test_notebook_convert.py index 67fc66c402..cbfbfb6e9b 100644 --- a/docs/tests/test_notebook_convert.py +++ b/docs/tests/test_notebook_convert.py @@ -31,9 +31,16 @@ def test_convert_notebook(): nb.cells.append(nbformat.v4.new_markdown_cell(md_cell_source)) # Add a code cell with a noqa comment - code_cell_source = "print('hello') # noqa: F401" + code_cell_source = "print('hello')" nb.cells.append(nbformat.v4.new_code_cell(code_cell_source)) nb.metadata.mode = "exec" body, _ = exporter.from_notebook_node(nb) - assert body == """""" + assert ( + _remove_consecutive_new_lines(body) + == """\ +This is a [link](example_notebook.ipynb) in markdown. +```python exec="1" source="below" result="ini" +print('hello') +```""" + ) From 9c5c0b1973b8e0202857830fe6641675a9e380f9 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:11:36 -0500 Subject: [PATCH 05/11] qxqx --- docs/_scripts/notebook_convert.py | 76 +++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index 3128b7248f..3fc01cc000 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -1,6 +1,8 @@ +import argparse import os import re from pathlib import Path +from typing import Literal import nbformat from nbconvert.exporters import MarkdownExporter @@ -8,17 +10,28 @@ class EscapePreprocessor(Preprocessor): + def __init__(self, rewrite_links: bool = True, **kwargs) -> None: + super().__init__(**kwargs) + self.rewrite_links = rewrite_links + def preprocess_cell(self, cell, resources, cell_index): if cell.cell_type == "markdown": - # Temporarily disable this. - # We'll re-enable, but keeping markdown format and replacing ipynb links - # with links to corresponding markdown files. - # # rewrite markdown links to html links (excluding image links) - # cell.source = re.sub( - # r"(?\1', - # cell.source, - # ) + if self.rewrite_links: + # We'll need to adjust the logic for this to keep markdown format + # but link to markdown files rather than ipynb files. + cell.source = re.sub( + r"(?\1', + cell.source, + ) + else: + # Keep format but replace the .ipynb extension with .md + cell.source = re.sub( + r"(? tags cell.source = re.sub( r' Path: + mode: Literal["markdown", "exec"] = "markdown", +) -> str: with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) - nb.metadata.target = "exec" - - body, _ = exporter.from_notebook_node(nb) + nb.metadata.mode = mode + if mode == "markdown": + body, _ = exporter.from_notebook_node(nb) + else: + body, _ = md_executable.from_notebook_node(nb) return body + + +HERE = Path(__file__).parent +DOCS = HERE.parent / "docs" + + +# Convert notebooks to markdown +def _convert_notebooks(output_dir: str) -> None: + """Converting notebooks.""" + output_dir_path = Path(output_dir) + for notebook in DOCS.rglob("*.ipynb"): + markdown = convert_notebook(notebook, mode="exec") + markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md") + markdown_path.parent.mkdir(parents=True, exist_ok=True) + with open(markdown_path, "w") as f: + f.write(markdown) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert notebooks to markdown") + parser.add_argument("output_dir", help="Directory to output markdown files") + args = parser.parse_args() + _convert_notebooks(args.output_dir) From 5e8844a2faad674002e2c4d4dbf514d7936cfaed Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:11:45 -0500 Subject: [PATCH 06/11] x --- .../md_executable/conf.json | 5 +++ .../md_executable/index.md.j2 | 38 +++++++++++++++++++ .../mdoutput/index.md.j2 | 1 - 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docs/_scripts/notebook_convert_templates/md_executable/conf.json create mode 100644 docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 diff --git a/docs/_scripts/notebook_convert_templates/md_executable/conf.json b/docs/_scripts/notebook_convert_templates/md_executable/conf.json new file mode 100644 index 0000000000..7adab7c92a --- /dev/null +++ b/docs/_scripts/notebook_convert_templates/md_executable/conf.json @@ -0,0 +1,5 @@ +{ + "mimetypes": { + "text/markdown": true + } +} \ No newline at end of file diff --git a/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 new file mode 100644 index 0000000000..7ef8d6e692 --- /dev/null +++ b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 @@ -0,0 +1,38 @@ +{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#} +{% extends 'markdown/index.md.j2' %} + +{% block input %} + +``` +{%- if 'magics_language' in cell.metadata -%} + {{ cell.metadata.magics_language}} +{%- elif 'name' in nb.metadata.get('language_info', {}) -%} + {{ nb.metadata.language_info.name }} exec="1" source="below" result="ini" +{%- endif %} +{{ cell.source}} +``` + +{% endblock input %} + +{%- block traceback_line -%} +{%- endblock traceback_line -%} + +{%- block stream -%} +{%- endblock stream -%} + +{%- block data_text scoped -%} +{%- endblock data_text -%} + +{%- block data_html scoped -%} +```html +{{ output.data['text/html'] | safe }} +``` +{%- endblock data_html -%} + +{%- block data_jpg scoped -%} +![](data:image/jpg;base64,{{ output.data['image/jpeg'] }}) +{%- endblock data_jpg -%} + +{%- block data_png scoped -%} +![](data:image/png;base64,{{ output.data['image/png'] }}) +{%- endblock data_png -%} diff --git a/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 b/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 index 449299b60e..9ba93ab308 100644 --- a/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 +++ b/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 @@ -3,7 +3,6 @@ {% block input %} -{{ nb.metadata.target }} ``` {%- if 'magics_language' in cell.metadata -%} {{ cell.metadata.magics_language}} From 060ebc8da0b135895d1e74973c9c7f0ea6464963 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:12:11 -0500 Subject: [PATCH 07/11] qxqx --- .../mdoutput/index.md.j2 | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 b/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 index 9ba93ab308..00e1e03057 100644 --- a/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 +++ b/docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2 @@ -1,23 +1,5 @@ -{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#} {% extends 'markdown/index.md.j2' %} -{% block input %} - -``` -{%- if 'magics_language' in cell.metadata -%} - {{ cell.metadata.magics_language}} -{%- elif 'name' in nb.metadata.get('language_info', {}) -%} - {%- if nb.metadata.get('mode') == 'exec' -%} - {{ nb.metadata.language_info.name }} exec="1" source="below" result="ini" - {%- else -%} - {{ nb.metadata.language_info.name }} - {%- endif %} -{%- endif %} -{{ cell.source}} -``` - -{% endblock input %} - {%- block traceback_line -%} ```output {{ line.rstrip() | strip_ansi }} From 9b7870b39e6c4b6513c0c0c20d13b829e978b804 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:18:25 -0500 Subject: [PATCH 08/11] qxqx --- docs/_scripts/notebook_convert.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index 3fc01cc000..d61d4e4819 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -2,7 +2,7 @@ import os import re from pathlib import Path -from typing import Literal +from typing import Literal, Optional import nbformat from nbconvert.exporters import MarkdownExporter @@ -165,19 +165,33 @@ def convert_notebook( # Convert notebooks to markdown -def _convert_notebooks(output_dir: str) -> None: +def _convert_notebooks( + *, output_dir: Optional[Path] = None, replace: bool = False +) -> None: """Converting notebooks.""" - output_dir_path = Path(output_dir) + if not output_dir and not replace: + raise ValueError("Either --output_dir or --replace must be specified") + + output_dir_path = DOCS if replace else Path(output_dir) for notebook in DOCS.rglob("*.ipynb"): markdown = convert_notebook(notebook, mode="exec") markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md") markdown_path.parent.mkdir(parents=True, exist_ok=True) with open(markdown_path, "w") as f: f.write(markdown) + if replace: + notebook.unlink(missing_ok=False) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Convert notebooks to markdown") - parser.add_argument("output_dir", help="Directory to output markdown files") + parser.add_argument( + "--output_dir", default=None, help="Directory to output markdown files" + ) + parser.add_argument( + "--replace", + action="store_true", + help="Replace original notebooks with markdown files", + ) args = parser.parse_args() - _convert_notebooks(args.output_dir) + _convert_notebooks(args.replace, args.output_dir) From 283c46c85967fd9985d9adfc611680383a1b7b78 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:19:53 -0500 Subject: [PATCH 09/11] qxqx --- docs/_scripts/notebook_convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index d61d4e4819..fd9dfb4429 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -186,7 +186,7 @@ def _convert_notebooks( if __name__ == "__main__": parser = argparse.ArgumentParser(description="Convert notebooks to markdown") parser.add_argument( - "--output_dir", default=None, help="Directory to output markdown files" + "--output_dir", default=None, help="Directory to output markdown files", ) parser.add_argument( "--replace", @@ -194,4 +194,4 @@ def _convert_notebooks( help="Replace original notebooks with markdown files", ) args = parser.parse_args() - _convert_notebooks(args.replace, args.output_dir) + _convert_notebooks(replace=args.replace, output_dir=args.output_dir) From 0dd4a72b225cdc599284057b3b0a3c1eae8f6aba Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:31:28 -0500 Subject: [PATCH 10/11] q --- docs/_scripts/notebook_convert.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/_scripts/notebook_convert.py b/docs/_scripts/notebook_convert.py index fd9dfb4429..868b0dac03 100644 --- a/docs/_scripts/notebook_convert.py +++ b/docs/_scripts/notebook_convert.py @@ -38,6 +38,11 @@ def preprocess_cell(self, cell, resources, cell_index): ) elif cell.cell_type == "code": + # Determine if the cell has bash or cell magic + if cell.source.startswith("%") or cell.source.startswith("!"): + # update metadata to denote that it's not a python cell + cell.metadata["language_info"] = {"name": "unknown"} + # Remove noqa comments cell.source = re.sub(r"#\s*noqa.*$", "", cell.source, flags=re.MULTILINE) # escape ``` in code From 1d1588c8f95113f1d002a3cb574fd0b2febc0c16 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 3 Feb 2025 17:35:47 -0500 Subject: [PATCH 11/11] qxq --- .../notebook_convert_templates/md_executable/index.md.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 index 7ef8d6e692..fee77ae6af 100644 --- a/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 +++ b/docs/_scripts/notebook_convert_templates/md_executable/index.md.j2 @@ -6,6 +6,10 @@ ``` {%- if 'magics_language' in cell.metadata -%} {{ cell.metadata.magics_language}} +{%- elif 'name' in cell.metadata.get('language_info', {}) -%} + {%- if cell.metadata['language_info']['name'] == "python" -%} + {{ cell.metadata.language_info.name }} exec="1" source="below" result="ini" + {%- endif -%} {%- elif 'name' in nb.metadata.get('language_info', {}) -%} {{ nb.metadata.language_info.name }} exec="1" source="below" result="ini" {%- endif %}