From f8a77cc12dbc1f6843ddcdc28479fe2387f7e3eb Mon Sep 17 00:00:00 2001 From: Albert Steppi Date: Tue, 7 Nov 2023 12:23:00 -0500 Subject: [PATCH] Add try_examples directive for adding interactivity to sphinx Examples sections (#111) * Add try_examples directive * Fix associated css for try_examples extension * Make global_enable_try_examples work * Make LaTeX processing work correctly * Fix rendering of Examples * Add support for multiline blocks in Examples * Make toggle examples <-> notebook work * Allow configuration of button css * Fix cases where no directive is inserted * Final tweaks * Get correct relative path to doc root * Allow leading whitespace in latex expression :math: * Handle edgecase, Examples is last section * Strip out content which should be ignored in notebook * Handle :Attributes: edge case for section header * Allow whitespace in processed by numpydoc part * Handle edgecase for processed by numpydoc * Handle case with multiple output lines * Fix incorrectly formatted arrays in output * Handle references in examples section -- replace href id with reference number * Reword some comments * Fix bugs in latex processing * Add global to global configuration var names * Add Python language to notebook metadata * Format code with black * Update jupyterlite_sphinx/jupyterlite_sphinx.js Co-authored-by: martinRenou --------- Co-authored-by: martinRenou --- jupyterlite_sphinx/_try_examples.py | 323 ++++++++++++++++++++++ jupyterlite_sphinx/jupyterlite_sphinx.css | 15 + jupyterlite_sphinx/jupyterlite_sphinx.js | 34 +++ jupyterlite_sphinx/jupyterlite_sphinx.py | 158 +++++++++++ pyproject.toml | 1 + 5 files changed, 531 insertions(+) create mode 100755 jupyterlite_sphinx/_try_examples.py diff --git a/jupyterlite_sphinx/_try_examples.py b/jupyterlite_sphinx/_try_examples.py new file mode 100755 index 0000000..64906c0 --- /dev/null +++ b/jupyterlite_sphinx/_try_examples.py @@ -0,0 +1,323 @@ +import nbformat as nbf +from nbformat.v4 import new_code_cell, new_markdown_cell +import re + + +def examples_to_notebook(input_lines): + """Parse examples section of a docstring and convert to Jupyter notebook. + + Parameters + ---------- + input_lines : iterable of str. + Lines within + + Returns + ------- + dict + json for a Jupyter Notebook + + Examples + -------- + >>> from jupyterlite_sphinx.generate_notebook import examples_to_notebook + + >>> input_lines = [ + >>> "Add two numbers. This block of text will appear as a\n", + >>> "markdown cell. The following block will become a code\n", + >>> "cell with the value 4 contained in the output.", + >>> "\n", + >>> ">>> x = 2\n", + >>> ">>> y = 2\n", + >>> ">>> x + y\n", + >>> "4\n", + >>> "\n", + >>> "Inline LaTeX like :math:`x + y = 4` will be converted\n", + >>> "correctly within markdown cells. As will block LaTeX\n", + >>> "such as\n", + >>> "\n", + >>> ".. math::\n", + >>> "\n", + >>> " x = 2,\;y = 2 + >>> "\n", + >>> " x + y = 4\n", + >>> ] + >>> notebook = examples_to_notebook(input_lines) + """ + nb = nbf.v4.new_notebook() + + code_lines = [] + md_lines = [] + output_lines = [] + inside_multiline_code_block = False + + ignore_directives = [".. plot::", ".. only::"] + inside_ignore_directive = False + + for line in input_lines: + line = line.rstrip("\n") + + # Content underneath some directives should be ignored when generating notebook. + if any(line.startswith(directive) for directive in ignore_directives): + inside_ignore_directive = True + continue + if inside_ignore_directive: + if line == "" or line[0].isspace(): + continue + else: + inside_ignore_directive = False + + if line.startswith(">>>"): # This is a code line. + if output_lines: + # If there are pending output lines, we must be starting a new + # code block. + _append_code_cell_and_clear_lines(code_lines, output_lines, nb) + if inside_multiline_code_block: + # A multiline codeblock is ending. + inside_multiline_code_block = False + # If there is any pending markdown text, add it to the notebook + if md_lines: + _append_markdown_cell_and_clear_lines(md_lines, nb) + + # Add line of code, removing '>>> ' prefix + code_lines.append(line[4:]) + elif line.startswith("...") and code_lines: + # This is a line of code in a multiline code block. + inside_multiline_code_block = True + code_lines.append(line[4:]) + elif line.rstrip("\n") == "" and code_lines: + # A blank line means a code block has ended. + _append_code_cell_and_clear_lines(code_lines, output_lines, nb) + elif code_lines: + # Non-blank non ">>>" prefixed line must be output of previous code block. + output_lines.append(line) + else: + # Anything else should be treated as markdown. + md_lines.append(line) + + # After processing all lines, add pending markdown or code to the notebook if + # any exists. + if md_lines: + _append_markdown_cell_and_clear_lines(md_lines, nb) + if code_lines: + _append_code_cell_and_clear_lines(code_lines, output_lines, nb) + + nb["metadata"] = { + "kernelspec": { + "display_name": "Python", + "language": "python", + "name": "python", + }, + "language_info": { + "name": "python", + }, + } + return nb + + +def _append_code_cell_and_clear_lines(code_lines, output_lines, notebook): + """Append new code cell to notebook, clearing lines.""" + code_text = "\n".join(code_lines) + cell = new_code_cell(code_text) + if output_lines: + combined_output = "\n".join(output_lines) + cell.outputs.append( + nbf.v4.new_output( + output_type="execute_result", + data={"text/plain": combined_output}, + ), + ) + notebook.cells.append(cell) + output_lines.clear() + code_lines.clear() + + +def _append_markdown_cell_and_clear_lines(markdown_lines, notebook): + """Append new markdown cell to notebook, clearing lines.""" + markdown_text = "\n".join(markdown_lines) + # Convert blocks of LaTeX equations + markdown_text = _process_latex(markdown_text) + markdown_text = _strip_ref_identifiers(markdown_text) + notebook.cells.append(new_markdown_cell(markdown_text)) + markdown_lines.clear() + + +_ref_identifier_pattern = re.compile(r"\[R[a-f0-9]+-(?P\d+)\]_") + + +def _strip_ref_identifiers(md_text): + """Remove identifiers from references in notebook. + + Each docstring gets a unique identifier in order to have unique internal + links for each docstring on a page. + + They look like [R4c2dbc17006a-1]_. We strip these out so they don't appear + in the notebooks. The above would be replaced with [1]_. + """ + return _ref_identifier_pattern.sub(r"[\g]", md_text) + + +def _process_latex(md_text): + # Map rst latex directive to $ so latex renders in notebook. + md_text = re.sub( + r":math:\s*`(?P.*?)`", r"$\g$", md_text, flags=re.DOTALL + ) + + lines = md_text.split("\n") + in_math_block = False + wrapped_lines = [] + equation_lines = [] + + for line in lines: + if line.strip() == ".. math::": + in_math_block = True + continue # Skip the '.. math::' line + + if in_math_block: + if line.strip() == "": + if equation_lines: + # Join and wrap the equations, then reset + wrapped_lines.append(f"$$ {' '.join(equation_lines)} $$") + equation_lines = [] + elif line.startswith(" ") or line.startswith("\t"): + equation_lines.append(line.strip()) + else: + wrapped_lines.append(line) + + # If you leave the indented block, the math block ends + if in_math_block and not ( + line.startswith(" ") or line.startswith("\t") or line.strip() == "" + ): + in_math_block = False + if equation_lines: + wrapped_lines.append(f"$$ {' '.join(equation_lines)} $$") + equation_lines = [] + wrapped_lines.append(line) + + return "\n".join(wrapped_lines) + + +# https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html#docstring-sections +_non_example_docstring_section_headers = ( + "Args", + "Arguments", + "Attention", + "Attributes", + "Caution", + "Danger", + "Error", + "Hint", + "Important", + "Keyword Args", + "Keyword Arguments", + "Methods", + "Note", + "Notes", + "Other Parameters", + "Parameters", + "Return", + "Returns", + "Raise", + "Raises", + "References", + "See Also", + "Tip", + "Todo", + "Warning", + "Warnings", + "Warns", + "Yield", + "Yields", +) + + +_examples_start_pattern = re.compile(r".. (rubric|admonition):: Examples") +_next_section_pattern = re.compile( + "|".join( + [ + rf".. (rubric|admonition)::\s*{header}" + for header in _non_example_docstring_section_headers + ] + # If examples section is last, processed by numpydoc may appear at end. + + [r"\!\! processed by numpydoc \!\!"] + # Attributes section sometimes has no directive. + + [r":Attributes:"] + ) +) + + +def insert_try_examples_directive(lines, **options): + """Adds try_examples directive to Examples section of a docstring. + + Hack to allow for a config option to enable try_examples functionality + in all Examples sections (unless a comment "..! disable_try_examples" is + added explicitly after the section header.) + + + Parameters + ---------- + docstring : list of str + Lines of a docstring at time of "autodoc-process-docstring", with section + headers denoted by `.. rubric::` or `.. admonition::`. + + + Returns + ------- + list of str + Updated version of the input docstring which has a try_examples directive + inserted in the Examples section (if one exists) with all Examples content + indented beneath it. Does nothing if the comment "..! disable_try_examples" + is included at the top of the Examples section. Also a no-op if the + try_examples directive is already included. + """ + # Search for start of an Examples section + for left_index, line in enumerate(lines): + if _examples_start_pattern.search(line): + break + else: + # No Examples section found + return lines[:] + + # Jump to next line + left_index += 1 + # Skip empty lines to get to the first content line + while left_index < len(lines) and not lines[left_index].strip(): + left_index += 1 + if left_index == len(lines): + # Examples section had no content, no need to insert directive. + return lines[:] + + # Check for the "..! disable_try_examples" comment. + if lines[left_index].strip() == "..! disable_try_examples::": + # If so, do not insert directive. + return lines[:] + + # Check if the ".. try_examples::" directive already exists + if ".. try_examples::" == lines[left_index].strip(): + # If so, don't need to insert again. + return lines[:] + + # Find the end of the Examples section + right_index = left_index + while right_index < len(lines) and not _next_section_pattern.search( + lines[right_index] + ): + right_index += 1 + if "!! processed by numpydoc !!" in lines[right_index]: + # Sometimes the .. appears on an earlier line than !! processed by numpydoc !! + if not re.search( + r"\.\.\s+\!\! processed by numpy doc \!\!", lines[right_index] + ): + while lines[right_index].strip() != "..": + right_index -= 1 + + # Add the ".. try_examples::" directive and indent the content of the Examples section + new_lines = ( + lines[:left_index] + + [".. try_examples::"] + + [f" :{key}: {value}" for key, value in options.items()] + + [""] + + [" " + line for line in lines[left_index:right_index]] + + [""] + + lines[right_index:] + ) + + return new_lines diff --git a/jupyterlite_sphinx/jupyterlite_sphinx.css b/jupyterlite_sphinx/jupyterlite_sphinx.css index 6081f8b..478129a 100644 --- a/jupyterlite_sphinx/jupyterlite_sphinx.css +++ b/jupyterlite_sphinx/jupyterlite_sphinx.css @@ -67,3 +67,18 @@ transform: translateY(-50%) translateX(-50%) scale(1.2); } } + +.try_examples_iframe_container { + position: relative; + cursor: pointer; +} + + +.try_examples_outer_container { + position: relative; +} + + +.hidden { + display: none; +} diff --git a/jupyterlite_sphinx/jupyterlite_sphinx.js b/jupyterlite_sphinx/jupyterlite_sphinx.js index 419e71b..96c2862 100644 --- a/jupyterlite_sphinx/jupyterlite_sphinx.js +++ b/jupyterlite_sphinx/jupyterlite_sphinx.js @@ -39,3 +39,37 @@ window.jupyterliteConcatSearchParams = (iframeSrc, params) => { return iframeSrc; } } + + +window.tryExamplesShowIframe = ( + examplesContainerId, iframeContainerId, iframeParentContainerId, iframeSrc +) => { + const examplesContainer = document.getElementById(examplesContainerId); + const iframeParentContainer = document.getElementById(iframeParentContainerId); + const iframeContainer = document.getElementById(iframeContainerId); + + let iframe = iframeContainer.querySelector('iframe.jupyterlite_sphinx_raw_iframe'); + + if (!iframe) { + const examples = examplesContainer.querySelector('.try_examples_content'); + iframe = document.createElement('iframe'); + iframe.src = iframeSrc; + iframe.style.width = '100%'; + iframe.style.height = `${examples.offsetHeight}px`; + iframe.classList.add('jupyterlite_sphinx_raw_iframe'); + examplesContainer.classList.add("hidden"); + iframeContainer.appendChild(iframe); + } else { + examplesContainer.classList.add("hidden"); + } + iframeParentContainer.classList.remove("hidden"); +} + + +window.tryExamplesHideIframe = (examplesContainerId, iframeParentContainerId) => { + const examplesContainer = document.getElementById(examplesContainerId); + const iframeParentContainer = document.getElementById(iframeParentContainerId); + + iframeParentContainer.classList.add("hidden"); + examplesContainer.classList.remove("hidden"); +} diff --git a/jupyterlite_sphinx/jupyterlite_sphinx.py b/jupyterlite_sphinx/jupyterlite_sphinx.py index 24ee569..92f8bfa 100644 --- a/jupyterlite_sphinx/jupyterlite_sphinx.py +++ b/jupyterlite_sphinx/jupyterlite_sphinx.py @@ -1,10 +1,12 @@ import os +import json from uuid import uuid4 import shutil import tempfile from warnings import warn import glob import re +import nbformat as nbf from pathlib import Path @@ -14,12 +16,16 @@ from docutils.parsers.rst import directives from docutils.nodes import SkipNode, Element +from docutils import nodes from sphinx.application import Sphinx +from sphinx.ext.doctest import DoctestDirective from sphinx.util.docutils import SphinxDirective from sphinx.util.fileutil import copy_asset from sphinx.parsers import RSTParser +from ._try_examples import examples_to_notebook, insert_try_examples_directive + try: import voici except ImportError: @@ -356,6 +362,148 @@ def parse(self, inputstring, document): ) +class TryExamplesDirective(SphinxDirective): + """Add button to try doctest examples in Jupyterlite notebook.""" + + has_content = True + required_arguments = 0 + option_spec = { + "toolbar": directives.unchanged, + "theme": directives.unchanged, + } + + def run(self): + if "generated_notebooks" not in self.env.temp_data: + self.env.temp_data["generated_notebooks"] = {} + + directive_key = f"{self.env.docname}-{self.lineno}" + notebook_unique_name = self.env.temp_data["generated_notebooks"].get( + directive_key + ) + + # We need to get the relative path back to the documentation root from + # whichever file the docstring content is in. + docname = self.env.docname + depth = len(docname.split("/")) - 1 + relative_path_to_root = "/".join([".."] * depth) + prefix = os.path.join(relative_path_to_root, JUPYTERLITE_DIR) + + lite_app = "retro/" + notebooks_path = "notebooks/" + + content_container_node = nodes.container( + classes=["try_examples_outer_container"] + ) + examples_div_id = uuid4() + content_container_node["ids"].append(examples_div_id) + # Parse the original content to create nodes + content_node = nodes.container() + content_node["classes"].append("try_examples_content") + self.state.nested_parse(self.content, self.content_offset, content_node) + content_container_node += content_node + + if notebook_unique_name is None: + nb = examples_to_notebook(self.content) + self.content = None + notebooks_dir = Path(self.env.app.srcdir) / CONTENT_DIR + notebook_unique_name = f"{uuid4()}.ipynb".replace("-", "_") + self.env.temp_data["generated_notebooks"][ + directive_key + ] = notebook_unique_name + # Copy the Notebook for RetroLite to find + os.makedirs(notebooks_dir, exist_ok=True) + with open(notebooks_dir / Path(notebook_unique_name), "w") as f: + # nbf.write incorrectly formats multiline arrays in output. + json.dump(nb, f, indent=4, ensure_ascii=False) + + self.options["path"] = notebook_unique_name + app_path = f"{lite_app}{notebooks_path}" + options = "&".join( + [f"{key}={quote(value)}" for key, value in self.options.items()] + ) + + iframe_parent_div_id = uuid4() + iframe_div_id = uuid4() + iframe_src = f'{prefix}/{app_path}{f"?{options}" if options else ""}' + + # Parent container (initially hidden) + iframe_parent_container_div_start = ( + f'
' + ) + + iframe_parent_container_div_end = "
" + iframe_container_div = ( + f'
' + f"
" + ) + + # Button with the onclick event to swap embedded notebook back to examples. + go_back_button_html = ( + '" + ) + + # Combine everything + notebook_container_html = ( + iframe_parent_container_div_start + + iframe_container_div + + go_back_button_html + + iframe_parent_container_div_end + ) + notebook_container = nodes.raw("", notebook_container_html, format="html") + + # Button with the onclick event to swap examples with embedded notebook. + try_it_button_html = ( + '" + ) + try_it_button_node = nodes.raw("", try_it_button_html, format="html") + # Add the button to the content_container_node + content_container_node += try_it_button_node + + # Allow css for button to be specified in conf.py + config = self.state.document.settings.env.config + try_examples_button_css = config.try_examples_global_button_css + + try_examples_button_css = f".try_examples_button {{{try_examples_button_css}}}" + style_tag = nodes.raw( + "", f"", format="html" + ) + + return [content_container_node, notebook_container, style_tag] + + +def _process_docstring_examples(app, docname, source): + source_path = app.env.doc2path(docname) + if source_path.endswith(".py"): + source[0] = insert_try_examples_directive(source[0]) + + +def _process_autodoc_docstrings(app, what, name, obj, options, lines): + try_examples_options = { + "toolbar": app.config.try_examples_global_toolbar, + "theme": app.config.try_examples_global_theme, + } + try_examples_options = { + key: value for key, value in try_examples_options.items() if value is not None + } + modified_lines = insert_try_examples_directive(lines, **try_examples_options) + lines.clear() + lines.extend(modified_lines) + + +def conditional_process_examples(app, config): + if config.global_enable_try_examples: + app.connect("source-read", _process_docstring_examples) + app.connect("autodoc-process-docstring", _process_autodoc_docstrings) + + def inited(app: Sphinx, config): # Create the content dir os.makedirs(os.path.join(app.srcdir, CONTENT_DIR), exist_ok=True) @@ -448,6 +596,12 @@ def setup(app): app.add_config_value("jupyterlite_dir", app.srcdir, rebuild="html") app.add_config_value("jupyterlite_contents", None, rebuild="html") app.add_config_value("jupyterlite_bind_ipynb_suffix", True, rebuild="html") + app.add_config_value("global_enable_try_examples", default=False, rebuild=True) + app.add_config_value( + "try_examples_global_button_css", default="float: right;", rebuild="html" + ) + app.add_config_value("try_examples_global_toolbar", default=None, rebuild=True) + app.add_config_value("try_examples_global_theme", default=None, rebuild=True) # Initialize RetroLite and JupyterLite directives app.add_node( @@ -491,6 +645,10 @@ def setup(app): ) app.add_directive("voici", VoiciDirective) + # Initialize TryExamples directive + app.add_directive("try_examples", TryExamplesDirective) + app.connect("config-inited", conditional_process_examples) + # CSS and JS assets copy_asset(str(HERE / "jupyterlite_sphinx.css"), str(Path(app.outdir) / "_static")) copy_asset(str(HERE / "jupyterlite_sphinx.js"), str(Path(app.outdir) / "_static")) diff --git a/pyproject.toml b/pyproject.toml index 449f670..9ed9bf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "jupyter_server", "jupyterlab_server", "jupyterlite-core >=0.1.0", + "nbformat", "sphinx>=4", ]