From 709bba1c42f43cedade660c7de0fdd59187f22ec Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 13 Apr 2016 09:18:46 +0000 Subject: [PATCH] Support extracting inline example documentation. Fixes #1 RELNOTES: Support inline example documentation. -- MOS_MIGRATED_REVID=119726298 --- skydoc/common.py | 120 ++++++++++++++++++++++++-------- skydoc/common_test.py | 107 ++++++++++++++++++++++------ skydoc/macro_extractor.py | 5 +- skydoc/rule.py | 3 +- skydoc/rule_extractor.py | 10 +-- skydoc/stubs/skylark_globals.py | 8 ++- skydoc/templates/html.jinja | 7 +- skydoc/templates/markdown.jinja | 10 ++- skylark/skylark.bzl | 97 ++++++++++++++++++++++++++ 9 files changed, 307 insertions(+), 60 deletions(-) diff --git a/skydoc/common.py b/skydoc/common.py index 8707479..517648d 100644 --- a/skydoc/common.py +++ b/skydoc/common.py @@ -15,52 +15,116 @@ """Common functions for skydoc.""" import re +import textwrap from xml.sax.saxutils import escape +ARGS_HEADING = "Args:" +EXAMPLES_HEADING = "Examples:" +EXAMPLE_HEADING = "Example:" + + def leading_whitespace(line): + """Returns the number of leading whitespace in the line.""" return len(line) - len(line.lstrip()) -def parse_attribute_doc(doc): - """Analyzes the documentation string for attributes. - This looks for the "Args:" separator to fetch documentation for each - attribute. The "Args" section ends at the first blank line. +def _parse_attribute_docs(attr_doc, lines, index): + """Extracts attribute documentation. Args: - doc: The documentation string + attr_doc: A dict used to store the extracted attribute documentation. + lines: List containing the input docstring split into lines. + index: The index in lines containing "Args:", which begins the argument + documentation. Returns: - The new documentation string and a dictionary that maps each attribute to - its documentation + Returns the next index after the attribute documentation to resume + processing documentation in the caller. """ - doc_attr = {} - lines = doc.split("\n") - if "Args:" not in lines: - return doc, doc_attr - start = lines.index("Args:") - - i = start + 1 - var = None # Current attribute name + attr = None # Current attribute name desc = None # Description for current attribute - args_leading_ws = leading_whitespace(lines[start]) - for i in xrange(start + 1, len(lines)): + args_leading_ws = leading_whitespace(lines[index]) + i = index + 1 + while i < len(lines): + line = lines[i] # If a blank line is encountered, we have finished parsing the "Args" # section. - if lines[i].strip() and leading_whitespace(lines[i]) == args_leading_ws: + if line.strip() and leading_whitespace(line) == args_leading_ws: break # In practice, users sometimes add a "-" prefix, so we strip it even # though it is not recommended by the style guide - match = re.search(r"^\s*-?\s*(\w+):\s*(.*)", lines[i]) + match = re.search(r"^\s*-?\s*(\w+):\s*(.*)", line) if match: # We have found a new attribute - if var: - doc_attr[var] = escape(desc) - var, desc = match.group(1), match.group(2) - elif var: + if attr: + attr_doc[attr] = escape(desc) + attr, desc = match.group(1), match.group(2) + elif attr: # Merge documentation when it is multiline - desc = desc + "\n" + lines[i].strip() + desc = desc + "\n" + line.strip() + i += + 1 + + if attr: + attr_doc[attr] = escape(desc).strip() + + return i + + +def _parse_example_docs(examples, lines, index): + """Extracts example documentation. + + Args: + examples: A list to contain the lines containing the example documentation. + lines: List containing the input docstring split into lines. + index: The index in lines containing "Example[s]:", which begins the + example documentation. + + Returns: + Returns the next index after the attribute documentation to resume + processing documentation in the caller. + """ + heading_leading_ws = leading_whitespace(lines[index]) + i = index + 1 + while i < len(lines): + line = lines[i] + if line.strip() and leading_whitespace(line) == heading_leading_ws: + break + examples.append(line) + i += 1 + + return i + + +def parse_docstring(doc): + """Analyzes the documentation string for attributes. + + This looks for the "Args:" separator to fetch documentation for each + attribute. The "Args" section ends at the first blank line. + + Args: + doc: The documentation string + + Returns: + The new documentation string and a dictionary that maps each attribute to + its documentation + """ + attr_doc = {} + examples = [] + lines = doc.split("\n") + docs = [] + i = 0 + while i < len(lines): + line = lines[i] + if line.strip() == ARGS_HEADING: + i = _parse_attribute_docs(attr_doc, lines, i) + continue + elif line.strip() == EXAMPLES_HEADING or line.strip() == EXAMPLE_HEADING: + i = _parse_example_docs(examples, lines, i) + continue + + docs.append(line) + i += 1 - if var: - doc_attr[var] = escape(desc) - doc = "\n".join(lines[:start - 1]) - return doc, doc_attr + doc = "\n".join(docs).strip() + examples_doc = textwrap.dedent("\n".join(examples)).strip() + return doc, attr_doc, examples_doc diff --git a/skydoc/common_test.py b/skydoc/common_test.py index 71587c8..7c36331 100644 --- a/skydoc/common_test.py +++ b/skydoc/common_test.py @@ -21,31 +21,32 @@ class CommonTest(unittest.TestCase): """Unit tests for common functions.""" def test_rule_doc_only(self): - doc = '"""Rule documentation only docstring."""\n' - doc, attr_doc = common.parse_attribute_doc(doc) + docstring = 'Rule documentation only docstring.' + doc, attr_doc, example_doc = common.parse_docstring(docstring) self.assertEqual('Rule documentation only docstring.', doc) self.assertDictEqual({}, attr_doc) + self.assertEqual('', example_doc) def test_rule_and_attribute_doc(self): - doc = ( - '"""Rule and attribute documentation.\n' + docstring = ( + 'Rule and attribute documentation.\n' '\n' 'Args:\n' ' name: A unique name for this rule.\n' - ' visibility: The visibility of this rule.\n' - '"""\n') + ' visibility: The visibility of this rule.\n') expected_attrs = { 'name': 'A unique name for this rule.', 'visibility': 'The visibility of this rule.' } - doc, attr_doc = common.parse_attribute_doc(doc) + doc, attr_doc, example_doc = common.parse_docstring(docstring) self.assertEqual('Rule and attribute documentation.', doc) self.assertDictEqual(expected_attrs, attr_doc) + self.assertEqual('', example_doc) def test_multi_line_doc(self): - doc = ( - '"""Multi-line rule and attribute documentation.\n' + docstring = ( + 'Multi-line rule and attribute documentation.\n' '\n' 'Rule doc continued here.\n' '\n' @@ -55,31 +56,95 @@ def test_multi_line_doc(self): ' Documentation for name continued here.\n' ' visibility: The visibility of this rule.\n' '\n' - ' Documentation for visibility continued here.\n' - '"""\n') + ' Documentation for visibility continued here.\n') expected_doc = ( 'Multi-line rule and attribute documentation.\n' + '\n' 'Rule doc continued here.') expected_attrs = { - 'name': ('A unique name for this rule.\n' - 'Documentation for name continued here'), - 'visibility': ('The visibility of this rule.\n' + 'name': ('A unique name for this rule.\n\n' + 'Documentation for name continued here.'), + 'visibility': ('The visibility of this rule.\n\n' 'Documentation for visibility continued here.') } - doc, attr_doc = common.parse_attribute_doc(doc) + doc, attr_doc, example_doc = common.parse_docstring(docstring) self.assertEqual(expected_doc, doc) self.assertDictEqual(expected_attrs, attr_doc) + self.assertEqual('', example_doc) def test_invalid_args(self): - doc = ( - '"""Rule and attribute documentation.\n' + docstring = ( + 'Rule and attribute documentation.\n' '\n' 'Foo:\n' ' name: A unique name for this rule.\n' - ' visibility: The visibility of this rule.\n' - '"""\n') + ' visibility: The visibility of this rule.\n') - doc, attr_doc = common.parse_attribute_doc(doc) - self.assertEqual('Rule and attribute documentation.', doc) + doc, attr_doc, example_doc = common.parse_docstring(docstring) + self.assertEqual(docstring.strip(), doc) self.assertDictEqual({}, attr_doc) + self.assertEqual('', example_doc) + + def test_example(self): + docstring = ( + 'Documentation with example\n' + '\n' + 'Examples:\n' + ' An example of how to use this rule:\n' + '\n' + ' example_rule()\n' + '\n' + ' Note about this example.\n' + '\n' + 'Args:\n' + ' name: A unique name for this rule.\n' + ' visibility: The visibility of this rule.\n') + expected_attrs = { + 'name': 'A unique name for this rule.', + 'visibility': 'The visibility of this rule.' + } + expected_example_doc = ( + 'An example of how to use this rule:\n' + '\n' + ' example_rule()\n' + '\n' + 'Note about this example.') + + doc, attr_doc, example_doc = common.parse_docstring(docstring) + self.assertEqual('Documentation with example', doc) + self.assertDictEqual(expected_attrs, attr_doc) + self.assertEqual(expected_example_doc, example_doc) + + def test_example_after_attrs(self): + docstring = ( + 'Documentation with example\n' + '\n' + 'Args:\n' + ' name: A unique name for this rule.\n' + ' visibility: The visibility of this rule.\n' + '\n' + 'Examples:\n' + ' An example of how to use this rule:\n' + '\n' + ' example_rule()\n' + '\n' + ' Note about this example.\n') + expected_attrs = { + 'name': 'A unique name for this rule.', + 'visibility': 'The visibility of this rule.' + } + expected_example_doc = ( + 'An example of how to use this rule:\n' + '\n' + ' example_rule()\n' + '\n' + 'Note about this example.') + + doc, attr_doc, example_doc = common.parse_docstring(docstring) + self.assertEqual('Documentation with example', doc) + self.assertDictEqual(expected_attrs, attr_doc) + self.assertEqual(expected_example_doc, example_doc) + +if __name__ == '__main__': + unittest.main() diff --git a/skydoc/macro_extractor.py b/skydoc/macro_extractor.py index e9dea81..8d65fea 100644 --- a/skydoc/macro_extractor.py +++ b/skydoc/macro_extractor.py @@ -72,8 +72,9 @@ def _add_macro_doc(self, stmt): doc = ast.get_docstring(stmt) if doc: - doc, attr_doc = common.parse_attribute_doc(doc) - rule.documentation = doc.strip() + doc, attr_doc, example_doc = common.parse_docstring(doc) + rule.documentation = doc + rule.example_documentation = example_doc else: doc = "" attr_doc = {} diff --git a/skydoc/rule.py b/skydoc/rule.py index f0f03bf..034749b 100644 --- a/skydoc/rule.py +++ b/skydoc/rule.py @@ -88,7 +88,8 @@ class Rule(object): def __init__(self, proto): self.__proto = proto self.name = proto.name - self.documentation = proto.documentation + self.documentation = mistune.markdown(proto.documentation) + self.example_documentation = mistune.markdown(proto.example_documentation) self.signature = self._get_signature(proto) self.attributes = [] for attribute in proto.attribute: diff --git a/skydoc/rule_extractor.py b/skydoc/rule_extractor.py index 3d5cb92..4df5c30 100644 --- a/skydoc/rule_extractor.py +++ b/skydoc/rule_extractor.py @@ -77,15 +77,16 @@ def _add_rule_doc(self, name, doc): name: The name of the rule. doc: The docstring extracted for the rule. """ - doc, attr_doc = common.parse_attribute_doc(doc) + doc, attr_doc, example_doc = common.parse_docstring(doc) if name in self.__extracted_rules: rule = self.__extracted_rules[name] - rule.doc = doc.strip() + rule.doc = doc + rule.example_doc = example_doc for attr_name, attr_doc in attr_doc.iteritems(): if attr_name in rule.attrs: rule.attrs[attr_name].doc = attr_doc - def _parse_docstrings(self, bzl_file): + def _extract_docstrings(self, bzl_file): """Extracts the docstrings for all public rules in the .bzl file. This function parses the .bzl file and extracts the docstrings for all @@ -129,6 +130,7 @@ def _assemble_protos(self): rule = self.__language.rule.add() rule.name = rule_desc.name rule.documentation = rule_desc.doc + rule.example_documentation = rule_desc.example_doc attrs = sorted(rule_desc.attrs.values(), cmp=attr.attr_compare) for attr_desc in attrs: @@ -155,7 +157,7 @@ def parse_bzl(self, bzl_file): bzl_file: The .bzl file to extract rule documentation from. """ self._process_skylark(bzl_file) - self._parse_docstrings(bzl_file) + self._extract_docstrings(bzl_file) self._assemble_protos() def proto(self): diff --git a/skydoc/stubs/skylark_globals.py b/skydoc/stubs/skylark_globals.py index 193f419..e3cf113 100644 --- a/skydoc/stubs/skylark_globals.py +++ b/skydoc/stubs/skylark_globals.py @@ -43,7 +43,8 @@ def __init__(self, label_string): class RuleDescriptor(object): def __init__(self, implementation, test=False, attrs={}, outputs=None, executable=False, output_to_genfiles=False, fragments=[], - host_fragments=[], local=False, doc='', type='rule'): + host_fragments=[], local=False, doc='', example_doc='', + type='rule'): """Constructor for RuleDescriptor Args: @@ -63,6 +64,10 @@ def __init__(self, implementation, test=False, attrs={}, outputs=None, (Only used if type='repository'). doc: Documentation for this rule. This parameter is used internally by skydoc and is not set by any Skylark code in .bzl files. + example_doc: Example documentation for this rule. This parameter is used + internally by skydoc and is not set by any Skylark code in .bzl files. + type: The type of rule (rule, repository_rule). This parameter is used + by skydoc and is not set by any Skylark code in .bzl files. """ self.is_rule = True self.implementation = implementation @@ -75,6 +80,7 @@ def __init__(self, implementation, test=False, attrs={}, outputs=None, self.host_fragments = host_fragments self.local = local self.doc = doc + self.example_doc = example_doc self.type = type for name, attr in self.attrs.iteritems(): attr.name = name diff --git a/skydoc/templates/html.jinja b/skydoc/templates/html.jinja index 8b2dc5e..557c359 100644 --- a/skydoc/templates/html.jinja +++ b/skydoc/templates/html.jinja @@ -64,9 +64,14 @@ Documentation generated by Skydoc % if rule.attributes[0] is defined:

Attributes

- % include "attributes.jinja" % endif + +% if rule.example_documentation +

Examples

+ {{ rule.example_documentation }} +% endif + % endfor diff --git a/skydoc/templates/markdown.jinja b/skydoc/templates/markdown.jinja index 7064cb2..54bb213 100644 --- a/skydoc/templates/markdown.jinja +++ b/skydoc/templates/markdown.jinja @@ -22,8 +22,6 @@ Documentation generated by Skydoc % include "overview.jinja" % for rule in ruleset.rules: -
- ## {{ rule.name }} @@ -39,4 +37,12 @@ Documentation generated by Skydoc % include "attributes.jinja" % endif + +% if rule.example_documentation + +### Examples + +{{ rule.example_documentation }} +#endif + % endfor diff --git a/skylark/skylark.bzl b/skylark/skylark.bzl index 43496c5..f986ae8 100644 --- a/skylark/skylark.bzl +++ b/skylark/skylark.bzl @@ -79,6 +79,74 @@ Args: srcs: List of `.bzl` files that are processed to create this target. deps: List of other `skylark_library` targets that are required by the Skylark files listed in `srcs`. + +Example: + If you would like to generate documentation for multiple .bzl files in various + packages in your workspace, you can use the `skylark_library` rule to create + logical collections of Skylark sources and add a single `skylark_doc` target for + building documentation for all of the rule sets. + + Suppose your project has the following structure: + + ``` + [workspace]/ + WORKSPACE + BUILD + checkstyle/ + BUILD + checkstyle.bzl + lua/ + BUILD + lua.bzl + luarocks.bzl + ``` + + In this case, you can have `skylark_library` targets in `checkstyle/BUILD` and + `lua/BUILD`: + + `checkstyle/BUILD`: + + ```python + load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_library") + + skylark_library( + name = "checkstyle-rules", + srcs = ["checkstyle.bzl"], + ) + ``` + + `lua/BUILD`: + + ```python + load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_library") + + skylark_library( + name = "lua-rules", + srcs = [ + "lua.bzl", + "luarocks.bzl", + ], + ) + ``` + + To build documentation for all the above `.bzl` files at once: + + `BUILD`: + + ```python + load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_doc") + + skylark_doc( + name = "docs", + deps = [ + "//checkstyle:checkstyle-rules", + "//lua:lua-rules", + ], + ) + ``` + + Running `bazel build //:docs` would build a single zip containing documentation + for all the `.bzl` files contained in the two `skylark_library` targets. """ _skylark_doc_attrs = { @@ -104,6 +172,35 @@ Args: files listed in `srcs`. format: The type of output to generate. Possible values are `"markdown"` and `"html"`. + +Example: + Suppose you have a project containing Skylark rules you want to document: + + ``` + [workspace]/ + WORKSPACE + checkstyle/ + BUILD + checkstyle.bzl + ``` + + To generate documentation for the rules and macros in `checkstyle.bzl`, add the + following target to `rules/BUILD`: + + ```python + load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_doc") + + skylark_doc( + name = "checkstyle-docs", + srcs = ["checkstyle.bzl"], + ) + ``` + + Running `bazel build //checkstyle:checkstyle-docs` will generate a zip file + containing documentation for the public rules and macros in `checkstyle.bzl`. + + By default, Skydoc will generate documentation in Markdown. To generate + a set of HTML pages that is ready to be served, set `format = "html"`. """ JINJA2_BUILD_FILE = """