Support extracting inline example documentation.

Fixes #1

RELNOTES: Support inline example documentation.

--
MOS_MIGRATED_REVID=119726298

skydoc/common.py

@@ -15,52 +15,116 @@
 """Common functions for skydoc."""
 
 import re
+import textwrap
 from xml.sax.saxutils import escape
 
 
+ARGS_HEADING = "Args:"
+EXAMPLES_HEADING = "Examples:"
+EXAMPLE_HEADING = "Example:"
+
+
 def leading_whitespace(line):
+  """Returns the number of leading whitespace in the line."""
   return len(line) - len(line.lstrip())
 
-def parse_attribute_doc(doc):
-  """Analyzes the documentation string for attributes.
 
-  This looks for the "Args:" separator to fetch documentation for each
-  attribute. The "Args" section ends at the first blank line.
+def _parse_attribute_docs(attr_doc, lines, index):
+  """Extracts attribute documentation.
 
   Args:
-    doc: The documentation string
+    attr_doc: A dict used to store the extracted attribute documentation.
+    lines: List containing the input docstring split into lines.
+    index: The index in lines containing "Args:", which begins the argument
+        documentation.
 
   Returns:
-    The new documentation string and a dictionary that maps each attribute to
-    its documentation
+    Returns the next index after the attribute documentation to resume
+    processing documentation in the caller.
   """
-  doc_attr = {}
-  lines = doc.split("\n")
-  if "Args:" not in lines:
-    return doc, doc_attr
-  start = lines.index("Args:")
-
-  i = start + 1
-  var = None  # Current attribute name
+  attr = None  # Current attribute name
   desc = None  # Description for current attribute
-  args_leading_ws = leading_whitespace(lines[start])
-  for i in xrange(start + 1, len(lines)):
+  args_leading_ws = leading_whitespace(lines[index])
+  i = index + 1
+  while i < len(lines):
+    line = lines[i]
     # If a blank line is encountered, we have finished parsing the "Args"
     # section.
-    if lines[i].strip() and leading_whitespace(lines[i]) == args_leading_ws:
+    if line.strip() and leading_whitespace(line) == args_leading_ws:
       break
     # In practice, users sometimes add a "-" prefix, so we strip it even
     # though it is not recommended by the style guide
-    match = re.search(r"^\s*-?\s*(\w+):\s*(.*)", lines[i])
+    match = re.search(r"^\s*-?\s*(\w+):\s*(.*)", line)
     if match:  # We have found a new attribute
-      if var:
-        doc_attr[var] = escape(desc)
-      var, desc = match.group(1), match.group(2)
-    elif var:
+      if attr:
+        attr_doc[attr] = escape(desc)
+      attr, desc = match.group(1), match.group(2)
+    elif attr:
       # Merge documentation when it is multiline
-      desc = desc + "\n" + lines[i].strip()
+      desc = desc + "\n" + line.strip()
+    i += + 1
+
+  if attr:
+    attr_doc[attr] = escape(desc).strip()
+
+  return i
+
+
+def _parse_example_docs(examples, lines, index):
+  """Extracts example documentation.
+
+  Args:
+    examples: A list to contain the lines containing the example documentation.
+    lines: List containing the input docstring split into lines.
+    index: The index in lines containing "Example[s]:", which begins the
+        example documentation.
+
+  Returns:
+    Returns the next index after the attribute documentation to resume
+    processing documentation in the caller.
+  """
+  heading_leading_ws = leading_whitespace(lines[index])
+  i = index + 1
+  while i < len(lines):
+    line = lines[i]
+    if line.strip() and leading_whitespace(line) == heading_leading_ws:
+      break
+    examples.append(line)
+    i += 1
+
+  return i
+
+
+def parse_docstring(doc):
+  """Analyzes the documentation string for attributes.
+
+  This looks for the "Args:" separator to fetch documentation for each
+  attribute. The "Args" section ends at the first blank line.
+
+  Args:
+    doc: The documentation string
+
+  Returns:
+    The new documentation string and a dictionary that maps each attribute to
+    its documentation
+  """
+  attr_doc = {}
+  examples = []
+  lines = doc.split("\n")
+  docs = []
+  i = 0
+  while i < len(lines):
+    line = lines[i]
+    if line.strip() == ARGS_HEADING:
+      i = _parse_attribute_docs(attr_doc, lines, i)
+      continue
+    elif line.strip() == EXAMPLES_HEADING or line.strip() == EXAMPLE_HEADING:
+      i = _parse_example_docs(examples, lines, i)
+      continue
+
+    docs.append(line)
+    i += 1
 
-  if var:
-    doc_attr[var] = escape(desc)
-  doc = "\n".join(lines[:start - 1])
-  return doc, doc_attr
+  doc = "\n".join(docs).strip()
+  examples_doc = textwrap.dedent("\n".join(examples)).strip()
+  return doc, attr_doc, examples_doc

skydoc/common_test.py

@@ -21,31 +21,32 @@ class CommonTest(unittest.TestCase):
   """Unit tests for common functions."""
 
   def test_rule_doc_only(self):
-    doc = '"""Rule documentation only docstring."""\n'
-    doc, attr_doc = common.parse_attribute_doc(doc)
+    docstring = 'Rule documentation only docstring.'
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
     self.assertEqual('Rule documentation only docstring.', doc)
     self.assertDictEqual({}, attr_doc)
+    self.assertEqual('', example_doc)
 
   def test_rule_and_attribute_doc(self):
-    doc = (
-        '"""Rule and attribute documentation.\n'
+    docstring = (
+        'Rule and attribute documentation.\n'
         '\n'
         'Args:\n'
         '  name: A unique name for this rule.\n'
-        '  visibility: The visibility of this rule.\n'
-        '"""\n')
+        '  visibility: The visibility of this rule.\n')
     expected_attrs = {
         'name': 'A unique name for this rule.',
         'visibility': 'The visibility of this rule.'
     }
 
-    doc, attr_doc = common.parse_attribute_doc(doc)
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
     self.assertEqual('Rule and attribute documentation.', doc)
     self.assertDictEqual(expected_attrs, attr_doc)
+    self.assertEqual('', example_doc)
 
   def test_multi_line_doc(self):
-    doc = (
-        '"""Multi-line rule and attribute documentation.\n'
+    docstring = (
+        'Multi-line rule and attribute documentation.\n'
         '\n'
         'Rule doc continued here.\n'
         '\n'
@@ -55,31 +56,95 @@ def test_multi_line_doc(self):
         '    Documentation for name continued here.\n'
         '  visibility: The visibility of this rule.\n'
         '\n'
-        '    Documentation for visibility continued here.\n'
-        '"""\n')
+        '    Documentation for visibility continued here.\n')
     expected_doc = (
         'Multi-line rule and attribute documentation.\n'
+        '\n'
         'Rule doc continued here.')
     expected_attrs = {
-        'name': ('A unique name for this rule.\n'
-            'Documentation for name continued here'),
-        'visibility': ('The visibility of this rule.\n'
+        'name': ('A unique name for this rule.\n\n'
+            'Documentation for name continued here.'),
+        'visibility': ('The visibility of this rule.\n\n'
             'Documentation for visibility continued here.')
     }
 
-    doc, attr_doc = common.parse_attribute_doc(doc)
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
     self.assertEqual(expected_doc, doc)
     self.assertDictEqual(expected_attrs, attr_doc)
+    self.assertEqual('', example_doc)
 
   def test_invalid_args(self):
-    doc = (
-        '"""Rule and attribute documentation.\n'
+    docstring = (
+        'Rule and attribute documentation.\n'
         '\n'
         'Foo:\n'
         '  name: A unique name for this rule.\n'
-        '  visibility: The visibility of this rule.\n'
-        '"""\n')
+        '  visibility: The visibility of this rule.\n')
 
-    doc, attr_doc = common.parse_attribute_doc(doc)
-    self.assertEqual('Rule and attribute documentation.', doc)
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
+    self.assertEqual(docstring.strip(), doc)
     self.assertDictEqual({}, attr_doc)
+    self.assertEqual('', example_doc)
+
+  def test_example(self):
+    docstring = (
+        'Documentation with example\n'
+        '\n'
+        'Examples:\n'
+        '  An example of how to use this rule:\n'
+        '\n'
+        '      example_rule()\n'
+        '\n'
+        '  Note about this example.\n'
+        '\n'
+        'Args:\n'
+        '  name: A unique name for this rule.\n'
+        '  visibility: The visibility of this rule.\n')
+    expected_attrs = {
+        'name': 'A unique name for this rule.',
+        'visibility': 'The visibility of this rule.'
+    }
+    expected_example_doc = (
+        'An example of how to use this rule:\n'
+        '\n'
+        '    example_rule()\n'
+        '\n'
+        'Note about this example.')
+
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
+    self.assertEqual('Documentation with example', doc)
+    self.assertDictEqual(expected_attrs, attr_doc)
+    self.assertEqual(expected_example_doc, example_doc)
+
+  def test_example_after_attrs(self):
+    docstring = (
+        'Documentation with example\n'
+        '\n'
+        'Args:\n'
+        '  name: A unique name for this rule.\n'
+        '  visibility: The visibility of this rule.\n'
+        '\n'
+        'Examples:\n'
+        '  An example of how to use this rule:\n'
+        '\n'
+        '      example_rule()\n'
+        '\n'
+        '  Note about this example.\n')
+    expected_attrs = {
+        'name': 'A unique name for this rule.',
+        'visibility': 'The visibility of this rule.'
+    }
+    expected_example_doc = (
+        'An example of how to use this rule:\n'
+        '\n'
+        '    example_rule()\n'
+        '\n'
+        'Note about this example.')
+
+    doc, attr_doc, example_doc = common.parse_docstring(docstring)
+    self.assertEqual('Documentation with example', doc)
+    self.assertDictEqual(expected_attrs, attr_doc)
+    self.assertEqual(expected_example_doc, example_doc)
+
+if __name__ == '__main__':
+  unittest.main()

skydoc/macro_extractor.py

@@ -72,8 +72,9 @@ def _add_macro_doc(self, stmt):
 
     doc = ast.get_docstring(stmt)
     if doc:
-      doc, attr_doc = common.parse_attribute_doc(doc)
-      rule.documentation = doc.strip()
+      doc, attr_doc, example_doc = common.parse_docstring(doc)
+      rule.documentation = doc
+      rule.example_documentation = example_doc
     else:
       doc = ""
       attr_doc = {}

skydoc/rule.py

@@ -88,7 +88,8 @@ class Rule(object):
   def __init__(self, proto):
     self.__proto = proto
     self.name = proto.name
-    self.documentation = proto.documentation
+    self.documentation = mistune.markdown(proto.documentation)
+    self.example_documentation = mistune.markdown(proto.example_documentation)
     self.signature = self._get_signature(proto)
     self.attributes = []
     for attribute in proto.attribute:

skydoc/rule_extractor.py

@@ -77,15 +77,16 @@ def _add_rule_doc(self, name, doc):
       name: The name of the rule.
       doc: The docstring extracted for the rule.
     """
-    doc, attr_doc = common.parse_attribute_doc(doc)
+    doc, attr_doc, example_doc = common.parse_docstring(doc)
     if name in self.__extracted_rules:
       rule = self.__extracted_rules[name]
-      rule.doc = doc.strip()
+      rule.doc = doc
+      rule.example_doc = example_doc
       for attr_name, attr_doc in attr_doc.iteritems():
         if attr_name in rule.attrs:
           rule.attrs[attr_name].doc = attr_doc
 
-  def _parse_docstrings(self, bzl_file):
+  def _extract_docstrings(self, bzl_file):
     """Extracts the docstrings for all public rules in the .bzl file.
 
     This function parses the .bzl file and extracts the docstrings for all
@@ -129,6 +130,7 @@ def _assemble_protos(self):
       rule = self.__language.rule.add()
       rule.name = rule_desc.name
       rule.documentation = rule_desc.doc
+      rule.example_documentation = rule_desc.example_doc
 
       attrs = sorted(rule_desc.attrs.values(), cmp=attr.attr_compare)
       for attr_desc in attrs:
@@ -155,7 +157,7 @@ def parse_bzl(self, bzl_file):
       bzl_file: The .bzl file to extract rule documentation from.
     """
     self._process_skylark(bzl_file)
-    self._parse_docstrings(bzl_file)
+    self._extract_docstrings(bzl_file)
     self._assemble_protos()
 
   def proto(self):

skydoc/stubs/skylark_globals.py

@@ -43,7 +43,8 @@ def __init__(self, label_string):
 class RuleDescriptor(object):
   def __init__(self, implementation, test=False, attrs={}, outputs=None,
                executable=False, output_to_genfiles=False, fragments=[],
-               host_fragments=[], local=False, doc='', type='rule'):
+               host_fragments=[], local=False, doc='', example_doc='',
+               type='rule'):
     """Constructor for RuleDescriptor
 
     Args:
@@ -63,6 +64,10 @@ def __init__(self, implementation, test=False, attrs={}, outputs=None,
           (Only used if type='repository').
       doc: Documentation for this rule. This parameter is used internally by
           skydoc and is not set by any Skylark code in .bzl files.
+      example_doc: Example documentation for this rule. This parameter is used
+          internally by skydoc and is not set by any Skylark code in .bzl files.
+      type: The type of rule (rule, repository_rule). This parameter is used
+          by skydoc and is not set by any Skylark code in .bzl files.
     """
     self.is_rule = True
     self.implementation = implementation
@@ -75,6 +80,7 @@ def __init__(self, implementation, test=False, attrs={}, outputs=None,
     self.host_fragments = host_fragments
     self.local = local
     self.doc = doc
+    self.example_doc = example_doc
     self.type = type
     for name, attr in self.attrs.iteritems():
       attr.name = name

skydoc/templates/html.jinja

@@ -64,9 +64,14 @@ Documentation generated by Skydoc
 
 % if rule.attributes[0] is defined:
           <h3 id="{{ rule.name }}_args">Attributes</h3>
-
 % include "attributes.jinja"
 % endif
+
+% if rule.example_documentation
+          <h3 id="{{ rule.name }}_examples">Examples</h3>
+          {{ rule.example_documentation }}
+% endif
+
 % endfor
         </div>