feat: Support {% tab %} and {% sample %} in AIPs. (#9)

.prettierrc.yaml

@@ -1,4 +1,8 @@
 ---
+overrides:
+  - files: '*.md.j2'
+    options:
+      parser: markdown
 printWidth: 79
 proseWrap: always
 singleQuote: true

aip_site/env.py → aip_site/jinja/env.py

@@ -15,15 +15,18 @@
 import os
 
 import jinja2
+import jinja2.ext
+import jinja2.nodes
 
 from aip_site import md
 
 
 TEMPLATE_DIR = os.path.realpath(
-    os.path.join(os.path.dirname(__file__), 'support', 'templates'),
+    os.path.join(os.path.dirname(__file__), '..', 'support', 'templates'),
 )
 
 
+# "Standard" jinja2 environment that loads from the filesystem.
 jinja_env = jinja2.Environment(
     loader=jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR),
     undefined=jinja2.StrictUndefined,

aip_site/jinja/ext/sample.py

@@ -0,0 +1,142 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os
+import re
+import textwrap
+
+import jinja2.ext
+import jinja2.nodes
+
+
+class SampleExtension(jinja2.ext.Extension):
+    tags = {'sample'}
+
+    def parse(self, parser):
+        # The first token is the token that started the tag, which is always
+        # "sample" because that is the only token we watch.
+        lineno = next(parser.stream).lineno
+
+        # Get the sample to be loaded.
+        # A sample includes both a filename and a symbol within that file,
+        # which is what is shown in the output.
+        fn = parser.stream.expect('string').value
+        symbols = []
+        while parser.stream.look().type == 'string':
+            parser.stream.expect('comma')
+            symbols.append(parser.stream.expect('string').value)
+
+        # Load the sample file.
+        aip = self.environment.loader.aip
+        filename = os.path.join(aip.path, fn)
+        try:
+            with io.open(filename, 'r') as f:
+                code = f.read()
+        except FileNotFoundError:
+            raise jinja2.TemplateSyntaxError(
+                filename=parser.filename,
+                lineno=lineno,
+                message=f'File not found: {filename}',
+            )
+
+        # Tease out the desired symbols and make snippets.
+        snippets = []
+        for symbol in symbols:
+            match = re.search(rf'^([\s]*)({symbol})', code, flags=re.MULTILINE)
+            if not match:
+                raise jinja2.TemplateSyntaxError(
+                    filename=parser.filename,
+                    lineno=lineno,
+                    message=f'Symbol not found: {symbol}',
+                )
+
+            # Determine the end of the symbol.
+            # This attempts to parse C-style brace syntax if it encounters a
+            # `{` character, or Python/YAML-style indenting if it encounters a
+            # `:` character.
+            #
+            # The first thing we need to know is which syntax we are using.
+            # We attempt to guess that by seeing which token we encounter next
+            # after our symbol.
+            start = match.start()
+            try:
+                ix, block_token = sorted([
+                    (loc + 1, i) for i in (':', '{', ';')
+                    if (loc := code.find(i, start)) != -1])[0]
+            except IndexError:
+                raise jinja2.TemplateSyntaxError(
+                    filename=filename,
+                    lineno=code.count('\n', 0, start) - 1,
+                    message=f'No block character (:, {{) found after {symbol}',
+                )
+
+            # Push the start marker backwards to include any leading comments.
+            lines = code[0:start - 1].split('\n')
+            for line in reversed(lines):
+                if re.search(r'^[\s]*(//|#)', line):
+                    start -= len(line) + 1
+                else:
+                    break
+
+            # If we got a `:`, we parse by indentation, stopping at the
+            # start of the next line with the same indentation as our match.
+            snippet = ''
+            if block_token == ':':
+                indent = match.groups()[0]
+                end_match = re.search(rf'^{indent}[\S]+', code[ix:],
+                    re.MULTILINE,
+                )
+                if end_match:
+                    snippet = code[start:end_match.start() + ix]
+                else:
+                    snippet = code[start:]
+                snippet = textwrap.dedent(snippet)
+
+            # We got a '{'; Find the corresponding closed brace.
+            elif block_token == '{':
+                cursor = match.start()
+                while (close_brace := code.find('}', cursor)) != -1:
+                    s, e = match.start(), close_brace + 1
+                    if code.count('{', s, e) == code.count('}', s, e):
+                        snippet = textwrap.dedent(code[start:e])
+                        break
+                    cursor = e
+                else:
+                    # Unable to find a corresponding closed brace; complain.
+                    raise jinja2.TemplateSyntaxError(
+                        filename=filename,
+                        message=f'No corresponding }} found for {symbol}.',
+                        lineno=code.count('\n', 0, start) - 1,
+                    )
+
+            # We got a ';'. Stop there.
+            else:
+                end = code.find(';', match.start()) + 1
+                snippet = textwrap.dedent(code[start:end])
+
+            # Append the snippet to the list of snippets.
+            snippets.append(snippet)
+
+        # We have a snippet. Time to put the Markdown together.
+        md = '\n'.join((
+            '```{0}'.format(filename.split('.')[-1]),
+            '\n\n'.join(snippets),
+            '```',
+        ))
+
+        # Finally, return a node to display it.
+        return jinja2.nodes.Output(
+            [jinja2.nodes.TemplateData(md)],
+        )

aip_site/jinja/ext/tab.py

@@ -0,0 +1,57 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jinja2.ext
+import jinja2.nodes
+
+
+class TabExtension(jinja2.ext.Extension):
+    tags = {'tab'}
+
+    def parse(self, parser, cursor=()):
+        # The first token is the token that started the tag, which is always
+        # "tab" because that is the only token we watch.
+        lineno = next(parser.stream).lineno
+
+        # Get the language for this tab.
+        lang = parser.stream.expect('name')
+        lang_title = {
+            'oas': 'OpenAPI 3.0',
+            'proto': 'Protocol buffers',
+        }.get(lang.value, lang.value.capitalize())
+
+        # Encase the tab's content in a Markdown tab, properly indented.
+        tab_title = jinja2.nodes.Output(
+            [jinja2.nodes.TemplateData(f'=== "{lang_title}"\n\n')],
+        )
+        body = parser.parse_statements(['name:endtabs', 'name:tab'])
+        indented_body = jinja2.nodes.FilterBlock(
+            body,
+            jinja2.nodes.Filter(
+                None, 'indent', (), [
+                    jinja2.nodes.Keyword('width', jinja2.nodes.Const(2)),
+                    jinja2.nodes.Keyword('first', jinja2.nodes.Const(True)),
+                ], None, None,
+            ),
+        ).set_lineno(lineno)
+        cursor += (tab_title, indented_body)
+
+        # If there is another tab, parse it too.
+        if parser.stream.current.value == 'tab':
+            return self.parse(parser, cursor=cursor)
+        else:
+            next(parser.stream)  # Drop endtabs.
+
+        # Done; return the content.
+        return list(cursor)

aip_site/jinja/loaders.py

@@ -0,0 +1,95 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os
+import re
+import typing
+
+import jinja2
+import jinja2.loaders
+
+from aip_site.md import MarkdownDocument
+
+if typing.TYPE_CHECKING:
+    from aip_site.models.aip import AIP
+
+
+class AIPLoader(jinja2.loaders.BaseLoader):
+    """Loader for loading AIPs."""
+
+    def __init__(self, aip: 'AIP'):
+        super().__init__()
+        self.aip = aip
+
+    def get_source(self, env, template: str) -> typing.Tuple[str, str, None]:
+        # Find the appropriate AIP file.
+        if template == 'generic':
+            fn = os.path.join(self.aip.path, 'aip.md')
+        else:
+            view = template.split('.')
+            while view:
+                view_str = '.'.join(view)
+                fn = os.path.join(self.aip.path, f'aip.{view_str}.md')
+                if os.path.isfile(fn) or os.path.isfile(f'{fn}.j2'):
+                    break
+                view = view[1:]
+
+        # Sanity check: Does the file exist?
+        # If not, raise an error.
+        if not os.path.isfile(fn) and not os.path.isfile(f'{fn}.j2'):
+            raise jinja2.TemplateNotFound(
+                f'Could not find {template} template for AIP-{self.aip.id}.',
+            )
+
+        # Are we loading a plain file or a Jinja2 template file?
+        if os.path.isfile(f'{fn}.j2'):
+            fn += '.j2'
+
+        # Load the contents.
+        with io.open(fn) as f:
+            contents = f.read()
+
+        # Add custom {% block %} tags corresponding to Markdown headings.
+        #
+        # Note: We only do this if the template does not already have
+        # {% block %} tags to avoid either stomping over input, or creating
+        # an invalid template.
+        if not re.search(r'\{%-? block', contents):
+            # Iterate over the individual components in the table
+            # of contents and make each into a block.
+            contents = MarkdownDocument(contents).blocked_content
+
+        # Return the template information.
+        return contents, fn, None
+
+    def list_templates(self) -> typing.Sequence[str]:
+        answer = []
+
+        # We sort the files in the directory to read more specific
+        # files first.
+        exts_regex = r'(\.(j2|md|proto|oas|yaml))*$'
+        for fn in sorted(os.listdir(self.aip.path),
+                key=lambda p: len(re.sub(exts_regex, '', p).split('.')),
+                reverse=True):
+
+            # Each file may specify a view, which corresponds to a separate
+            # template.
+            view = '.'.join(re.sub(exts_regex, '', fn).split('.')[1:])
+            if view and view not in answer:
+                answer.append(view)
+
+        # There is always a generic view.
+        answer.append('generic')
+        return answer