From 55dafa9e7204742f9c3329241d8fdf6d667288ee Mon Sep 17 00:00:00 2001 From: "David Z. Chen" Date: Fri, 25 Mar 2016 15:55:54 -0700 Subject: [PATCH] Initial import. --- AUTHORS | 9 + BUILD | 4 + CONTRIBUTING.md | 27 ++ CONTRIBUTORS | 12 + LICENSE | 202 +++++++++++++++ README.md | 279 +++++++++++++++++++++ WORKSPACE | 10 + skydoc/BUILD | 76 ++++++ skydoc/common.py | 67 +++++ skydoc/common_test.py | 85 +++++++ skydoc/macro_extractor.py | 116 +++++++++ skydoc/macro_extractor_test.py | 290 ++++++++++++++++++++++ skydoc/main.py | 236 ++++++++++++++++++ skydoc/rule.py | 120 +++++++++ skydoc/rule_extractor.py | 164 +++++++++++++ skydoc/rule_extractor_test.py | 393 ++++++++++++++++++++++++++++++ skydoc/sass/BUILD | 8 + skydoc/sass/main.scss | 240 ++++++++++++++++++ skydoc/stubs/BUILD | 12 + skydoc/stubs/attr.py | 112 +++++++++ skydoc/stubs/skylark_globals.py | 90 +++++++ skydoc/templates/BUILD | 13 + skydoc/templates/attributes.jinja | 32 +++ skydoc/templates/html.jinja | 88 +++++++ skydoc/templates/markdown.jinja | 42 ++++ skydoc/templates/nav.jinja | 28 +++ skydoc/templates/overview.jinja | 22 ++ skydoc/templates/toc.jinja | 26 ++ skylark/BUILD | 9 + skylark/skylark.bzl | 237 ++++++++++++++++++ 30 files changed, 3049 insertions(+) create mode 100644 AUTHORS create mode 100644 BUILD create mode 100644 CONTRIBUTING.md create mode 100644 CONTRIBUTORS create mode 100644 LICENSE create mode 100644 README.md create mode 100644 WORKSPACE create mode 100644 skydoc/BUILD create mode 100644 skydoc/common.py create mode 100644 skydoc/common_test.py create mode 100644 skydoc/macro_extractor.py create mode 100644 skydoc/macro_extractor_test.py create mode 100755 skydoc/main.py create mode 100644 skydoc/rule.py create mode 100644 skydoc/rule_extractor.py create mode 100644 skydoc/rule_extractor_test.py create mode 100644 skydoc/sass/BUILD create mode 100644 skydoc/sass/main.scss create mode 100644 skydoc/stubs/BUILD create mode 100644 skydoc/stubs/attr.py create mode 100644 skydoc/stubs/skylark_globals.py create mode 100644 skydoc/templates/BUILD create mode 100644 skydoc/templates/attributes.jinja create mode 100644 skydoc/templates/html.jinja create mode 100644 skydoc/templates/markdown.jinja create mode 100644 skydoc/templates/nav.jinja create mode 100644 skydoc/templates/overview.jinja create mode 100644 skydoc/templates/toc.jinja create mode 100644 skylark/BUILD create mode 100644 skylark/skylark.bzl diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..8f95963 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,9 @@ +# This the official list of Bazel authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. + +Google Inc. diff --git a/BUILD b/BUILD new file mode 100644 index 0000000..9a2055d --- /dev/null +++ b/BUILD @@ -0,0 +1,4 @@ +filegroup( + name = "dummy", + visibility = ["//visibility:public"], +) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9d3d1aa --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +Want to contribute? Great! First, read this page (including the small print at the end). + +### Before you contribute +**Before we can use your code, you must sign the +[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) +(CLA)**, which you can do online. + +The CLA is necessary mainly because you own the copyright to your changes, +even after your contribution becomes part of our codebase, so we need your +permission to use and distribute your code. We also need to be sure of +various other things — for instance that you'll tell us if you know that +your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. + +Before you start working on a larger contribution, you should get in touch +with us first. Use the issue tracker to explain your idea so we can help and +possibly guide you. + +### Code reviews and other contributions. +**All submissions, including submissions by project members, require review.** +Please follow the instructions in [the contributors documentation](http://bazel.io/contributing.html). + +### The small print +Contributions made by corporations are covered by a different agreement than +the one above, the +[Software Grant and Corporate Contributor License Agreement](https://cla.developers.google.com/about/google-corporate). diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..e8e0f12 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,12 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# Names should be added to this file as: +# Name + +David Z. Chen diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0ed8998 --- /dev/null +++ b/README.md @@ -0,0 +1,279 @@ +# Skydoc - Skylark Documentation Generator + +Skydoc is a documentation generator for [Bazel](http://bazel.io) build rules +written in [Skylark](http://bazel.io/docs/skylark/index.html). + +Skydoc provides a set of Skylark rules (`skylark_library` and `skylark_doc`) +that can be used to build documentation for Skylark rules in either Markdown or +HTML. Skydoc generates one documentation page per `.bzl` file. + +* [Setup](#setup) +* [Documentation Format](#format) + * [Rule Documentation](#format-rule-documentation) + * [Macro Documentation](#format-macro-documentation) + * [File Documentation](#format-file-documentation) +* [Usage](#usage) + * [Single Target](#usage-single-target) + * [Multiple Targets](#usage-multiple-targets) +* [Roadmap](#roadmap) + + +## Setup + +To use Skydoc, add the following to your `WORKSPACE` file: + +```python +git_repository( + name = "io_bazel_rules_sass", + remote = "https://github.com/bazelbuild/rules_sass.git", + tag = "0.0.1", +) +load("@io_bazel_rules_sass//sass:sass.bzl", "sass_repositories") +sass_repositories() + +git_repository( + name = "io_bazel_skydoc", + remote = "https://github.com/bazelbuild/skydoc.git", + tag = "0.0.1", +) +load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories") +skydoc_repositories() +``` + +Note that the Sass repositoires also need to be loaded since Skydoc uses the +Bazel Sass rules. + +If you would like to load all Skydoc rules by default using Bazel's prelude, add +the following to the file `tools/build_rules/prelude_bazel` in your repository: + +```python +load( + "@io_bazel_skydoc//skylark:skylark.bzl", + "skydoc_repositories", + "skylark_library", + "skylark_doc", +) +``` + + +## Documentation Format + +Since Skylark is a subset of Python, it uses Python docstrings for inline +documentation for Skylark rules and macros as well as file (module) docstrings +for documenting `.bzl` files. Skydoc supports Markdown for all inline +documentation. + +When generating documentation, Skydoc parses the `.bzl` file to extract the +inline documentation as well as evaluates the Skylark code to determine the +types for rule attributes. Skydoc will generate documentation for all public +rules and macros. For undocumented rules and macros, Skydoc will still generate +the rule signature and table of attributes. + +Private rules and macros (i.e. those whose names begin with `_`) will not +appear in generated documentation. + + +### Rule Documentation + +[Skylark Rules](http://bazel.io/docs/skylark/rules.html) are declared using the +`rule()` function as global variables. As a result, they are documented using +variable docstrings, similar to those supported by +[epydoc](http://epydoc.sourceforge.net/manual-docstring.html). + +Attributes are documented in a special `Args:` section. Begin the documentation +for each attribute on an indented line with the attribute name followed by a +colon `:`. The documentation for an attribute can span multiple lines as long as +each line is indented from the first line: + +```python +checkstyle = rule( + implementation = _impl, + attrs = { + "srcs": attr.label_list(allow_files = FileType([".java"]), + "config": attr.label(), + }, +) +"""Runs checkstyle on the given source files. + +This rule runs [Checkstyle](http://checkstyle.sourceforge.net/) on a set of +Java source files. + +Args: + srcs: The Java source files to run checkstyle against. + config: The checkstyle configuration file to use. + + If no configuration file is provided, then the default `checkstyle.xml` will + be used. +""" +``` + +The `name` attribute that is common to all rules is documented by default, but +the default documentation can be overwridden by adding documentation for `name` +in `Args`. + + +### Macro Documentation + +[Skylark Macros](http://bazel.io/docs/skylark/macros.html) are Python functions +and are thus documented using function docstrings: + +```python +def rat_check(name, srcs=[], format, visibility): + """Runs Apache Rat license checks on the given source files. + + This rule runs [Apache Rat](http://creadur.apache.org/rat/) license checks on + a given set of source files. Use `bazel build` to run the check. + + Args: + name: A unique name for this rule. + srcs: Source files to run the Rat license checks against. + + Note that the Bazel glob() function can be used to specify which source + files to include and which to exclude. + format: The format to write the Rat check report in. + visibility: The visibility of this rule. + """ + if format not in ['text', 'html', 'xml']: + fail('Invalid format: %s' % format, 'format') + + _rat_check( + name = name, + srcs = srcs, + format = format, + visibility = visibility, + ) +``` + +Note that the format of the docstrings for rules and macros are identical. + + +### File Documentation + +Skydoc also supports file docstrings (similar to Python's module docstrings), +which can be used to document a `.bzl` file, such as providing an overview of +the rules and macros implemented in the file: + +```python +"""Checkstyle Rules + +Skylark rules for running [Checkstyle](http://checkstyle.sourceforge.net/) on +Java source files. +""" +``` + +If a file docstring is provided, the short docstring will be used as the title +on the generated documentation file, and the rest of the docstring (separated +from the title with an empty line) will be used to generate an Overview section +on the page. + + +## Usage + +The following are some examples of how to use Skydoc. + + +### Single Target + +Suppose you have a project containing Skylark rules you want to document: + +``` +[workspace]/ + WORKSPACE + checkstyle/ + BUILD + checkstyle.bzl +``` + +To generate documentation for the rules and macros in `checkstyle.bzl`, add the +following target to `rules/BUILD`: + +```python +load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_doc") + +skylark_doc( + name = "checkstyle-docs", + srcs = ["checkstyle.bzl"], +) +``` + +Running `bazel build //checkstyle:checkstyle-docs` will generate a zip file +containing documentation for the public rules and macros in `checkstyle.bzl`. + +By default, Skydoc will generate documentation in Markdown. To generate +a set of HTML pages that is ready to be served, set `format = "html"`. + + +### Multiple Targets + +If you would like to generate documentation for multiple .bzl files in various +packages in your workspace, you can use the `skylark_library` rule to create +logical collections of Skylark sources and add a single `skylark_doc` target for +building documentation for all of the rule sets. + +Suppose your project has the following structure: + +``` +[workspace]/ + WORKSPACE + BUILD + checkstyle/ + BUILD + checkstyle.bzl + lua/ + BUILD + lua.bzl + luarocks.bzl +``` + +In this case, you can have `skylark_library` targets in `checkstyle/BUILD` and +`lua/BUILD`: + +`checkstyle/BUILD`: + +```python +load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_library") + +skylark_library( + name = "checkstyle-rules", + srcs = ["checkstyle.bzl"], +) +``` + +`lua/BUILD`: + +```python +load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_library") + +skylark_library( + name = "lua-rules", + srcs = [ + "lua.bzl", + "luarocks.bzl", + ], +) +``` + +To build documentation for all the above `.bzl` files at once: + +`BUILD`: + +```python +load("@io_bazel_skydoc//skylark:skylark.bzl", "skylark_doc") + +skylark_doc( + name = "docs", + deps = [ + "//checkstyle:checkstyle-rules", + "//lua:lua-rules", + ], +) +``` + +Running `bazel build //:docs` would build a single zip containing documentation +for all the `.bzl` files contained in the two `skylark_library` targets. + + +## Roadmap + +* Support syntax for providing examples for rule and macro documentation. +* Document default values for rule and macro attributes. diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..98a5047 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,10 @@ +git_repository( + name = "io_bazel_rules_sass", + remote = "https://github.com/bazelbuild/rules_sass.git", + tag = "0.0.1", +) +load("@io_bazel_rules_sass//sass:sass.bzl", "sass_repositories") +sass_repositories() + +load("//skylark:skylark.bzl", "skydoc_repositories") +skydoc_repositories() diff --git a/skydoc/BUILD b/skydoc/BUILD new file mode 100644 index 0000000..fbc0c1b --- /dev/null +++ b/skydoc/BUILD @@ -0,0 +1,76 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "common", + srcs = ["common.py"], + deps = [ + "@bazel_tools//src/main/protobuf:build_pb_py", + ], +) + +py_test( + name = "common_test", + srcs = ["common_test.py"], + deps = [ + ":common", + ], +) + +py_library( + name = "macro_extractor", + srcs = ["macro_extractor.py"], + deps = [":common"], +) + +py_test( + name = "macro_extractor_test", + srcs = ["macro_extractor_test.py"], + deps = [ + ":macro_extractor", + "@bazel_tools//src/main/protobuf:build_pb_py", + ], +) + +py_library( + name = "rule_extractor", + srcs = ["rule_extractor.py"], + deps = [ + ":common", + "//skydoc/stubs", + ], +) + +py_test( + name = "rule_extractor_test", + srcs = ["rule_extractor_test.py"], + deps = [ + ":rule_extractor", + "@bazel_tools//src/main/protobuf:build_pb_py", + ], +) + +py_library( + name = "rule", + srcs = ["rule.py"], + deps = [ + "//external:mistune", + "@bazel_tools//src/main/protobuf:build_pb_py", + ], +) + +py_binary( + name = "skydoc", + srcs = ["main.py"], + main = "main.py", + deps = [ + ":macro_extractor", + ":rule", + ":rule_extractor", + "//external:jinja2", + "//external:gflags", + ], + data = [ + "//skydoc/templates", + "//skydoc/sass:main.css", + ], +) diff --git a/skydoc/common.py b/skydoc/common.py new file mode 100644 index 0000000..448fb09 --- /dev/null +++ b/skydoc/common.py @@ -0,0 +1,67 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common functions for skydoc.""" + +import re +from src.main.protobuf import build_pb2 +from xml.sax.saxutils import escape + + +def leading_whitespace(line): + return len(line) - len(line.lstrip()) + +def parse_attribute_doc(doc): + """Analyzes the documentation string for attributes. + + This looks for the "Args:" separator to fetch documentation for each + attribute. The "Args" section ends at the first blank line. + + Args: + doc: The documentation string + + Returns: + The new documentation string and a dictionary that maps each attribute to + its documentation + """ + doc_attr = {} + lines = doc.split("\n") + if "Args:" not in lines: + return doc, doc_attr + start = lines.index("Args:") + + i = start + 1 + var = None # Current attribute name + desc = None # Description for current attribute + args_leading_ws = leading_whitespace(lines[start]) + for i in xrange(start + 1, len(lines)): + # If a blank line is encountered, we have finished parsing the "Args" + # section. + if lines[i].strip() and leading_whitespace(lines[i]) == args_leading_ws: + break + # In practice, users sometimes add a "-" prefix, so we strip it even + # though it is not recommended by the style guide + match = re.search(r"^\s*-?\s*(\w+):\s*(.*)", lines[i]) + if match: # We have found a new attribute + if var: + doc_attr[var] = escape(desc) + var, desc = match.group(1), match.group(2) + elif var: + # Merge documentation when it is multiline + desc = desc + "\n" + lines[i].strip() + + if var: + doc_attr[var] = escape(desc) + doc = "\n".join(lines[:start - 1]) + return doc, doc_attr diff --git a/skydoc/common_test.py b/skydoc/common_test.py new file mode 100644 index 0000000..71587c8 --- /dev/null +++ b/skydoc/common_test.py @@ -0,0 +1,85 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from skydoc import common + + +class CommonTest(unittest.TestCase): + """Unit tests for common functions.""" + + def test_rule_doc_only(self): + doc = '"""Rule documentation only docstring."""\n' + doc, attr_doc = common.parse_attribute_doc(doc) + self.assertEqual('Rule documentation only docstring.', doc) + self.assertDictEqual({}, attr_doc) + + def test_rule_and_attribute_doc(self): + doc = ( + '"""Rule and attribute documentation.\n' + '\n' + 'Args:\n' + ' name: A unique name for this rule.\n' + ' visibility: The visibility of this rule.\n' + '"""\n') + expected_attrs = { + 'name': 'A unique name for this rule.', + 'visibility': 'The visibility of this rule.' + } + + doc, attr_doc = common.parse_attribute_doc(doc) + self.assertEqual('Rule and attribute documentation.', doc) + self.assertDictEqual(expected_attrs, attr_doc) + + def test_multi_line_doc(self): + doc = ( + '"""Multi-line rule and attribute documentation.\n' + '\n' + 'Rule doc continued here.\n' + '\n' + 'Args:\n' + ' name: A unique name for this rule.\n' + '\n' + ' Documentation for name continued here.\n' + ' visibility: The visibility of this rule.\n' + '\n' + ' Documentation for visibility continued here.\n' + '"""\n') + expected_doc = ( + 'Multi-line rule and attribute documentation.\n' + 'Rule doc continued here.') + expected_attrs = { + 'name': ('A unique name for this rule.\n' + 'Documentation for name continued here'), + 'visibility': ('The visibility of this rule.\n' + 'Documentation for visibility continued here.') + } + + doc, attr_doc = common.parse_attribute_doc(doc) + self.assertEqual(expected_doc, doc) + self.assertDictEqual(expected_attrs, attr_doc) + + def test_invalid_args(self): + doc = ( + '"""Rule and attribute documentation.\n' + '\n' + 'Foo:\n' + ' name: A unique name for this rule.\n' + ' visibility: The visibility of this rule.\n' + '"""\n') + + doc, attr_doc = common.parse_attribute_doc(doc) + self.assertEqual('Rule and attribute documentation.', doc) + self.assertDictEqual({}, attr_doc) diff --git a/skydoc/macro_extractor.py b/skydoc/macro_extractor.py new file mode 100644 index 0000000..0b9db12 --- /dev/null +++ b/skydoc/macro_extractor.py @@ -0,0 +1,116 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Extractor for Skylark macro documentation.""" + +import ast + +from src.main.protobuf import build_pb2 +from skydoc import common + +def get_type(expr): + """Find the type of an expression. + + Args: + expr: The expression to check. + + Returns: + The type of the expression. + """ + if isinstance(expr, ast.Num): + return build_pb2.Attribute.INTEGER + elif isinstance(expr, ast.Str): + return build_pb2.Attribute.STRING + elif isinstance(expr, ast.List): + return build_pb2.Attribute.STRING_LIST + else: + return build_pb2.Attribute.UNKNOWN + +class MacroDocExtractor(object): + """Extracts documentation for macros from a .bzl file""" + + def __init__(self): + """Inits MacroDocExtractor with a new BuildLanguage proto""" + self.__language = build_pb2.BuildLanguage() + self.title = "" + self.description = "" + + def _add_file_docs(self, tree): + """Extracts the file docstring of the .bzl file.""" + docstring = ast.get_docstring(tree) + if docstring == None: + return + lines = docstring.split("\n") + i = 0 + for line in lines: + if line != '': + i = i + 1 + else: + break + + self.title = " ".join(lines[:i]) + self.description = "\n".join(lines[i + 1:]) + + def _add_macro_doc(self, stmt): + # The defaults array contains default values for the last arguments. + # The first shift arguments are mandatory. + shift = len(stmt.args.args) - len(stmt.args.defaults) + + rule = self.__language.rule.add() + rule.name = stmt.name + + doc = ast.get_docstring(stmt) + if doc: + doc, attr_doc = common.parse_attribute_doc(doc) + rule.documentation = doc.strip() + else: + doc = "" + attr_doc = {} + + for i in range(len(stmt.args.args)): + attr = rule.attribute.add() + attr_name = stmt.args.args[i].id + attr.name = attr_name + + if attr_name in attr_doc: + attr.documentation = attr_doc[attr_name] + + if i < shift: # The first arguments are mandatory + attr.mandatory = True + attr.type = build_pb2.Attribute.UNKNOWN + else: + attr.mandatory = False + attr.type = get_type(stmt.args.defaults[i - shift]) + + def parse_bzl(self, bzl_file): + """Extracts documentation for all public macros from the given .bzl file. + + Args: + bzl_file: The .bzl file to extract macro documentation from. + """ + try: + tree = ast.parse(open(bzl_file).read(), bzl_file) + self._add_file_docs(tree) + for stmt in tree.body: + if isinstance(stmt, ast.FunctionDef) and not stmt.name.startswith("_"): + self._add_macro_doc(stmt) + except IOError as e: + # Ignore missing extension + print("Failed to parse {0}: {1}".format(bzl_file, e.strerror)) + pass + + def proto(self): + """Returns the proto containing the macro documentation.""" + return self.__language + diff --git a/skydoc/macro_extractor_test.py b/skydoc/macro_extractor_test.py new file mode 100644 index 0000000..6bacea0 --- /dev/null +++ b/skydoc/macro_extractor_test.py @@ -0,0 +1,290 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import tempfile +import textwrap +import unittest +from google.protobuf import text_format +from skydoc import macro_extractor +from src.main.protobuf import build_pb2 + + +class MacroExtractorTest(unittest.TestCase): + + def check_protos(self, src, expected): + with tempfile.NamedTemporaryFile() as tf: + tf.write(src) + tf.flush() + + expected_proto = build_pb2.BuildLanguage() + text_format.Merge(expected, expected_proto) + + extractor = macro_extractor.MacroDocExtractor() + extractor.parse_bzl(tf.name) + proto = extractor.proto() + self.assertEqual(expected_proto, proto) + + def test_multi_line_description(self): + src = textwrap.dedent("""\ + def multiline(name, foo=False, visibility=None): + \"\"\"A rule with multiline documentation. + + Some more documentation about this rule here. + + Args: + name: A unique name for this rule. + foo: A test argument. + + Documentation for foo continued here. + visibility: The visibility of this rule. + + Documentation for visibility continued here. + \"\"\" + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "multiline" + documentation: "A rule with multiline documentation.\\n\\nSome more documentation about this rule here." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "foo" + type: UNKNOWN + mandatory: false + documentation: "A test argument.\\n\\nDocumentation for foo continued here." + } + attribute { + name: "visibility" + type: UNKNOWN + mandatory: false + documentation: "The visibility of this rule.\\n\\nDocumentation for visibility continued here." + } + } + """) + + self.check_protos(src, expected) + + def test_undocumented(self): + src = textwrap.dedent("""\ + def undocumented(name, visibility=None): + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "undocumented" + attribute { + name: "name" + type: UNKNOWN + mandatory: true + } + attribute { + name: "visibility" + type: UNKNOWN + mandatory: false + } + } + """) + + self.check_protos(src, expected) + + def test_private_macros_skipped(self): + src = textwrap.dedent("""\ + def _private(name, visibility=None): + \"\"\"A private macro that should not appear in docs. + + Args: + name: A unique name for this rule. + visibility: The visibility of this rule. + \"\"\" + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + + def public(name, visibility=None): + \"\"\"A public macro that should appear in docs. + + Args: + name: A unique name for this rule. + visibility: The visibility of this rule. + \"\"\" + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "public" + documentation: "A public macro that should appear in docs." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "visibility" + type: UNKNOWN + mandatory: false + documentation: "The visibility of this rule." + } + } + """) + + self.check_protos(src, expected) + + def test_rule_macro_mix(self): + src = textwrap.dedent("""\ + def _impl(ctx): + return struct() + + example_rule = rule( + implementation = _impl, + attrs = { + "arg_label": attr.label(), + "arg_string": attr.string(), + }, + ) + \"\"\"An example rule. + + Args: + name: A unique name for this rule. + arg_label: A label argument. + arg_string: A string argument. + \"\"\" + + def example_macro(name, foo, visibility=None): + \"\"\"An example macro. + + Args: + name: A unique name for this rule. + foo: A test argument. + visibility: The visibility of this rule. + \"\"\" + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "example_macro" + documentation: "An example macro." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "foo" + type: UNKNOWN + mandatory: true + documentation: "A test argument." + } + attribute { + name: "visibility" + type: UNKNOWN + mandatory: false + documentation: "The visibility of this rule." + } + } + """) + + self.check_protos(src, expected) + + def test_file_doc_title_only(self): + src = textwrap.dedent("""\ + \"\"\"Example rules\"\"\" + """) + with tempfile.NamedTemporaryFile() as tf: + tf.write(src) + tf.flush() + + extractor = macro_extractor.MacroDocExtractor() + extractor.parse_bzl(tf.name) + self.assertEqual('Example rules', extractor.title) + self.assertEqual('', extractor.description) + + def test_file_doc_title_description(self): + src = textwrap.dedent("""\ + \"\"\"Example rules + + This file contains example Bazel rules. + + Documentation continued here. + \"\"\" + """) + with tempfile.NamedTemporaryFile() as tf: + tf.write(src) + tf.flush() + + extractor = macro_extractor.MacroDocExtractor() + extractor.parse_bzl(tf.name) + self.assertEqual('Example rules', extractor.title) + self.assertEqual('This file contains example Bazel rules.' + '\n\nDocumentation continued here.', + extractor.description) + + def test_file_doc_title_multiline(self): + src = textwrap.dedent("""\ + \"\"\"Example rules + for Bazel + + This file contains example Bazel rules. + + Documentation continued here. + \"\"\" + """) + with tempfile.NamedTemporaryFile() as tf: + tf.write(src) + tf.flush() + + extractor = macro_extractor.MacroDocExtractor() + extractor.parse_bzl(tf.name) + self.assertEqual('Example rules for Bazel', extractor.title) + self.assertEqual('This file contains example Bazel rules.' + '\n\nDocumentation continued here.', + extractor.description) + +if __name__ == '__main__': + unittest.main() diff --git a/skydoc/main.py b/skydoc/main.py new file mode 100755 index 0000000..1ee7ca3 --- /dev/null +++ b/skydoc/main.py @@ -0,0 +1,236 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Documentation generator for Skylark""" + +import gflags +import jinja2 +import os +import re +import shutil +import sys +import tempfile +import zipfile + +from skydoc import macro_extractor +from skydoc import rule +from skydoc import rule_extractor +from src.main.protobuf import build_pb2 + +gflags.DEFINE_string('output_dir', '', + 'The directory to write the output generated documentation to if ' + '--zip=false') +gflags.DEFINE_string('output_file', '', + 'The output zip archive file to write if --zip=true.') +gflags.DEFINE_string('format', 'markdown', + 'The output format. Possible values are markdown and html') +gflags.DEFINE_bool('zip', True, + 'Whether to generate a ZIP arhive containing the output files. If ' + '--zip is true, then skydoc will generate a zip file, skydoc.zip by ' + 'default or as specified by --output_file. If --zip is false, then ' + 'skydoc will generate documentation, either in Markdown or HTML as ' + 'specifed by --format, in the current directory or --output_dir if set.') + +FLAGS = gflags.FLAGS + +DEFAULT_OUTPUT_DIR = '.' +DEFAULT_OUTPUT_FILE = 'skydoc.zip' + +TEMPLATE_PATH = 'skydoc/templates' +CSS_PATH = 'skydoc/sass' +CSS_FILE = 'main.css' +CSS_DIR = 'css' + + +# TODO(dzc): Remove this workaround once we switch to a self-contained Python +# binary format such as PEX. +def _runfile_path(path): + """Prepends the given path with the path to the root of the runfiles tree. + + The files that skydoc depend on are generated in the Bazel runfiles tree. + There is no built-in way to get the root of the runfiles tree but it is + possible to generate it from sys.argv[0]. The code here is adapted from the + Bazel Python launcher stub. + + Args: + path: The relative path from the root of the runfiles tree. + + Returns: + Returns path prepended with the absolute path to the root of the runfiles + tree. + """ + script_filename = os.path.abspath(sys.argv[0]) + while True: + runfiles_dir = script_filename + '.runfiles' + if os.path.isdir(runfiles_dir): + break + + # Follow a symlink and try again. + if os.path.islink(script_filename): + link = os.readlink(script_filename) + script_filename = os.path.join(os.path.dirname(script_filename), link) + continue + + matchobj = re.match("(.*\.runfiles)/.*", os.path.abspath(sys.argv[0])) + if matchobj: + runfiles_dir = matchobj.group(1) + break + + raise AssertionError('Cannot find .runfiles directory.') + return os.path.join(runfiles_dir, path) + +def merge_languages(macro_language, rule_language): + for rule in rule_language.rule: + new_rule = macro_language.rule.add() + new_rule.CopyFrom(rule) + return macro_language + +class MarkdownWriter(object): + """Writer for generating documentation in Markdown.""" + + def __init__(self, output_dir, output_file, output_zip): + self.__output_dir = output_dir + self.__output_file = output_file + self.__output_zip = output_zip + + def write(self, rulesets): + """Write the documentation for the rules contained in rulesets.""" + try: + temp_dir = tempfile.mkdtemp() + output_files = [] + for ruleset in rulesets: + output_files.append(self._write_ruleset(temp_dir, ruleset)) + + if self.__output_zip: + # We are generating a zip archive containing all the documentation. + # Write each documentation file generated in the temp directory to the + # zip file. + with zipfile.ZipFile(self.__output_file, 'w') as zf: + for output_file in output_files: + zf.write(output_file, os.path.basename(output_file)) + else: + # We are generating documentation in the output_dir directory. Copy each + # documentation file to output_dir. + for output_file in output_files: + shutil.copyfile( + output_file, + os.path.join(self.__output_dir, os.path.basename(output_file))) + + finally: + # Delete temporary directory. + shutil.rmtree(temp_dir) + + def _write_ruleset(self, output_dir, ruleset): + # Load template and render Markdown. + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(_runfile_path(TEMPLATE_PATH))) + template = env.get_template('markdown.jinja') + out = template.render(ruleset=ruleset) + + # Write output to file. + output_file = "%s/%s.md" % (output_dir, ruleset.name) + with open(output_file, "w") as f: + f.write(out) + return output_file + +class HtmlWriter(object): + """Writer for generating documentation in HTML.""" + + def __init__(self, output_dir, output_file, output_zip): + self.__output_dir = output_dir + self.__output_file = output_file + self.__output_zip = output_zip + self.__env = jinja2.Environment( + loader=jinja2.FileSystemLoader(_runfile_path(TEMPLATE_PATH))) + + def write(self, rulesets): + # Generate navigation used for all rules. + nav_template = self.__env.get_template('nav.jinja') + nav = nav_template.render(rulesets=rulesets) + + try: + temp_dir = tempfile.mkdtemp() + output_files = [] + for ruleset in rulesets: + output_files.append(self._write_ruleset(temp_dir, ruleset, nav)) + + if self.__output_zip: + with zipfile.ZipFile(self.__output_file, 'w') as zf: + for output_file in output_files: + zf.write(output_file, os.path.basename(output_file)) + zf.write(os.path.join(_runfile_path(CSS_PATH), CSS_FILE), + 'css/%s' % CSS_FILE) + else: + for output_file in output_files: + shutil.copyfile( + output_file, + os.path.join(self.__output_dir, os.path.basename(output_file))) + + # Copy CSS file. + css_dir = os.path.join(self.__output_dir, CSS_DIR) + os.mkdir(css_dir) + shutil.copyfile(os.path.join(_runfile_path(CSS_PATH), CSS_FILE), + os.path.join(css_dir, CSS_FILE)) + finally: + # Delete temporary directory. + shutil.rmtree(temp_dir) + + def _write_ruleset(self, output_dir, ruleset, nav): + # Load template and render markdown. + template = self.__env.get_template('html.jinja') + out = template.render(ruleset=ruleset, nav=nav) + + # Write output to file. + output_file = "%s/%s.html" % (output_dir, ruleset.name) + with open(output_file, "w") as f: + f.write(out) + return output_file + +def main(argv): + if FLAGS.output_dir and FLAGS.output_file: + sys.stderr.write('Only one of --output_dir or --output_file can be set.') + sys.exit(1) + + if not FLAGS.output_dir: + FLAGS.output_dir = DEFAULT_OUTPUT_DIR + if not FLAGS.output_file: + FLAGS.output_file = DEFAULT_OUTPUT_FILE + + rulesets = [] + for bzl_file in argv[1:]: + macro_doc_extractor = macro_extractor.MacroDocExtractor() + rule_doc_extractor = rule_extractor.RuleDocExtractor() + macro_doc_extractor.parse_bzl(bzl_file) + rule_doc_extractor.parse_bzl(bzl_file) + merged_language = merge_languages(macro_doc_extractor.proto(), + rule_doc_extractor.proto()) + file_basename = os.path.basename(bzl_file) + rulesets.append(rule.RuleSet(file_basename, merged_language, + macro_doc_extractor.title, + macro_doc_extractor.description)) + + if FLAGS.format == "markdown": + markdown_writer = MarkdownWriter(FLAGS.output_dir, FLAGS.output_file, + FLAGS.zip) + markdown_writer.write(rulesets) + elif FLAGS.format == "html": + html_writer = HtmlWriter(FLAGS.output_dir, FLAGS.output_file, FLAGS.zip) + html_writer.write(rulesets) + else: + sys.stderr.write( + 'Invalid output format: %s. Possible values are markdown and html' + % FLAGS.format) + +if __name__ == '__main__': + main(FLAGS(sys.argv)) diff --git a/skydoc/rule.py b/skydoc/rule.py new file mode 100644 index 0000000..abf802e --- /dev/null +++ b/skydoc/rule.py @@ -0,0 +1,120 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Representations used for rendering documentation templates.""" + +import mistune +from src.main.protobuf import build_pb2 + +class Attribute(object): + """Representation of an attribute used to render documentation templates.""" + + NAME_LINK = 'Name' + LABEL_LINK = 'Label' + LABELS_LINK = ( + 'labels') + + def __init__(self, proto): + self.__proto = proto + self.name = proto.name + self.type = self._get_type_str(proto) + if proto.name == 'name' and not proto.documentation: + self.documentation = 'A unique name for this rule.' + else: + self.documentation = mistune.markdown(proto.documentation) + + def _get_type_str(self, proto): + type_str = '' + if proto.type == build_pb2.Attribute.INTEGER: + type_str = 'Integer' + elif proto.type == build_pb2.Attribute.STRING: + type_str = 'String' + elif proto.type == build_pb2.Attribute.LABEL: + type_str = self.LABEL_LINK + elif proto.type == build_pb2.Attribute.OUTPUT: + type_str = 'Output' + elif proto.type == build_pb2.Attribute.STRING_LIST: + type_str = 'List of strings' + elif proto.type == build_pb2.Attribute.LABEL_LIST: + type_str = 'List of %s' % self.LABELS_LINK + elif proto.type == build_pb2.Attribute.OUTPUT_LIST: + type_str = 'List of outputs' + elif proto.type == build_pb2.Attribute.DISTRIBUTION_SET: + type_str = 'Distribution Set' + elif proto.type == build_pb2.Attribute.LICENSE: + type_str = 'License' + elif proto.type == build_pb2.Attribute.STRING_DICT: + type_str = 'Dictionary mapping strings to string' + elif proto.type == build_pb2.Attribute.FILESET_ENTRY_LIST: + type_str = 'List of FilesetEntry' + elif proto.type == build_pb2.Attribute.LABEL_LIST_DICT: + type_str = 'Dictionary mapping strings to lists of %s' % self.LABELS_LINK + elif proto.type == build_pb2.Attribute.STRING_LIST_DICT: + type_str = 'Dictionary mapping strings to lists of strings' + elif proto.type == build_pb2.Attribute.BOOLEAN: + type_str = 'Boolean' + elif proto.type == build_pb2.Attribute.TRISTATE: + type_str = 'Tristate' + elif proto.type == build_pb2.Attribute.INTEGER_LIST: + type_str = 'List of integers' + elif proto.type == build_pb2.Attribute.STRING_DICT_UNARY: + type_str = 'String Dict Unary' + elif proto.type == build_pb2.Attribute.LABEL_DICT_UNARY: + type_str = 'Label Dict Unary' + elif proto.type == build_pb2.Attribute.SELECTOR_LIST: + type_str = 'Selector List' + else: + if proto.name == 'name': + type_str = self.NAME_LINK + else: + type_str = 'Unknown' + type_str += '; Required' if proto.mandatory else '; Optional' + return type_str + +class Rule(object): + """Representation of a rule used to render documentation templates.""" + + def __init__(self, proto): + self.__proto = proto + self.name = proto.name + self.documentation = proto.documentation + self.signature = self._get_signature(proto) + self.attributes = [] + for attribute in proto.attribute: + self.attributes.append(Attribute(attribute)) + + def _get_signature(self, proto): + """Returns the rule signature for this rule.""" + signature = proto.name + '(' + for i in range(len(proto.attribute)): + attr = proto.attribute[i] + signature += '%s' % (proto.name, attr.name, + attr.name) + if i < len(proto.attribute) - 1: + signature += ', ' + signature += ')' + return signature + +class RuleSet(object): + """Representation of a rule set used to render documentation templates.""" + + def __init__(self, file_name, language, title, description): + self.file_name = file_name + self.name = file_name.replace('.bzl', '') + self.language = language + self.title = title if title else "%s Rules" % self.name + self.description = description + self.rules = [] + for rule_proto in language.rule: + self.rules.append(Rule(rule_proto)) diff --git a/skydoc/rule_extractor.py b/skydoc/rule_extractor.py new file mode 100644 index 0000000..cce1551 --- /dev/null +++ b/skydoc/rule_extractor.py @@ -0,0 +1,164 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Extractor for Skylark rule documentation.""" + +import ast + +from src.main.protobuf import build_pb2 +from skydoc import common +from skydoc.stubs import attr +from skydoc.stubs import skylark_globals + +SKYLARK_STUBS = { + "attr": attr, + "aspect": skylark_globals.aspect, + "DATA_CFG": skylark_globals.DATA_CFG, + "HOST_CFG": skylark_globals.HOST_CFG, + "PACKAGE_NAME": skylark_globals.PACKAGE_NAME, + "REPOSITORY_NAME": skylark_globals.REPOSITORY_NAME, + "provider": skylark_globals.provider, + "FileType": skylark_globals.FileType, + "Label": skylark_globals.Label, + "select": skylark_globals.select, + "struct": skylark_globals.struct, + "repository_rule": skylark_globals.repository_rule, + "rule": skylark_globals.rule, +} +"""Stubs for Skylark globals to be used to evaluate the .bzl file.""" + +class RuleDocExtractor(object): + """Extracts documentation for rules from a .bzl file.""" + + def __init__(self): + """Inits RuleDocExtractor with a new BuildLanguage proto""" + self.__language = build_pb2.BuildLanguage() + self.__extracted_rules = {} + + def _process_skylark(self, bzl_file): + """Evaluates the Skylark code in the .bzl file. + + This function evaluates the Skylark code in the .bzl file as Python against + Skylark stubs to extract the rules and attributes defined in the file. The + extracted rules are kept in the __extracted_rules map keyed by rule name. + + Args: + bzl_file: The .bzl file to evaluate. + """ + skylark_locals = {} + compiled = compile(open(bzl_file).read(), bzl_file, "exec") + exec(compiled) in SKYLARK_STUBS, skylark_locals + + for name, obj in skylark_locals.iteritems(): + if hasattr(obj, "is_rule") and not name.startswith("_"): + obj.attrs["name"] = attr.AttrDescriptor( + type=build_pb2.Attribute.UNKNOWN, mandatory=True, name="name") + self.__extracted_rules[name] = obj + + def _add_rule_doc(self, name, doc): + """Parses the attribute documentation from the docstring. + + Parses the attribute documentation in the given docstring and associates the + rule and attribute documentation with the corresponding rule extracted from + the .bzl file. + + Args: + name: The name of the rule. + doc: The docstring extracted for the rule. + """ + doc, attr_doc = common.parse_attribute_doc(doc) + if name in self.__extracted_rules: + rule = self.__extracted_rules[name] + rule.doc = doc.strip() + for attr_name, attr_doc in attr_doc.iteritems(): + if attr_name in rule.attrs: + rule.attrs[attr_name].doc = attr_doc + + def _parse_docstrings(self, bzl_file): + """Extracts the docstrings for all public rules in the .bzl file. + + This function parses the .bzl file and extracts the docstrings for all + public rules in the file that were extracted in _process_skylark. It calls + _add_rule_doc for to parse the attribute documentation in each docstring + and associate them with the extracted rules and attributes. + + Args: + bzl_file: The .bzl file to extract docstrings from. + """ + try: + tree = ast.parse(open(bzl_file).read(), bzl_file) + key = None + for node in ast.iter_child_nodes(tree): + if isinstance(node, ast.Assign): + name = node.targets[0].id + if not name.startswith("_"): + key = name + continue + elif isinstance(node, ast.Expr) and key: + self._add_rule_doc(key, node.value.s.strip()) + key = None + except IOError: + print("Failed to parse {0}: {1}".format(bzl_file, e.strerror)) + pass + + def _assemble_protos(self): + """Builds the BuildLanguage protos for the extracted rule documentation. + + Iterates through the map of extracted rule documentation and builds a + BuildLanguage proto containing the documentation for publid rules extracted + from the .bzl file. + """ + rules = [] + for rule_name, rule_desc in self.__extracted_rules.iteritems(): + rule_desc.name = rule_name + rules.append(rule_desc) + rules = sorted(rules, key=lambda rule_desc: rule_desc.name) + + for rule_desc in rules: + rule = self.__language.rule.add() + rule.name = rule_desc.name + rule.documentation = rule_desc.doc + + attrs = sorted(rule_desc.attrs.values(), cmp=attr.attr_compare) + for attr_desc in attrs: + if attr_desc.name.startswith("_"): + continue + attr_proto = rule.attribute.add() + attr_proto.name = attr_desc.name + attr_proto.documentation = attr_desc.doc + attr_proto.type = attr_desc.type + attr_proto.mandatory = attr_desc.mandatory + # TODO(dzc): Save the default value of the attribute. This will require + # adding a proto field to the AttributeDefinition proto, perhaps as a + # oneof. + + def parse_bzl(self, bzl_file): + """Extracts the documentation for all public rules from the given .bzl file. + + The Skylark code is first evaluated against stubs to extract rule and + attributes with complete type information. Then, the .bzl file is parsed + to extract the docstrings for each of the rules. Finally, the BuildLanguage + proto is assembled with the extracted rule documentation. + + Args: + bzl_file: The .bzl file to extract rule documentation from. + """ + self._process_skylark(bzl_file) + self._parse_docstrings(bzl_file) + self._assemble_protos() + + def proto(self): + """Returns the proto containing the macro documentation.""" + return self.__language + diff --git a/skydoc/rule_extractor_test.py b/skydoc/rule_extractor_test.py new file mode 100644 index 0000000..61d979a --- /dev/null +++ b/skydoc/rule_extractor_test.py @@ -0,0 +1,393 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import os +import tempfile +import textwrap +from google.protobuf import text_format +from skydoc import rule_extractor +from src.main.protobuf import build_pb2 + + +class RuleExtractorTest(unittest.TestCase): + + def check_protos(self, src, expected): + with tempfile.NamedTemporaryFile() as tf: + tf.write(src) + tf.flush() + + expected_proto = build_pb2.BuildLanguage() + text_format.Merge(expected, expected_proto) + + extractor = rule_extractor.RuleDocExtractor() + extractor.parse_bzl(tf.name) + proto = extractor.proto() + self.assertEqual(expected_proto, proto) + + def test_all_types(self): + src = textwrap.dedent("""\ + def impl(ctx): + return struct() + + all_types = rule( + implementation = impl, + attrs = { + "arg_bool": attr.bool(), + "arg_int": attr.int(), + "arg_int_list": attr.int_list(), + "arg_label": attr.label(), + "arg_label_list": attr.label_list(), + "arg_license": attr.license(), + "arg_output": attr.output(), + "arg_output_list": attr.output_list(), + "arg_string": attr.string(), + "arg_string_dict": attr.string_dict(), + "arg_string_list": attr.string_list(), + "arg_string_list_dict": attr.string_list_dict(), + }, + ) + \"\"\"Test rule with all types. + + Args: + name: A unique name for this rule. + arg_bool: A boolean argument. + arg_int: An integer argument. + arg_int_list: A list of integers argument. + arg_label: A label argument. + arg_label_list: A list of labels argument. + arg_license: A license argument. + arg_output: An output argument. + arg_output_list: A list of outputs argument. + arg_string: A string argument. + arg_string_dict: A dictionary mapping string to string argument. + arg_string_list: A list of strings argument. + arg_string_list_dict: A dictionary mapping string to list of string argument. + \"\"\" + """) + + expected = textwrap.dedent(""" + rule { + name: "all_types" + documentation: "Test rule with all types." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "arg_bool" + type: BOOLEAN + mandatory: false + documentation: "A boolean argument." + } + attribute { + name: "arg_int" + type: INTEGER + mandatory: false + documentation: "An integer argument." + } + attribute { + name: "arg_int_list" + type: INTEGER_LIST + mandatory: false + documentation: "A list of integers argument." + } + attribute { + name: "arg_label" + type: LABEL + mandatory: false + documentation: "A label argument." + } + attribute { + name: "arg_label_list" + type: LABEL_LIST + mandatory: false + documentation: "A list of labels argument." + } + attribute { + name: "arg_license" + type: LICENSE + mandatory: false + documentation: "A license argument." + } + attribute { + name: "arg_output" + type: OUTPUT + mandatory: false + documentation: "An output argument." + } + attribute { + name: "arg_output_list" + type: OUTPUT_LIST + mandatory: false + documentation: "A list of outputs argument." + } + attribute { + name: "arg_string" + type: STRING + mandatory: false + documentation: "A string argument." + } + attribute { + name: "arg_string_dict" + type: STRING_DICT + mandatory: false + documentation: "A dictionary mapping string to string argument." + } + attribute { + name: "arg_string_list" + type: STRING_LIST + mandatory: false + documentation: "A list of strings argument." + } + attribute { + name: "arg_string_list_dict" + type: STRING_LIST_DICT + mandatory: false + documentation: "A dictionary mapping string to list of string argument." + } + } + """) + + self.check_protos(src, expected) + + def test_undocumented(self): + src = textwrap.dedent("""\ + def _impl(ctx): + return struct() + + undocumented = rule( + implementation = _impl, + attrs = { + "arg_label": attr.label(), + "arg_string": attr.string(), + }, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "undocumented" + documentation: "" + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "" + } + attribute { + name: "arg_label" + type: LABEL + mandatory: false + documentation: "" + } + attribute { + name: "arg_string" + type: STRING + mandatory: false + documentation: "" + } + } + """) + + self.check_protos(src, expected) + + def test_private_rules_skipped(self): + src = textwrap.dedent("""\ + def _private_impl(ctx): + return struct() + + def _public_impl(ctx): + return struct() + + _private = rule( + implementation = _private_impl, + attrs = { + "arg_label": attr.label(), + "arg_string": attr.string(), + }, + ) + \"\"\"A private rule that should not appear in documentation. + + Args: + name: A unique name for this rule. + arg_label: A label argument. + arg_string: A string argument. + \"\"\" + + public = rule( + implementation = _public_impl, + attrs = { + "arg_label": attr.label(), + "arg_string": attr.string(), + }, + ) + \"\"\"A public rule that should appear in documentation. + + Args: + name: A unique name for this rule. + arg_label: A label argument. + arg_string: A string argument. + \"\"\" + """) + + expected = textwrap.dedent(""" + rule { + name: "public" + documentation: "A public rule that should appear in documentation." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "arg_label" + type: LABEL + mandatory: false + documentation: "A label argument." + } + attribute { + name: "arg_string" + type: STRING + mandatory: false + documentation: "A string argument." + } + } + """) + + self.check_protos(src, expected) + + def test_multi_line_description(self): + src = textwrap.dedent("""\ + def _impl(ctx): + return struct() + + multiline = rule( + implementation = _impl, + attrs = { + "arg_bool": attr.bool(), + "arg_label": attr.label(), + }, + ) + \"\"\"A rule with multiline documentation. + + Some more documentation about this rule here. + + Args: + name: A unique name for this rule. + arg_bool: A boolean argument. + + Documentation for arg_bool continued here. + arg_label: A label argument. + + Documentation for arg_label continued here. + \"\"\" + """) + + expected = textwrap.dedent("""\ + rule { + name: "multiline" + documentation: "A rule with multiline documentation.\\n\\nSome more documentation about this rule here." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "arg_bool" + type: BOOLEAN + mandatory: false + documentation: "A boolean argument.\\n\\nDocumentation for arg_bool continued here." + } + attribute { + name: "arg_label" + type: LABEL + mandatory: false + documentation: "A label argument.\\n\\nDocumentation for arg_label continued here." + } + } + """) + + self.check_protos(src, expected) + + def test_rule_macro_mix(self): + src = textwrap.dedent("""\ + def _impl(ctx): + return struct() + + example_rule = rule( + implementation = _impl, + attrs = { + "arg_label": attr.label(), + "arg_string": attr.string(), + }, + ) + \"\"\"An example rule. + + Args: + name: A unique name for this rule. + arg_label: A label argument. + arg_string: A string argument. + \"\"\" + + def example_macro(name, foo, visibility=None): + \"\"\"An example macro. + + Args: + name: A unique name for this rule. + foo: A test argument. + visibility: The visibility of this rule. + \"\"\" + native.genrule( + name = name, + out = ["foo"], + cmd = "touch $@", + visibility = visibility, + ) + """) + + expected = textwrap.dedent("""\ + rule { + name: "example_rule" + documentation: "An example rule." + attribute { + name: "name" + type: UNKNOWN + mandatory: true + documentation: "A unique name for this rule." + } + attribute { + name: "arg_label" + type: LABEL + mandatory: false + documentation: "A label argument." + } + attribute { + name: "arg_string" + type: STRING + mandatory: false + documentation: "A string argument." + } + } + """) + + self.check_protos(src, expected) + +if __name__ == '__main__': + unittest.main() diff --git a/skydoc/sass/BUILD b/skydoc/sass/BUILD new file mode 100644 index 0000000..8b5adbc --- /dev/null +++ b/skydoc/sass/BUILD @@ -0,0 +1,8 @@ +package(default_visibility = ["//skydoc:__pkg__"]) + +load("@io_bazel_rules_sass//sass:sass.bzl", "sass_binary") + +sass_binary( + name = "main", + src = "main.scss", +) diff --git a/skydoc/sass/main.scss b/skydoc/sass/main.scss new file mode 100644 index 0000000..8cf8de4 --- /dev/null +++ b/skydoc/sass/main.scss @@ -0,0 +1,240 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +$primary-color: #4caf50; + +body { + background-color: #fafafa; +} + +pre, +code { + font-family: 'Liberation Mono', Consolas, Monaco, 'Andale Mono', monospace; +} + +pre { + background-color: #eee; + padding: 20px; + overflow-x: auto; + word-wrap: normal; + + code { + overflow-wrap: normal; + white-space: pre; + } +} + +code { + display: inline-block; + font-size: 90%; + white-space: pre-wrap; +} + +.mdl-layout__drawer { + background-color: #fff; + + .mdl-layout-title { + border-bottom: 1px solid #e0e0e0; + padding-left: 24px; + } +} + +.drawer-nav { + ul { + list-style: none; + padding-left: 0; + + li { + display: block; + padding: 0; + + ul { + li { + a { + padding-left: 44px; + font-weight: 400; + } + } + } + + a { + display: block; + flex-shrink: 0; + padding: 15px 0 15px 22px; + margin: 0; + font-weight: 600; + color: #757575; + line-height: 1em; + text-decoration: none; + cursor: pointer; + + &:active, + &:hover { + background-color: #f0f0f0; + } + } + + &.active { + a { + color: $primary-color; + font-weight: 500; + } + } + } + } +} + +h1.page-title { + font-size: 34px; + font-weight: 400; + line-height: 40px; + margin-bottom: 30px; + color: $primary-color; +} + +p.lead { + font-size: 20px; + line-height: 32px; +} + +table { + border-collapse: collapse; + border-spacing: 0; + background-color: #fff; + table-layout: auto; + + thead { + th { + background-color: #fafafa; + border: 1px solid #eee; + color: #757575; + padding: 12px 12px 12px 24px; + vertical-align: top; + } + } + + tbody { + td { + border: 1px solid #eee; + padding: 12px 12px 12px 24px; + vertical-align: top; + } + } +} + +table.params-table { + width: 100%; + + col.col-param{ + width: 25%; + } + + col.col-description { + width: 75%; + } +} + +hr { + margin-top: 80px; + margin-bottom: 80px; +} + +nav.toc { + border-left: 5px solid $primary-color; + padding-left: 20px; + margin-bottom: 48px; + + h1, + h2 { + font-size: 15px; + line-height: 16px; + padding-bottom: 12px; + margin-bottom: 0; + font-weight: 400; + color: #757575; + } + + ul { + list-style: none; + margin-top: 0; + padding-left: 0; + + li { + font-size: 20px; + line-height: 40px; + + a { + color: $primary-color; + } + } + } +} + +.page-content { + margin-left: auto; + margin-right: auto; + padding-top: 60px; + padding-bottom: 60px; + width: 760px; + + a { + text-decoration: none; + } + + h1 { + font-size: 34px; + font-weight: 400; + line-height: 40px; + margin-bottom: 30px; + color: $primary-color; + } + + h2 { + font-size: 24px; + font-weight: 400; + line-height: 32px; + margin-bottom: 30px; + color: $primary-color; + } + + h3 { + font-size: 20px; + font-weight: 400; + line-height: 32px; + margin-bottom: 30px; + color: $primary-color; + } +} + +@media (max-width: 768px) { + .page-content { + width: 360px; + } +} + +@media (min-width: 768px) { + .page-content { + width: 760px; + } +} + +@media (min-width: 1476px) { + .page-content { + width: 1160px; + } +} + +.mdl-mini-footer { + padding-left: 40px; +} diff --git a/skydoc/stubs/BUILD b/skydoc/stubs/BUILD new file mode 100644 index 0000000..85907c8 --- /dev/null +++ b/skydoc/stubs/BUILD @@ -0,0 +1,12 @@ +package(default_visibility = ["//skydoc:__pkg__"]) + +py_library( + name = "stubs", + srcs = [ + "attr.py", + "skylark_globals.py", + ], + deps = [ + "@bazel_tools//src/main/protobuf:build_pb_py", + ], +) diff --git a/skydoc/stubs/attr.py b/skydoc/stubs/attr.py new file mode 100644 index 0000000..4db645f --- /dev/null +++ b/skydoc/stubs/attr.py @@ -0,0 +1,112 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from src.main.protobuf import build_pb2 + +def strcmp(s1, s2): + if s1 > s2: + return 1 + elif s1 < s2: + return -1 + else: + return 0 + +def attr_compare(a, b): + if a.compare_priority() > b.compare_priority(): + return 1 + elif a.compare_priority() < b.compare_priority(): + return -1 + else: + return strcmp(a.name, b.name) + +class AttrDescriptor(object): + ATTRIBUTE_ORDERING = { + "name": -99, + "deps": -98, + "src": -97, + "srcs": -86, + "data": -95, + "resource": -94, + "resources": -93, + "out": -92, + "outs": -91, + "hdrs": -90, + } + + def __init__(self, type=build_pb2.Attribute.UNKNOWN, default=None, + mandatory=False, doc="", name=""): + """Constructor for AttrDescriptor + + Args: + self: The current instance + type: The type of attribute based on the enum in the Attribute proto. + default: The default value of the attribute. + mandatory: True if the attribute is required, false if optional. + doc: Documentation for this attribute. This parameter is used internally + by skydoc and is not set by any Skylark code in .bzl files. + name: Name of this attribute. This parameter is used internally by skydoc + and is not set by any Skylark code in .bzl files. + """ + self.type = type + self.default = default + self.mandatory = mandatory + self.doc = doc + self.name = name + + def compare_priority(self): + if self.name in AttrDescriptor.ATTRIBUTE_ORDERING: + return AttrDescriptor.ATTRIBUTE_ORDERING[self.name] + else: + return 0 + +def bool(default=False, mandatory=False): + return AttrDescriptor(build_pb2.Attribute.BOOLEAN, default=default, + mandatory=mandatory) + +def int(default=0, mandatory=False, values=[]): + return AttrDescriptor(build_pb2.Attribute.INTEGER, default, mandatory) + +def int_list(default=[], mandatory=False, non_empty=False): + return AttrDescriptor(build_pb2.Attribute.INTEGER_LIST, default, mandatory) + +def label(default=None, executable=False, allow_files=False, mandatory=False, + providers=[], allow_rules=None, single_file=False, cfg=None): + return AttrDescriptor(build_pb2.Attribute.LABEL, default, mandatory) + +def label_list(default=[], allow_files=False, allow_rules=None, providers=[], + flags=[], mandatory=False, non_empty=False, cfg=None, + aspects=[]): + return AttrDescriptor(build_pb2.Attribute.LABEL_LIST, default, mandatory) + +def license(default=None, mandatory=False): + return AttrDescriptor(build_pb2.Attribute.LICENSE, default, mandatory) + +def output(default=None, mandatory=False): + return AttrDescriptor(build_pb2.Attribute.OUTPUT, default, mandatory) + +def output_list(default=[], mandatory=False, non_empty=False): + return AttrDescriptor(build_pb2.Attribute.OUTPUT_LIST, default, mandatory) + +def string(default='', mandatory=False, values=[]): + return AttrDescriptor(build_pb2.Attribute.STRING, default, mandatory) + +def string_dict(default={}, mandatory=False, non_empty=False): + return AttrDescriptor(build_pb2.Attribute.STRING_DICT, default, mandatory) + +def string_list(default=[], mandatory=False, non_empty=False): + return AttrDescriptor(build_pb2.Attribute.STRING_LIST, default, mandatory) + +def string_list_dict(default={}, mandatory=False, non_empty=False): + return AttrDescriptor(build_pb2.Attribute.STRING_LIST_DICT, default, + mandatory) diff --git a/skydoc/stubs/skylark_globals.py b/skydoc/stubs/skylark_globals.py new file mode 100644 index 0000000..193f419 --- /dev/null +++ b/skydoc/stubs/skylark_globals.py @@ -0,0 +1,90 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stubs for Skylark globals""" + + +def FileType(filetypes=[]): + return filetypes + +HOST_CFG = 'HOST_CFG' +DATA_CFG = 'DATA_CFG' +PACKAGE_NAME = 'PACKAGE_NAME' +REPOSITORY_NAME = 'REPOSITORY_NAME' + +def aspect(implementation, attr_aspects=[], attrs=None, fragments=[], + host_fragments=[]): + return None + +def provider(target, type): + return None + +def select(x): + return None + +def struct(**kwargs): + return None + +class Label(object): + def __init__(self, label_string): + self.label_string = label_string + +class RuleDescriptor(object): + def __init__(self, implementation, test=False, attrs={}, outputs=None, + executable=False, output_to_genfiles=False, fragments=[], + host_fragments=[], local=False, doc='', type='rule'): + """Constructor for RuleDescriptor + + Args: + self: The current instance. + implementation: The implementation function for the rule (not used). + test: Whether this is a test rule + attrs: Dictionary mapping attribute name to attribute descriptor + outputs: List of outputs (not used). + executable: Whether this rule produces an executable. + output_to_genfiles: Whether the rule generates files in the genfiles + directory rather than the bin directory (not used). + fragments: List of names of configuration fragments the rule requires in + the target configuration (not used). + host_fragments: List of names of configuration fragments that the rule + requires in the host configuration (not used). + local: Indicates that the rule fetches everything from the local system. + (Only used if type='repository'). + doc: Documentation for this rule. This parameter is used internally by + skydoc and is not set by any Skylark code in .bzl files. + """ + self.is_rule = True + self.implementation = implementation + self.test = test + self.attrs = attrs + self.outputs = outputs + self.executable = executable + self.output_to_genfiles = output_to_genfiles + self.fragments = fragments + self.host_fragments = host_fragments + self.local = local + self.doc = doc + self.type = type + for name, attr in self.attrs.iteritems(): + attr.name = name + +def rule(implementation, test=False, attrs={}, outputs=None, + executable=False, output_to_genfiles=False, fragments=[], + host_fragments=[]): + return RuleDescriptor(implementation, test, attrs, outputs, executable, + output_to_genfiles, fragments, host_fragments) + +def repository_rule(implementation, attrs={}, local=False): + return RuleDescriptor(implementation, attrs=attrs, local=local, + type='repository') diff --git a/skydoc/templates/BUILD b/skydoc/templates/BUILD new file mode 100644 index 0000000..ba76e9f --- /dev/null +++ b/skydoc/templates/BUILD @@ -0,0 +1,13 @@ +package(default_visibility = ["//skydoc:__pkg__"]) + +filegroup( + name = "templates", + srcs = [ + "attributes.jinja", + "html.jinja", + "markdown.jinja", + "nav.jinja", + "overview.jinja", + "toc.jinja", + ], +) diff --git a/skydoc/templates/attributes.jinja b/skydoc/templates/attributes.jinja new file mode 100644 index 0000000..bc96070 --- /dev/null +++ b/skydoc/templates/attributes.jinja @@ -0,0 +1,32 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + + + + + + +{% for attribute in rule.attributes %} + + + + +{% endfor %} + +
{{ attribute.name }} +

{{ attribute.type }}

+ {{ attribute.documentation }} +
diff --git a/skydoc/templates/html.jinja b/skydoc/templates/html.jinja new file mode 100644 index 0000000..3ef4cb3 --- /dev/null +++ b/skydoc/templates/html.jinja @@ -0,0 +1,88 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + + + + + + + + + {{ ruleset.title }} + + + + + + + + +
+
+
+ {{ ruleset.title }} +
+
+
+ Bazel + +
+ +
+
+

{{ ruleset.title }}

+ +{% include "toc.jinja" %} +{% include "overview.jinja" %} + +{% for rule in ruleset.rules %} +
+ +

{{ rule.name }}

+ + 
{{ rule.signature }}
+ + {{ rule.documentation }} + +{% if rule.attributes[0] is defined %} +

Attributes

+ +{% include "attributes.jinja" %} +{% endif %} + +{% endfor %} +
+ + +
+
+ + diff --git a/skydoc/templates/markdown.jinja b/skydoc/templates/markdown.jinja new file mode 100644 index 0000000..5c0ca80 --- /dev/null +++ b/skydoc/templates/markdown.jinja @@ -0,0 +1,42 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +

{{ ruleset.title }}

+ +{% include "toc.jinja" %} +{% include "overview.jinja" %} + +{% for rule in ruleset.rules %} +
+ + +## {{ rule.name }} + +
+{{ rule.signature }}
+
+ +{{ rule.documentation }} + +{% if rule.attributes[0] is defined %} + +### Attributes + +{% include "attributes.jinja" %} +{% endif %} +{% endfor %} diff --git a/skydoc/templates/nav.jinja b/skydoc/templates/nav.jinja new file mode 100644 index 0000000..9b367a6 --- /dev/null +++ b/skydoc/templates/nav.jinja @@ -0,0 +1,28 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +{% for ruleset in rulesets %} +
  • + {{ ruleset.name }} +
      +{% if ruleset.description %} +
    • Overview
      • +{% endif %} +{% for rule in ruleset.rules %} +
    • {{ rule.name }}
      • +{% endfor %} +
    +
    • +{% endfor %} diff --git a/skydoc/templates/overview.jinja b/skydoc/templates/overview.jinja new file mode 100644 index 0000000..f6a9ba5 --- /dev/null +++ b/skydoc/templates/overview.jinja @@ -0,0 +1,22 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} +{% if ruleset.description %} +
    + +

    Overview

    + +{{ ruleset.description }} +{% endif %} diff --git a/skydoc/templates/toc.jinja b/skydoc/templates/toc.jinja new file mode 100644 index 0000000..b420d94 --- /dev/null +++ b/skydoc/templates/toc.jinja @@ -0,0 +1,26 @@ +{# +Copyright 2016 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + diff --git a/skylark/BUILD b/skylark/BUILD new file mode 100644 index 0000000..9611c6d --- /dev/null +++ b/skylark/BUILD @@ -0,0 +1,9 @@ +package(default_visibility = ["//visibility:public"]) + +load("//skylark:skylark.bzl", "skylark_doc") + +skylark_doc( + name = "skylark-docs", + srcs = ["skylark.bzl"], + format = "html", +) diff --git a/skylark/skylark.bzl b/skylark/skylark.bzl new file mode 100644 index 0000000..43496c5 --- /dev/null +++ b/skylark/skylark.bzl @@ -0,0 +1,237 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Skylark rules""" + +_SKYLARK_FILETYPE = FileType([".bzl"]) + +ZIP_PATH = "/usr/bin/zip" + +def _get_transitive_sources(deps): + """Collects source files of transitive dependencies." + + Args: + deps: List of deps labels from ctx.attr.deps. + + Returns: + Returns a list of Files containing sources of transitive dependencies. + """ + transitive_sources = set(order="compile") + for dep in deps: + transitive_sources += dep.transitive_bzl_files + return transitive_sources + +def _skylark_library_impl(ctx): + """Implementation of the skylark_library rule.""" + sources = _get_transitive_sources(ctx.attr.deps) + ctx.files.srcs + return struct(files = set(), + transitive_bzl_files = sources) + +def _skydoc(ctx): + for f in ctx.files._skydoc: + if not f.path.endswith(".py"): + return f + +def _skylark_doc_impl(ctx): + """Implementation of the skylark_doc rule.""" + skylark_doc_zip = ctx.outputs.skylark_doc_zip + inputs = _get_transitive_sources(ctx.attr.deps) + ctx.files.srcs + sources = [source.path for source in inputs] + args = [ + "--format=%s" % ctx.attr.format, + "--output_file=%s" % ctx.outputs.skylark_doc_zip.path, + ] + sources + skydoc = _skydoc(ctx) + ctx.action( + inputs = list(inputs) + [skydoc], + executable = skydoc, + arguments = args, + outputs = [skylark_doc_zip], + mnemonic = "Skydoc", + use_default_shell_env = True, + progress_message = ("Generating Skylark doc for %s (%d files)" + % (ctx.label.name, len(sources)))) + +_skylark_common_attrs = { + "srcs": attr.label_list(allow_files = _SKYLARK_FILETYPE), + "deps": attr.label_list(providers = ["transitive_bzl_files"], + allow_files = False), +} + +skylark_library = rule( + _skylark_library_impl, + attrs = _skylark_common_attrs, +) +"""Creates a logical collection of Skylark .bzl files. + +Args: + srcs: List of `.bzl` files that are processed to create this target. + deps: List of other `skylark_library` targets that are required by the Skylark + files listed in `srcs`. +""" + +_skylark_doc_attrs = { + "format": attr.string(default = "markdown"), + "_skydoc": attr.label( + default = Label("//skydoc"), + cfg = HOST_CFG, + executable = True), +} + +skylark_doc = rule( + _skylark_doc_impl, + attrs = dict(_skylark_common_attrs.items() + _skylark_doc_attrs.items()), + outputs = { + "skylark_doc_zip": "%{name}-skydoc.zip", + }, +) +"""Generates Skylark rule documentation. + +Args: + srcs: List of `.bzl` files that are processed to create this target. + deps: List of other `skylark_library` targets that are required by the Skylark + files listed in `srcs`. + format: The type of output to generate. Possible values are `"markdown"` and + `"html"`. +""" + +JINJA2_BUILD_FILE = """ +py_library( + name = "jinja2", + srcs = glob(["jinja2/*.py"]), + srcs_version = "PY2AND3", + deps = [ + "@markupsafe_archive//:markupsafe", + ], + visibility = ["//visibility:public"], +) +""" + +MARKUPSAFE_BUILD_FILE = """ +py_library( + name = "markupsafe", + srcs = glob(["markupsafe/*.py"]), + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) +""" + +MISTUNE_BUILD_FILE = """ +py_library( + name = "mistune", + srcs = ["mistune.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) +""" + +SIX_BUILD_FILE = """ +py_library( + name = "six", + srcs = ["six.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) +""" + +GFLAGS_BUILD_FILE = """ +py_library( + name = "gflags", + srcs = [ + "gflags.py", + "gflags_validators.py", + ], + visibility = ["//visibility:public"], +) +""" + +def skydoc_repositories(): + """Adds the external repositories used by the skylark rules.""" + native.git_repository( + name = "protobuf", + remote = "https://github.com/google/protobuf.git", + commit = "60a0d41a2988a40cf3a94a4cb602f5f1c94135e9", + ) + + # Protobuf expects an //external:python_headers label which would contain the + # Python headers if fast Python protos is enabled. Since we are not using fast + # Python protos, bind python_headers to a dummy target. + native.bind( + name = "python_headers", + actual = "//:dummy", + ) + + native.new_http_archive( + name = "markupsafe_archive", + url = "https://pypi.python.org/packages/source/M/MarkupSafe/MarkupSafe-0.23.tar.gz#md5=f5ab3deee4c37cd6a922fb81e730da6e", + sha256 = "a4ec1aff59b95a14b45eb2e23761a0179e98319da5a7eb76b56ea8cdc7b871c3", + build_file_content = MARKUPSAFE_BUILD_FILE, + strip_prefix = "MarkupSafe-0.23", + ) + + native.bind( + name = "markupsafe", + actual = "@markupsafe_archive//:markupsafe", + ) + + native.new_http_archive( + name = "jinja2_archive", + url = "https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.8.tar.gz#md5=edb51693fe22c53cee5403775c71a99e", + sha256 = "bc1ff2ff88dbfacefde4ddde471d1417d3b304e8df103a7a9437d47269201bf4", + build_file_content = JINJA2_BUILD_FILE, + strip_prefix = "Jinja2-2.8", + ) + + native.bind( + name = "jinja2", + actual = "@jinja2_archive//:jinja2", + ) + + native.new_http_archive( + name = "mistune_archive", + url = "https://pypi.python.org/packages/source/m/mistune/mistune-0.7.1.tar.gz#md5=057bc28bf629d6a1283d680a34ed9d0f", + sha256 = "6076dedf768348927d991f4371e5a799c6a0158b16091df08ee85ee231d929a7", + build_file_content = MISTUNE_BUILD_FILE, + strip_prefix = "mistune-0.7.1", + ) + + native.bind( + name = "mistune", + actual = "@mistune_archive//:mistune", + ) + + native.new_http_archive( + name = "six_archive", + url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55", + sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", + build_file_content = SIX_BUILD_FILE, + strip_prefix = "six-1.10.0", + ) + + native.bind( + name = "six", + actual = "@six_archive//:six", + ) + + native.new_git_repository( + name = "gflags_repo", + remote = "https://github.com/google/python-gflags", + tag = "python-gflags-2.0", + build_file_content = GFLAGS_BUILD_FILE, + ) + + native.bind( + name = "gflags", + actual = "@gflags_repo//:gflags", + )