From 6b423be77b525b9c76ec99b7f59ec52df7ba9336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Mon, 30 Oct 2023 11:51:31 +0100 Subject: [PATCH] Refactor generating of stories and lint check docs The code generating rst files from stories and linter sources was part of `docs/conf.py`. Hopefully, there will be more generated rst sources from tmt code, therefore doing a small cleanup: * code generating doc sources from tmt code has been moved into stand-alone scripts, with a bit of docs and polishing. * `docs/Makefile` and `Makefile` have been modified to offer nice help and use the new scripts for generating stuff, in a more make-like fashion, instead of letting `conf.py` control their creation. --- .readthedocs.yaml | 11 ++-- Makefile | 2 +- docs/Makefile | 64 +++++++++++++++++------- docs/conf.py | 76 ---------------------------- docs/scripts/generate-lint-checks | 42 ++++++++++++++++ docs/scripts/generate-stories | 83 +++++++++++++++++++++++++++++++ plans/install/docs.fmf | 8 +-- pyproject.toml | 3 +- 8 files changed, 184 insertions(+), 105 deletions(-) create mode 100755 docs/scripts/generate-lint-checks create mode 100755 docs/scripts/generate-stories diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cbec6e5ffa..27b0b6f594 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,8 +4,9 @@ build: os: ubuntu-22.04 tools: python: "3" -python: - install: - - method: pip - path: . - extra_requirements: [docs] + # Override RTD's default build commands, install hatch & delegate + # the rest to our Makefiles. + commands: + - cat docs/conf.py + - pip install hatch + - make BUILDDIR=$READTHEDOCS_OUTPUT docs diff --git a/Makefile b/Makefile index 3ddacbe501..b297bb350e 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ tags: clean: ## Remove all temporary files, packaging artifacts and docs rm -rf $(TMP) build dist tmt.1 rm -rf .cache .mypy_cache .ruff_cache - rm -rf docs/{_build,stories,spec} + make -C docs clean find . -type f -name "*.py[co]" -delete find . -type f -name "*,cover" -delete find . -type d -name "__pycache__" -delete diff --git a/docs/Makefile b/docs/Makefile index 504bb5f40c..5ce6e40f98 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,17 +1,14 @@ # Makefile for Sphinx documentation # +.DEFAULT_GOAL := help + # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter @@ -19,35 +16,66 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" +.PHONY: help generate-stories clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext clean: - rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR) stories spec -html: +## +## Building documentation +## +html: generate ## Make standalone HTML files $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -latexpdf: +latexpdf: generate ## Make LaTeX files and run them through pdflatex $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -text: +text: generate ## Make text files $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." -man: +man: generate ## Make manual pages $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +## +## Generate documentation sources from inputs +## +REPODIR = .. +TMTDIR = $(REPODIR)/tmt +SCRIPTSDIR = scripts + +generate: spec stories generate-lint-checks generate-stories ## Refresh all generated documentation sources + +spec: + mkdir -p spec + +stories: + mkdir -p stories + +spec/lint.rst: $(SCRIPTSDIR)/generate-lint-checks lint-checks.rst.j2 $(TMTDIR)/base.py + $(SCRIPTSDIR)/generate-lint-checks lint-checks.rst.j2 $@ + +generate-lint-checks: spec spec/lint.rst ## Generate documentation sources for lint checks + +generate-stories: stories ## Generate documentation sources for stories + $(SCRIPTSDIR)/generate-stories story.rst.j2 + +## +## Help! +## +help:: ## Show this help text + @gawk -vG=$$(tput setaf 2) -vR=$$(tput sgr0) ' \ + match($$0, "^(([^#:]*[^ :]) *:)?([^#]*)##([^#].+|)$$",a) { \ + if (a[2] != "") { printf " make %s%-22s%s %s\n", G, a[2], R, a[4]; next }\ + if (a[3] == "") { print a[4]; next }\ + printf "\n%-36s %s\n","",a[4]\ + }' $(MAKEFILE_LIST) + @echo "" # blank line at the end diff --git a/docs/conf.py b/docs/conf.py index b6a61610ac..472d312455 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,13 +15,8 @@ import os import sys from typing import List, Optional, Tuple -from unittest.mock import Mock as MagicMock -import tmt.base -import tmt.lint -import tmt.plugins import tmt.utils -from tmt.utils import Path _POSSIBLE_THEMES: List[Tuple[Optional[str], str]] = [ # Fall back to sphinx_rtd_theme if available @@ -296,74 +291,3 @@ def _load_theme( # If true, show URL addresses after external links. # man_show_urls = False - - -# Mock extra modules -class Mock(MagicMock): - @classmethod - def __getattr__(cls, name: str) -> 'Mock': - return Mock() - - -MOCK_MODULES = ['testcloud', 'testcloud.image', 'testcloud.instance'] -sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) - -# We will need a logger... -logger = tmt.Logger.create() - -# Explore available *export* plugins - do not import other plugins, we don't need them. -tmt.plugins.explore_export_package(logger) - -# Generate stories -tree = tmt.Tree(logger=logger, path=Path.cwd()) -story_template_filepath = Path('story.rst.j2') - -areas = { - '/stories/docs': 'Documentation', - '/stories/cli': 'Command Line', - '/stories/install': 'Installation', - '/stories/features': 'Features', - '/spec/core': 'Core', - '/spec/tests': 'Tests', - '/spec/plans': 'Plans', - '/spec/stories': 'Stories', - '/spec/context': 'Context', - '/spec/hardware': 'Hardware', - } - -os.makedirs('stories', exist_ok=True) -os.makedirs('spec', exist_ok=True) -for area in areas: - with open(f"{area.lstrip('/')}.rst", 'w') as doc: - # Anchor and title - doc.write(f'.. _{area}:\n\n') - doc.write(f"{areas[area]}\n{'=' * len(areas[area])}\n") - # Included stories - for story in tree.stories(names=[area], whole=True): - if not story.enabled: - continue - - rendered = story.export( - format='rst', - include_title=story.name != area, - template=story_template_filepath) - - doc.write(rendered) - doc.write('\n\n') - - -# Render list of lint checks -def _sort_linters(linters: List[tmt.lint.Linter]) -> List[tmt.lint.Linter]: - return sorted(linters, key=lambda x: x.id) - - -linters = { - 'TEST_LINTERS': _sort_linters(tmt.base.Test.get_linter_registry()), - 'PLAN_LINTERS': _sort_linters(tmt.base.Plan.get_linter_registry()), - 'STORY_LINTERS': _sort_linters(tmt.base.Story.get_linter_registry()), - } - - -with open('spec/lint.rst', 'w') as f: - f.write(tmt.utils.render_template_file(tmt.utils.Path('lint-checks.rst.j2'), **linters)) - f.flush() diff --git a/docs/scripts/generate-lint-checks b/docs/scripts/generate-lint-checks new file mode 100755 index 0000000000..7b7c1e73aa --- /dev/null +++ b/docs/scripts/generate-lint-checks @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import sys +import textwrap +from typing import List + +from tmt.base import Plan, Story, Test +from tmt.lint import Linter +from tmt.utils import Path, render_template_file + +HELP = textwrap.dedent(""" +Usage: generate-lint-checks + +Generate docs for all known lint checks. +""").strip() + + +def _sort_linters(linters: List[Linter]) -> List[Linter]: + """ Sort a list of linters by their ID """ + return sorted(linters, key=lambda x: x.id) + + +def main() -> None: + if len(sys.argv) != 3: + print(HELP) + + sys.exit(1) + + template_filepath = Path(sys.argv[1]) + output_filepath = Path(sys.argv[2]) + + linters = { + 'TEST_LINTERS': _sort_linters(Test.get_linter_registry()), + 'PLAN_LINTERS': _sort_linters(Plan.get_linter_registry()), + 'STORY_LINTERS': _sort_linters(Story.get_linter_registry()), + } + + output_filepath.write_text(render_template_file(template_filepath, **linters)) + + +if __name__ == '__main__': + main() diff --git a/docs/scripts/generate-stories b/docs/scripts/generate-stories new file mode 100755 index 0000000000..d8809f627b --- /dev/null +++ b/docs/scripts/generate-stories @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import sys +import textwrap +from unittest.mock import Mock as MagicMock + +import tmt.plugins +from tmt.utils import Path + +HELP = textwrap.dedent(""" +Usage: generate-stories + +Generate pages for stories from their fmf specifications. +""").strip() + +# Mock extra modules + + +class Mock(MagicMock): + @classmethod + def __getattr__(cls, name: str) -> 'Mock': + return Mock() + + +MOCK_MODULES = ['testcloud', 'testcloud.image', 'testcloud.instance'] +sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) + + +AREA_TITLES = { + '/stories/docs': 'Documentation', + '/stories/cli': 'Command Line', + '/stories/install': 'Installation', + '/stories/features': 'Features', + '/spec/core': 'Core', + '/spec/tests': 'Tests', + '/spec/plans': 'Plans', + '/spec/stories': 'Stories', + '/spec/context': 'Context', + '/spec/hardware': 'Hardware', + } + + +def main() -> None: + if len(sys.argv) != 2: + print(HELP) + + sys.exit(1) + + story_template_filepath = Path(sys.argv[1]) + + # We will need a logger... + logger = tmt.Logger.create() + logger.add_console_handler() + + # Explore available *export* plugins - do not import other plugins, we don't need them. + tmt.plugins.explore_export_package(logger) + + # Generate stories + tree = tmt.Tree(logger=logger, path=Path.cwd()) + + for area in AREA_TITLES: + logger.info(f'Generating rst files from {area}') + + with open(f"{area.lstrip('/')}.rst", 'w') as doc: + # Anchor and title + doc.write(f'.. _{area}:\n\n') + doc.write(f"{AREA_TITLES[area]}\n{'=' * len(AREA_TITLES[area])}\n") + # Included stories + for story in tree.stories(names=[area], whole=True): + if not story.enabled: + continue + + rendered = story.export( + format='rst', + include_title=story.name != area, + template=story_template_filepath) + + doc.write(rendered) + doc.write('\n\n') + + +if __name__ == '__main__': + main() diff --git a/plans/install/docs.fmf b/plans/install/docs.fmf index e3ccfda1bd..d650531d0b 100644 --- a/plans/install/docs.fmf +++ b/plans/install/docs.fmf @@ -4,13 +4,13 @@ description: well and there is no warning displayed during the build. prepare: how: install - package: python3-pip + package: + - make + - python3-pip execute: script: | set -ex set -o pipefail pip3 install .[docs] - cd docs - python3 -m sphinx -T -E -b html -d _build/doctrees \ - . _build/html 2>&1 | tee output + make -C docs html 2>&1 | tee output egrep 'ERROR|WARNING' output && exit 1 || exit 0 diff --git a/pyproject.toml b/pyproject.toml index 94cea1df24..f75d8edb87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ all = [ # Needed for readthedocs and man page build. Not being packaged in rpm. docs = [ "sphinx_rtd_theme>=1.2.2", # The theme pulls a specific sphinx version + "readthedocs-sphinx-ext", "docutils>=0.18.1", "fmf>=1.3.0", ] @@ -157,7 +158,7 @@ dependencies = ["tmt[docs]"] detached = true [tool.hatch.envs.docs.scripts] -html = "cd {root}/docs && make html" +html = "make -C {root}/docs html" man = [ "cp {root}/docs/header.txt {root}/man.rst", "tail -n+8 docs/overview.rst >> {root}/man.rst",