Refactor generating of stories and lint check docs

.readthedocs.yaml

@@ -4,8 +4,9 @@ build:
     os: ubuntu-22.04
     tools:
         python: "3"
-python:
-    install:
-      - method: pip
-        path: .
-        extra_requirements: [docs]
+    # Override RTD's default build commands, install hatch & delegate
+    # the rest to our Makefiles.
+    commands:
+      - cat docs/conf.py
+      - pip install hatch
+      - make BUILDDIR=$READTHEDOCS_OUTPUT docs

Makefile

@@ -95,7 +95,7 @@ tags:
 clean:  ## Remove all temporary files, packaging artifacts and docs
 	rm -rf $(TMP) build dist tmt.1
 	rm -rf .cache .mypy_cache .ruff_cache
-	rm -rf docs/{_build,stories,spec}
+	make -C docs clean
 	find . -type f -name "*.py[co]" -delete
 	find . -type f -name "*,cover" -delete
 	find . -type d -name "__pycache__" -delete

docs/Makefile

@@ -1,53 +1,81 @@
 # Makefile for Sphinx documentation
 #
 
+.DEFAULT_GOAL := help
+
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
 
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
-
-help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
+.PHONY: help generate-stories clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
 
 clean:
-	rm -rf $(BUILDDIR)/*
+	rm -rf $(BUILDDIR) stories spec
 
-html:
+##
+## Building documentation
+##
+html: generate  ## Make standalone HTML files
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-latexpdf:
+latexpdf: generate  ## Make LaTeX files and run them through pdflatex
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: generate  ## Make text files
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: generate  ## Make manual pages
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+##
+## Generate documentation sources from inputs
+##
+REPODIR       = ..
+TMTDIR        = $(REPODIR)/tmt
+SCRIPTSDIR    = scripts
+
+generate: spec stories generate-lint-checks generate-stories  ## Refresh all generated documentation sources
+
+spec:
+	mkdir -p spec
+
+stories:
+	mkdir -p stories
+
+spec/lint.rst: $(SCRIPTSDIR)/generate-lint-checks lint-checks.rst.j2 $(TMTDIR)/base.py
+	$(SCRIPTSDIR)/generate-lint-checks lint-checks.rst.j2 $@
+
+generate-lint-checks: spec spec/lint.rst  ## Generate documentation sources for lint checks
+
+generate-stories: stories  ## Generate documentation sources for stories
+	$(SCRIPTSDIR)/generate-stories story.rst.j2
+
+##
+## Help!
+##
+help:: ## Show this help text
+	@gawk -vG=$$(tput setaf 2) -vR=$$(tput sgr0) ' \
+	  match($$0, "^(([^#:]*[^ :]) *:)?([^#]*)##([^#].+|)$$",a) { \
+	    if (a[2] != "") { printf "    make %s%-22s%s %s\n", G, a[2], R, a[4]; next }\
+	    if (a[3] == "") { print a[4]; next }\
+	    printf "\n%-36s %s\n","",a[4]\
+	  }' $(MAKEFILE_LIST)
+	@echo "" # blank line at the end

docs/conf.py

@@ -15,13 +15,8 @@
 import os
 import sys
 from typing import List, Optional, Tuple
-from unittest.mock import Mock as MagicMock
 
-import tmt.base
-import tmt.lint
-import tmt.plugins
 import tmt.utils
-from tmt.utils import Path
 
 _POSSIBLE_THEMES: List[Tuple[Optional[str], str]] = [
     # Fall back to sphinx_rtd_theme if available
@@ -296,74 +291,3 @@ def _load_theme(
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
-
-
-# Mock extra modules
-class Mock(MagicMock):
-    @classmethod
-    def __getattr__(cls, name: str) -> 'Mock':
-        return Mock()
-
-
-MOCK_MODULES = ['testcloud', 'testcloud.image', 'testcloud.instance']
-sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
-
-# We will need a logger...
-logger = tmt.Logger.create()
-
-# Explore available *export* plugins - do not import other plugins, we don't need them.
-tmt.plugins.explore_export_package(logger)
-
-# Generate stories
-tree = tmt.Tree(logger=logger, path=Path.cwd())
-story_template_filepath = Path('story.rst.j2')
-
-areas = {
-    '/stories/docs': 'Documentation',
-    '/stories/cli': 'Command Line',
-    '/stories/install': 'Installation',
-    '/stories/features': 'Features',
-    '/spec/core': 'Core',
-    '/spec/tests': 'Tests',
-    '/spec/plans': 'Plans',
-    '/spec/stories': 'Stories',
-    '/spec/context': 'Context',
-    '/spec/hardware': 'Hardware',
-    }
-
-os.makedirs('stories', exist_ok=True)
-os.makedirs('spec', exist_ok=True)
-for area in areas:
-    with open(f"{area.lstrip('/')}.rst", 'w') as doc:
-        # Anchor and title
-        doc.write(f'.. _{area}:\n\n')
-        doc.write(f"{areas[area]}\n{'=' * len(areas[area])}\n")
-        # Included stories
-        for story in tree.stories(names=[area], whole=True):
-            if not story.enabled:
-                continue
-
-            rendered = story.export(
-                format='rst',
-                include_title=story.name != area,
-                template=story_template_filepath)
-
-            doc.write(rendered)
-            doc.write('\n\n')
-
-
-# Render list of lint checks
-def _sort_linters(linters: List[tmt.lint.Linter]) -> List[tmt.lint.Linter]:
-    return sorted(linters, key=lambda x: x.id)
-
-
-linters = {
-    'TEST_LINTERS': _sort_linters(tmt.base.Test.get_linter_registry()),
-    'PLAN_LINTERS': _sort_linters(tmt.base.Plan.get_linter_registry()),
-    'STORY_LINTERS': _sort_linters(tmt.base.Story.get_linter_registry()),
-    }
-
-
-with open('spec/lint.rst', 'w') as f:
-    f.write(tmt.utils.render_template_file(tmt.utils.Path('lint-checks.rst.j2'), **linters))
-    f.flush()

docs/scripts/generate-lint-checks

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import sys
+import textwrap
+from typing import List
+
+from tmt.base import Plan, Story, Test
+from tmt.lint import Linter
+from tmt.utils import Path, render_template_file
+
+HELP = textwrap.dedent("""
+Usage: generate-lint-checks <TEMPLATE-PATH> <OUTPUT-PATH>
+
+Generate docs for all known lint checks.
+""").strip()
+
+
+def _sort_linters(linters: List[Linter]) -> List[Linter]:
+    """ Sort a list of linters by their ID """
+    return sorted(linters, key=lambda x: x.id)
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        print(HELP)
+
+        sys.exit(1)
+
+    template_filepath = Path(sys.argv[1])
+    output_filepath = Path(sys.argv[2])
+
+    linters = {
+        'TEST_LINTERS': _sort_linters(Test.get_linter_registry()),
+        'PLAN_LINTERS': _sort_linters(Plan.get_linter_registry()),
+        'STORY_LINTERS': _sort_linters(Story.get_linter_registry()),
+        }
+
+    output_filepath.write_text(render_template_file(template_filepath, **linters))
+
+
+if __name__ == '__main__':
+    main()

docs/scripts/generate-stories

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import sys
+import textwrap
+from unittest.mock import Mock as MagicMock
+
+import tmt.plugins
+from tmt.utils import Path
+
+HELP = textwrap.dedent("""
+Usage: generate-stories <TEMPLATE-PATH>
+
+Generate pages for stories from their fmf specifications.
+""").strip()
+
+# Mock extra modules
+
+
+class Mock(MagicMock):
+    @classmethod
+    def __getattr__(cls, name: str) -> 'Mock':
+        return Mock()
+
+
+MOCK_MODULES = ['testcloud', 'testcloud.image', 'testcloud.instance']
+sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
+
+AREA_TITLES = {
+    '/stories/docs': 'Documentation',
+    '/stories/cli': 'Command Line',
+    '/stories/install': 'Installation',
+    '/stories/features': 'Features',
+    '/spec/core': 'Core',
+    '/spec/tests': 'Tests',
+    '/spec/plans': 'Plans',
+    '/spec/stories': 'Stories',
+    '/spec/context': 'Context',
+    '/spec/hardware': 'Hardware',
+    }
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        print(HELP)
+
+        sys.exit(1)
+
+    story_template_filepath = Path(sys.argv[1])
+
+    # We will need a logger...
+    logger = tmt.Logger.create()
+    logger.add_console_handler()
+
+    # Explore available *export* plugins - do not import other plugins, we don't need them.
+    tmt.plugins.explore_export_package(logger)
+
+    # Generate stories
+    tree = tmt.Tree(logger=logger, path=Path.cwd())
+
+    for area in AREA_TITLES:
+        logger.info(f'Generating rst files from {area}')
+
+        with open(f"{area.lstrip('/')}.rst", 'w') as doc:
+            # Anchor and title
+            doc.write(f'.. _{area}:\n\n')
+            doc.write(f"{AREA_TITLES[area]}\n{'=' * len(AREA_TITLES[area])}\n")
+            # Included stories
+            for story in tree.stories(names=[area], whole=True):
+                if not story.enabled:
+                    continue
+
+                rendered = story.export(
+                    format='rst',
+                    include_title=story.name != area,
+                    template=story_template_filepath)
+
+                doc.write(rendered)
+                doc.write('\n\n')
+
+
+if __name__ == '__main__':
+    main()

plans/install/docs.fmf

@@ -4,13 +4,13 @@ description:
     well and there is no warning displayed during the build.
 prepare:
     how: install
-    package: python3-pip
+    package:
+      - make
+      - python3-pip
 execute:
     script: |
         set -ex
         set -o pipefail
         pip3 install .[docs]
-        cd docs
-        python3 -m sphinx -T -E -b html -d _build/doctrees \
-            . _build/html 2>&1 | tee output
+        make -C docs html 2>&1 | tee output
         egrep 'ERROR|WARNING' output && exit 1 || exit 0

pyproject.toml

@@ -75,6 +75,7 @@ all = [
 # Needed for readthedocs and man page build. Not being packaged in rpm.
 docs = [
     "sphinx_rtd_theme>=1.2.2", # The theme pulls a specific sphinx version
+    "readthedocs-sphinx-ext",
     "docutils>=0.18.1",
     "fmf>=1.3.0",
     ]
@@ -157,7 +158,7 @@ dependencies = ["tmt[docs]"]
 detached = true
 
 [tool.hatch.envs.docs.scripts]
-html = "cd {root}/docs && make html"
+html = "make -C {root}/docs html"
 man = [
     "cp {root}/docs/header.txt {root}/man.rst",
     "tail -n+8 docs/overview.rst >> {root}/man.rst",