From 6927eb698d8bf517c559eeeb25af2e6a79abb1c4 Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Wed, 23 Apr 2025 16:18:53 -0300 Subject: [PATCH 1/2] Add repository finding mechanism similar to PATH --- src/pulp_docs/cli.py | 21 +++++++- src/pulp_docs/context.py | 1 + src/pulp_docs/plugin.py | 108 ++++++++++++++++++++++++++++----------- 3 files changed, 99 insertions(+), 31 deletions(-) diff --git a/src/pulp_docs/cli.py b/src/pulp_docs/cli.py index d22079f..588f629 100644 --- a/src/pulp_docs/cli.py +++ b/src/pulp_docs/cli.py @@ -5,7 +5,7 @@ from mkdocs.__main__ import cli as mkdocs_cli from mkdocs.config import load_config -from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft +from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft, ctx_path def blog_callback(ctx: click.Context, param: click.Parameter, value: bool) -> bool: @@ -25,6 +25,12 @@ def draft_callback(ctx: click.Context, param: click.Parameter, value: bool) -> b return value +def find_path_callback(ctx: click.Context, param: click.Parameter, value: bool) -> bool: + result = [item.strip() for item in value.split(":") if item.strip()] + ctx_path.set(result) + return result + + blog_option = click.option( "--blog/--no-blog", default=True, @@ -47,6 +53,15 @@ def draft_callback(ctx: click.Context, param: click.Parameter, value: bool) -> b help="Don't fail if repositories are missing.", ) +path_option = click.option( + "--path", + envvar="PULPDOCS_PATH", + expose_value=False, + default="", + callback=find_path_callback, + help="A colon separated list of repository paths. Accepts glob patterns.", +) + async def clone_repositories(repositories: list[str], dest_dir: Path) -> None: """Clone multiple repositories concurrently.""" @@ -105,3 +120,7 @@ def fetch(dest, config_file): draft_option(sub_command) blog_option(sub_command) docstrings_option(sub_command) + path_option(sub_command) + serve_options = sub_command.params + config_file_opt = next(filter(lambda opt: opt.name == "config_file", serve_options)) + config_file_opt.envvar = "PULPDOCS_DIR" diff --git a/src/pulp_docs/context.py b/src/pulp_docs/context.py index 39f42f7..bd54325 100644 --- a/src/pulp_docs/context.py +++ b/src/pulp_docs/context.py @@ -4,3 +4,4 @@ ctx_blog = ContextVar("ctx_blog", default=True) ctx_docstrings = ContextVar("ctx_docstrings", default=True) ctx_draft = ContextVar("ctx_draft", default=False) +ctx_path = ContextVar("ctx_path", default=None) diff --git a/src/pulp_docs/plugin.py b/src/pulp_docs/plugin.py index 0ab9b33..464eb51 100644 --- a/src/pulp_docs/plugin.py +++ b/src/pulp_docs/plugin.py @@ -3,19 +3,21 @@ import json import tomllib import yaml +import glob import httpx +from dataclasses import dataclass from git import Repo, GitCommandError from mkdocs.config import Config, config_options from mkdocs.config.defaults import MkDocsConfig from mkdocs.exceptions import PluginError -from mkdocs.plugins import event_priority, get_plugin_logger, BasePlugin +from mkdocs.plugins import get_plugin_logger, BasePlugin from mkdocs.structure.files import File, Files from mkdocs.structure.nav import Navigation, Section, Link from mkdocs.structure.pages import Page from mkdocs.utils.templates import TemplateContext -from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft +from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft, ctx_path log = get_plugin_logger(__name__) @@ -35,6 +37,43 @@ class ComponentOption(Config): rest_api = config_options.Type(str, default="") +@dataclass(frozen=True) +class Component: + title: str + path: str + kind: str + git_url: str + rest_api: str + + version: str + repository_dir: Path + component_dir: Path + + @classmethod + def build(cls, find_path: list[str], component_opt: ComponentOption): + body = dict(component_opt) + repository_name = component_opt.path.split("/")[0] + expanded_path = [] + for dir in find_path: + expanded_path.extend(glob.glob(dir)) + for dir in expanded_path: + dir = Path(dir) + component_dir = dir.parent / component_opt.path + found_dir = repository_name == dir.name and component_dir.exists() + if found_dir: + version = "unknown" + try: + pyproject = component_dir / "pyproject.toml" + version = tomllib.loads(pyproject.read_text())["project"]["version"] + except Exception: + pass + body["version"] = version + body["repository_dir"] = dir + body["component_dir"] = component_dir + return cls(**body) + return None + + class PulpDocsPluginConfig(Config): components = config_options.ListOfItems(ComponentOption, default=[]) @@ -175,18 +214,12 @@ def _render_sitemap_item(nav_item: Page | Section) -> str: def component_data( - component: ComponentOption, - component_dir: Path, + component: Component, ) -> dict[str, str | list[str]]: """Generate data for rendering md templates.""" + component_dir = component.component_dir path = component_dir.name - version = "unknown" - try: - pyproject = component_dir / "pyproject.toml" - version = tomllib.loads(pyproject.read_text())["project"]["version"] - except Exception: - pass github_org = "pulp" try: template_config = component_dir / "template_config.yml" @@ -204,7 +237,7 @@ def component_data( return { "title": f"[{component.title}](site:{path}/)", "kind": component.kind, - "version": version, + "version": component.version, "links": links, } @@ -225,34 +258,49 @@ def rss_items() -> list: return rss_feed["items"][:20] +def load_components(find_path: list[str], config: MkDocsConfig, draft: bool): + loaded_components = [] + for component_opt in config.components: + component = Component.build(find_path, component_opt) + if component: + loaded_components.append(component) + all_components = {o.path for o in config.components} + not_loaded_components = all_components.difference( + {o.path for o in loaded_components} + ) + if not_loaded_components: + not_loaded_components = sorted(not_loaded_components) + if draft: + log.warning(f"Skip missing components {not_loaded_components}.") + else: + raise PluginError(f"Components missing: {not_loaded_components}.") + return loaded_components + + class PulpDocsPlugin(BasePlugin[PulpDocsPluginConfig]): def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: - # Two directories up from docs is where we expect all the other repositories. self.blog = ctx_blog.get() self.docstrings = ctx_docstrings.get() self.draft = ctx_draft.get() self.pulp_docs_dir = Path(config.docs_dir).parent - self.repositories_dir = self.pulp_docs_dir.parent + # Two directories up from docs is where we expect all the other repositories by default. + self.find_path = ctx_path.get() or [f"{self.pulp_docs_dir.parent}/*"] + + loaded_components = load_components(self.find_path, self.config, self.draft) + self.config.components = loaded_components + log.info( + f"Using components={[str(c.component_dir) for c in loaded_components]}" + ) mkdocstrings_config = config.plugins["mkdocstrings"].config components_var = [] - new_components = [] for component in self.config.components: - component_dir = self.repositories_dir / component.path - if component_dir.exists(): - components_var.append(component_data(component, component_dir)) - config.watch.append(str(component_dir / "docs")) - mkdocstrings_config.handlers["python"]["paths"].append( - str(component_dir) - ) - new_components.append(component) - else: - if self.draft: - log.warning(f"Skip missing component '{component.title}'.") - else: - raise PluginError(f"Component '{component.title}' missing.") - self.config.components = new_components + components_var.append(component_data(component)) + config.watch.append(str(component.component_dir / "docs")) + mkdocstrings_config.handlers["python"]["paths"].append( + str(component.component_dir) + ) macros_plugin = config.plugins["macros"] macros_plugin.register_macros({"rss_items": rss_items}) @@ -273,10 +321,10 @@ def on_files(self, files: Files, /, *, config: MkDocsConfig) -> Files | None: user_nav: dict[str, t.Any] = {} dev_nav: dict[str, t.Any] = {} for component in self.config.components: - component_dir = self.repositories_dir / component.path + component_dir = component.component_dir log.info(f"Fetching docs from '{component.title}'.") - git_repository_dir = self.repositories_dir / Path(component.path).parts[0] + git_repository_dir = component.repository_dir try: git_branch = Repo(git_repository_dir).active_branch.name except TypeError: From 1c3e21053024a4c26bc0d95911839d32cad20ae7 Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Tue, 29 Apr 2025 13:06:56 -0300 Subject: [PATCH 2/2] Add repository specifier to find_path paths --- src/pulp_docs/cli.py | 2 +- src/pulp_docs/plugin.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/pulp_docs/cli.py b/src/pulp_docs/cli.py index 588f629..1102c59 100644 --- a/src/pulp_docs/cli.py +++ b/src/pulp_docs/cli.py @@ -59,7 +59,7 @@ def find_path_callback(ctx: click.Context, param: click.Parameter, value: bool) expose_value=False, default="", callback=find_path_callback, - help="A colon separated list of repository paths. Accepts glob patterns.", + help="A colon separated list of lookup paths in the form: [repo1@]path1 [:[repo2@]path2 [...]].", ) diff --git a/src/pulp_docs/plugin.py b/src/pulp_docs/plugin.py index 464eb51..3a6cc86 100644 --- a/src/pulp_docs/plugin.py +++ b/src/pulp_docs/plugin.py @@ -53,14 +53,13 @@ class Component: def build(cls, find_path: list[str], component_opt: ComponentOption): body = dict(component_opt) repository_name = component_opt.path.split("/")[0] - expanded_path = [] - for dir in find_path: - expanded_path.extend(glob.glob(dir)) - for dir in expanded_path: - dir = Path(dir) - component_dir = dir.parent / component_opt.path - found_dir = repository_name == dir.name and component_dir.exists() - if found_dir: + for dir_spec in find_path: + repo_filter, _, basedir = dir_spec.rpartition("@") + if repo_filter and repo_filter != repository_name: + continue + basedir = Path(basedir) + component_dir = basedir / component_opt.path + if component_dir.exists(): version = "unknown" try: pyproject = component_dir / "pyproject.toml" @@ -68,7 +67,7 @@ def build(cls, find_path: list[str], component_opt: ComponentOption): except Exception: pass body["version"] = version - body["repository_dir"] = dir + body["repository_dir"] = basedir / repository_name body["component_dir"] = component_dir return cls(**body) return None @@ -284,15 +283,13 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: self.draft = ctx_draft.get() self.pulp_docs_dir = Path(config.docs_dir).parent - # Two directories up from docs is where we expect all the other repositories by default. - self.find_path = ctx_path.get() or [f"{self.pulp_docs_dir.parent}/*"] + self.find_path = ctx_path.get() or [str(self.pulp_docs_dir.parent)] loaded_components = load_components(self.find_path, self.config, self.draft) self.config.components = loaded_components + loaded_component_dirnames = [str(c.component_dir) for c in loaded_components] + log.info(f"Using components={loaded_component_dirnames}") - log.info( - f"Using components={[str(c.component_dir) for c in loaded_components]}" - ) mkdocstrings_config = config.plugins["mkdocstrings"].config components_var = [] for component in self.config.components: