diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py index 980c3ec..daf6ab9 100644 --- a/src/pulp_docs/mkdocs_macros.py +++ b/src/pulp_docs/mkdocs_macros.py @@ -28,17 +28,22 @@ # the name of the docs in the source repositories SRC_DOCS_DIRNAME = "staging_docs" +# the dir to lookup for local repo checkouts +CHECKOUT_WORKDIR = Path().absolute().parent + log = logging.getLogger("mkdocs") -def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None): +def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None, use_cache: bool = True): tmpdir_basepath = ( Path(custom_tmpdir) if custom_tmpdir else Path(tempfile.gettempdir()) ) tmpdir = tmpdir_basepath / "pulp-docs-tmp" - if tmpdir.exists(): + + # Clean tmpdir only if not using cache + if tmpdir.exists() and use_cache is False: shutil.rmtree(tmpdir) - tmpdir.mkdir() + tmpdir.mkdir() return tmpdir @@ -82,15 +87,21 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): # Download/copy source code to tmpdir repo_sources = TMPDIR / "repo_sources" repo_docs = TMPDIR / "repo_docs" + shutil.rmtree(repo_sources, ignore_errors=True) + shutil.rmtree(repo_docs, ignore_errors=True) + for repo in repos.all: - # 1. Download repo (copy locally or fetch from GH) + # 1. Check for local checkout if config doesnt specify a local_basepath + checkout_dir = Path().absolute().parent / repo.name + if repo.local_basepath is None and checkout_dir.exists(): + repo.status.use_local_checkout = True + repo.local_basepath = Path().absolute().parent + + # 2. Download repo (copy locally or fetch from GH) this_src_dir = repo_sources / repo.name - log.info( - "[pulp-docs] Downloading '{}', at '{}'".format(repo.name, this_src_dir) - ) repo.download(dest_dir=this_src_dir) - # 2. Isolate docs dir from codebase (makes mkdocs happy) + # 3. Isolate docs dir from codebase (makes mkdocs happy) this_docs_dir = repo_docs / repo.name log.info( "Moving doc files:\nfrom '{}'\nto '{}'".format(this_src_dir, this_docs_dir) @@ -100,20 +111,21 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): try: shutil.copy(this_src_dir / "CHANGELOG.md", this_docs_dir / "CHANGELOG.md") except FileNotFoundError: - log.warn("CHANGELOG.md does not exist. Keep going") + repo.status.has_changelog = False try: shutil.copy(this_src_dir / "README.md", this_docs_dir / "README.md") except FileNotFoundError: - log.warn("README.md does not exist. Keep going") + repo.status.has_readme = False - # 3. Generate REST Api pages (workaround) - log.info("Generating REST_API page") - rest_api_page = this_docs_dir / "docs" / "rest_api.md" - rest_api_page.touch() - md_title = f"# {repo.title} REST Api" - md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})" - rest_api_page.write_text(f"{md_title}\n\n{md_body}") + # 4. Generate REST Api pages (workaround) + if repo.type == "content": + log.info("Generating REST_API page") + rest_api_page = this_docs_dir / "docs" / "rest_api.md" + rest_api_page.touch() + md_title = f"# {repo.title} REST Api" + md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})" + rest_api_page.write_text(f"{md_title}\n\n{md_body}") # Copy template-files (from this plugin) to tmpdir log.info("[pulp-docs] Moving pulp-docs /docs to final destination") @@ -124,10 +136,27 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): repo_sources / repos.core.name / SRC_DOCS_DIRNAME / "index.md", repo_docs / "index.md", ) - log.info("[pulp-docs] Done downloading sources.") + + # Log + log.info("[pulp-docs] Done downloading sources. Here are the sources used:") + for repo in repos.all: + log.info({repo.name: repo.status}) + return (repo_docs, repo_sources) +def log_local_checkout(repos: Repos): + """Emit log.info about local checkout being used or warn if none.""" + local_checkouts = [ + repo.name for repo in repos.all if repo.status.use_local_checkout is True + ] + + log.info(f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)}") + log.info(f"[pulp-docs] Local checkouts in use: {local_checkouts}") + if len(local_checkouts) == 0: + log.warning("[pulp-docs] No local checkouts found. Serving in read-only mode.") + + def create_no_content_page(tmpdir: Path): """Create placeholder.md file to be used when section is empty""" placeholder_page = Path(tmpdir / "placeholder.md") @@ -233,33 +262,41 @@ def from_core(url: str): def define_env(env): """The mkdocs-marcros 'on_configuration' hook. Used to setup the project.""" - # === + # Load configuration from environment log.info("[pulp-docs] Loading configuration from environ") base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None) - log.info(f"{base_repolist=}\n") - log.info("[pulp-docs] Loading repolist file") - if base_repolist == "testing": - repos = Repos.test_fixtures() - else: + if base_repolist: repos = Repos.from_yaml(Path(base_repolist)) - log.info(f"{repos}") + else: + repos = ( + Repos.test_fixtures() + ) # try to use fixtures if there is no BASE_REPOLIST + log.info(f"Repository configurations loaded: {[repo.name for repo in repos.all]}") - # === + # Download and organize repository files log.info("[pulp-docs] Preparing repositories") TMPDIR = create_clean_tmpdir() docs_dir, source_dir = prepare_repositories(TMPDIR, repos) - # === + # Configure mkdocstrings log.info("[pulp-docs] Configuring mkdocstrings") code_sources = [str(source_dir / repo.name) for repo in repos.all] env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"][ "paths" ] = code_sources - # === + # Configure navigation log.info("[pulp-docs] Configuring navigation") env.conf["docs_dir"] = docs_dir env.conf["nav"] = get_navigation(docs_dir, repos) log.info("[pulp-docs] Done with pulp-docs.") + env.conf["pulp_repos"] = repos + + +def on_post_build(env): + # Log relevant most useful information for end-user + log.info("*" * 79) + log_local_checkout(repos=env.conf["pulp_repos"]) + log.info("*" * 79) diff --git a/src/pulp_docs/plugin_repos.py b/src/pulp_docs/plugin_repos.py index 8ee3450..09090e2 100644 --- a/src/pulp_docs/plugin_repos.py +++ b/src/pulp_docs/plugin_repos.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging +import os import shutil import subprocess import tarfile @@ -24,6 +25,20 @@ RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html" +@dataclass +class RepoStatus: + """ + Usefull status information about a downloaded repository. + """ + + download_source: t.Optional[str] = None + use_local_checkout: bool = False + has_readme: bool = True + has_changelog: bool = True + has_staging_docs: bool = True + using_cache: bool = False + + @dataclass class Repo: """ @@ -37,6 +52,8 @@ class Repo: owner: str = "pulp" branch: str = "main" local_basepath: t.Optional[Path] = None + status: RepoStatus = RepoStatus() + type: t.Optional[str] = None @property def local_url(self): @@ -47,53 +64,78 @@ def local_url(self): def rest_api_link(self): return RESTAPI_TEMPLATE.format(self.name) - def download(self, dest_dir: Path) -> Path: + def download(self, dest_dir: Path) -> str: """ Download repository source from url into the {dest_dir} Path. + Uses local in the following cases and order (else, downloads from github): + - local_basepath is explicitly set + - parent directory contain dir with self.name + For remote download, uses GitHub API to get latest source code: - https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release Args: dest: The destination directory where source files will be saved. + e.g /tmp/pulp-tmp/repo_sources/pulpcore + Returns: + The download url used """ - # Copy from local filesystem + log.info("Downloading '{}' to '{}'".format(self.name, dest_dir.absolute())) + + # Download from local filesystem + download_url = None if self.local_basepath is not None: + log.warning(f"Using local checkout: {str(self.local_url)}") + download_url = self.local_url.absolute() shutil.copytree( self.local_url, dest_dir, ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"), ) - return self.local_basepath + # Download from remote + elif not dest_dir.exists(): + download_url = download_from_gh_main( + dest_dir, self.owner, self.name, self.branch + ) + else: + log.warning(f"Using cache: {str(dest_dir.absolute())}") + self.status.using_cache = True + download_url = str(dest_dir.absolute()) - # or Download from remote - # download_from_gh_latest(dest_dir, self.owner, self.name) - download_from_gh_main(dest_dir, self.owner, self.name, self.branch) - return dest_dir + # Return url used + self.status.download_source = str(download_url) + return self.status.download_source def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str): - """Download repository source-code from main""" + """ + Download repository source-code from main + + Returns the download url. + """ url = f"https://github.com/{owner}/{name}.git" cmd = ("git", "clone", "--depth", "1", "--branch", branch, url, str(dest_dir)) log.info("Downloading from Github with:\n{}".format(" ".join(cmd))) try: subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: log.error( "An error ocurred while trying to download '{name}' source-code:".format( name=name ) ) - log.error(f"{e}") + raise log.info("Done.") + return url def download_from_gh_latest(dest_dir: Path, owner: str, name: str): """ - Download repository source-code from latest GitHub Release. + Download repository source-code from latest GitHub Release (w/ GitHub API). + + See: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release - Uses GitHub API. + Returns the download url. """ latest_release_link_url = ( "https://api.github.com/repos/{}/{}/releases/latest".format(owner, name) @@ -117,6 +159,7 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str): # Reference: # https://www.python-httpx.org/quickstart/#binary-response-content # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall + return latest_release_tar_url @dataclass @@ -149,25 +192,38 @@ def from_yaml(cls, path: str): title: Maven ``` """ + log.info("[pulp-docs] Loading repolist file from repofile.yml") file = Path(path) if not file.exists(): raise ValueError("File does not exist:", file) + log.info(f"repofile={str(file.absolute())}") with open(file, "r") as f: data = yaml.load(f, Loader=yaml.SafeLoader) repos = data["repos"] - core_repo = Repo(**repos["core"][0]) - content_repos = [Repo(**repo) for repo in repos["content"]] - other_repos = [Repo(**repo) for repo in repos["other"]] + core_repo = Repo(**repos["core"][0], type="core") + content_repos = [Repo(**repo, type="content") for repo in repos["content"]] + other_repos = [Repo(**repo, type="other") for repo in repos["other"]] return Repos(core=core_repo, content=content_repos, other=other_repos) @classmethod def test_fixtures(cls): """Factory of test Repos. Uses fixtures shipped in package data.""" - DEFAULT_CORE = Repo("Pulp Core", "core") + log.info("[pulp-docs] Loading repolist file from fixtures") + DEFAULT_CORE = Repo("Pulp Core", "core", type="core") DEFAULT_CONTENT_REPOS = [ - Repo("Rpm Packages", "new_repo1", local_basepath=FIXTURE_WORKDIR), - Repo("Debian Packages", "new_repo2", local_basepath=FIXTURE_WORKDIR), - Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR), + Repo( + "Rpm Packages", + "new_repo1", + local_basepath=FIXTURE_WORKDIR, + type="content", + ), + Repo( + "Debian Packages", + "new_repo2", + local_basepath=FIXTURE_WORKDIR, + type="content", + ), + Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR, type="content"), ] return Repos(core=DEFAULT_CORE, content=DEFAULT_CONTENT_REPOS)