diff --git a/src/pulp_docs/main.py b/src/pulp_docs/main.py index ac0b9b5..1e3fddc 100644 --- a/src/pulp_docs/main.py +++ b/src/pulp_docs/main.py @@ -3,8 +3,8 @@ It defines its interface. """ -import subprocess import os +import subprocess import sys from pathlib import Path @@ -21,16 +21,34 @@ def get_abspath(name: str) -> Path: class Config: """ + Configuration shared among CLI and mkdocs_macro.py hooks. + Params: mkdocs_file: the base mkdocs used in serving/building repolist: the configuration repositories (which and how to fetch) + clear_cache: whether to clear cache before downloading from remote """ - def __init__(self): - self.verbose = False - self.workdir = Path() - self.mkdocs_file = files("pulp_docs").joinpath("data/mkdocs.yml") - self.repolist = files("pulp_docs").joinpath("data/repolist.yml") + def __init__(self, from_environ: bool = False): + if from_environ is False: + self.verbose = False + self.workdir = Path().absolute() + self.mkdocs_file = files("pulp_docs").joinpath("data/mkdocs.yml").absolute() + self.repolist = files("pulp_docs").joinpath("data/repolist.yml").absolute() + self.clear_cache = False + else: + self.verbose = bool(os.environ["PULPDOCS_VERBOSE"]) + self.workdir = Path(os.environ["PULPDOCS_WORKDIR"]) + self.mkdocs_file = Path(os.environ["PULPDOCS_MKDOCS_FILE"]) + self.repolist = Path(os.environ["PULPDOCS_REPOLIST"]) + self.clear_cache = ( + False + if os.environ["PULPDOCS_CLEAR_CACHE"].lower() in ("f", "false") + else True + ) + + def get_environ_dict(self): + return {f"PULPDOCS_{k.upper()}": str(v) for k, v in self.__dict__.items()} pass_config = click.make_pass_decorator(Config, ensure=True) @@ -45,11 +63,18 @@ def main(config: Config): @main.command() +@click.option( + "--clear-cache", + default=False, + is_flag=True, + help="Whether to clear the cache before serving (default=False).", +) @pass_config -def serve(config: Config): - """Run mkdocs server""" +def serve(config: Config, clear_cache: bool): + """Run mkdocs server.""" env = os.environ.copy() - env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())}) + config.clear_cache = clear_cache + env.update(config.get_environ_dict()) options = (("--config-file", config.mkdocs_file),) cmd = ["mkdocs", "serve"] @@ -63,7 +88,7 @@ def serve(config: Config): @main.command() @pass_config def build(config: Config): - """Build mkdocs site""" + """Build mkdocs site.""" env = os.environ.copy() env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())}) @@ -80,16 +105,8 @@ def build(config: Config): @main.command() @pass_config -def pull(config: Config): - """Pull repositories source from remote into WORKDIR""" - repolist = [ - LocalRepo("Pulp Rpm", get_abspath("new_repo1")), - LocalRepo("Pulp Rpm", get_abspath("new_repo2")), - LocalRepo("Pulp Rpm", get_abspath("new_repo3")), - ] - - repo_paths = download_repos(repolist, TMP_DIR) - print(repo_paths) +def status(config: Config): + """Print relevant information about repositories that will be used.""" if __name__ == "__main__": diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py index daf6ab9..42651eb 100644 --- a/src/pulp_docs/mkdocs_macros.py +++ b/src/pulp_docs/mkdocs_macros.py @@ -23,6 +23,7 @@ from importlib_resources import as_file, files +from pulp_docs.main import Config from pulp_docs.plugin_repos import Repos # the name of the docs in the source repositories @@ -47,7 +48,7 @@ def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None, use_cache: bool return tmpdir -def prepare_repositories(TMPDIR: Path, repos: Repos): +def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config): """ Download repositories into tmpdir and organize them in a convenient way to mkdocs and its plugins. @@ -99,7 +100,7 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): # 2. Download repo (copy locally or fetch from GH) this_src_dir = repo_sources / repo.name - repo.download(dest_dir=this_src_dir) + repo.download(dest_dir=this_src_dir, clear_cache=config.clear_cache) # 3. Isolate docs dir from codebase (makes mkdocs happy) this_docs_dir = repo_docs / repo.name @@ -145,23 +146,30 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): return (repo_docs, repo_sources) -def log_local_checkout(repos: Repos): - """Emit log.info about local checkout being used or warn if none.""" - local_checkouts = [ - repo.name for repo in repos.all if repo.status.use_local_checkout is True - ] - - log.info(f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)}") - log.info(f"[pulp-docs] Local checkouts in use: {local_checkouts}") +def print_user_repo(repos: Repos, config: Config): + """Emit report about local checkout being used or warn if none.""" + local_checkouts = [] + cached_repos = [] + downloaded_repos = [] + for repo in repos.all: + record = {repo.name: repo.status.download_source} + if repo.status.use_local_checkout is True: + local_checkouts.append(record) + elif repo.status.using_cache is True: + cached_repos.append(record) + else: + downloaded_repos.append(record) + + # log.info( + # f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)} (where pulp-docs is looking for local checkouts)" + # ) + log.info(f"[pulp-docs] Config: {config.get_environ_dict()}") + log.info(f"[pulp-docs] Cached repos: {cached_repos}") + log.info(f"[pulp-docs] Downloaded repos: {downloaded_repos}") if len(local_checkouts) == 0: log.warning("[pulp-docs] No local checkouts found. Serving in read-only mode.") - - -def create_no_content_page(tmpdir: Path): - """Create placeholder.md file to be used when section is empty""" - placeholder_page = Path(tmpdir / "placeholder.md") - placeholder_page.write_text("# Placeholder Page\n\nNo content here yet.") - return placeholder_page + else: + log.info(f"[pulp-docs] Local checkouts: {local_checkouts}") def get_navigation(tmpdir: Path, repos: Repos): @@ -264,10 +272,10 @@ def define_env(env): """The mkdocs-marcros 'on_configuration' hook. Used to setup the project.""" # Load configuration from environment log.info("[pulp-docs] Loading configuration from environ") - base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None) + config = Config(from_environ=True) - if base_repolist: - repos = Repos.from_yaml(Path(base_repolist)) + if config.repolist: + repos = Repos.from_yaml(config.repolist) else: repos = ( Repos.test_fixtures() @@ -277,7 +285,7 @@ def define_env(env): # Download and organize repository files log.info("[pulp-docs] Preparing repositories") TMPDIR = create_clean_tmpdir() - docs_dir, source_dir = prepare_repositories(TMPDIR, repos) + docs_dir, source_dir = prepare_repositories(TMPDIR, repos, config) # Configure mkdocstrings log.info("[pulp-docs] Configuring mkdocstrings") @@ -293,10 +301,11 @@ def define_env(env): log.info("[pulp-docs] Done with pulp-docs.") env.conf["pulp_repos"] = repos + env.conf["pulp_config"] = config def on_post_build(env): # Log relevant most useful information for end-user log.info("*" * 79) - log_local_checkout(repos=env.conf["pulp_repos"]) + print_user_repo(repos=env.conf["pulp_repos"], config=env.conf["pulp_config"]) log.info("*" * 79) diff --git a/src/pulp_docs/plugin_repos.py b/src/pulp_docs/plugin_repos.py index 09090e2..4b60c1e 100644 --- a/src/pulp_docs/plugin_repos.py +++ b/src/pulp_docs/plugin_repos.py @@ -22,6 +22,7 @@ log = logging.getLogger("mkdocs") FIXTURE_WORKDIR = Path("tests/fixtures").absolute() +DOWNLOAD_CACHE_DIR = Path(tempfile.gettempdir()) / "repo_downloads" RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html" @@ -55,16 +56,11 @@ class Repo: status: RepoStatus = RepoStatus() type: t.Optional[str] = None - @property - def local_url(self): - """Return local url for respository as {self.local_basepath}/{self.name}""" - return self.local_basepath / self.name - @property def rest_api_link(self): return RESTAPI_TEMPLATE.format(self.name) - def download(self, dest_dir: Path) -> str: + def download(self, dest_dir: Path, clear_cache: bool = False) -> str: """ Download repository source from url into the {dest_dir} Path. @@ -77,33 +73,50 @@ def download(self, dest_dir: Path) -> str: Args: dest: The destination directory where source files will be saved. e.g /tmp/pulp-tmp/repo_sources/pulpcore + clear_cache: Whether the cache should be cleared before downloading. Returns: The download url used """ log.info("Downloading '{}' to '{}'".format(self.name, dest_dir.absolute())) - # Download from local filesystem - download_url = None + if clear_cache is True: + log.info("Clearing cache dir") + shutil.rmtree(DOWNLOAD_CACHE_DIR, ignore_errors=True) + DOWNLOAD_CACHE_DIR.mkdir() + + cached_repo = Path(DOWNLOAD_CACHE_DIR / self.name).absolute() + download_from = cached_repo + copy_path = cached_repo + log_header = "" + + # from local filesystem if self.local_basepath is not None: - log.warning(f"Using local checkout: {str(self.local_url)}") - download_url = self.local_url.absolute() - shutil.copytree( - self.local_url, - dest_dir, - ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"), - ) - # Download from remote - elif not dest_dir.exists(): - download_url = download_from_gh_main( - dest_dir, self.owner, self.name, self.branch - ) - else: - log.warning(f"Using cache: {str(dest_dir.absolute())}") + log_header = "Using local checkout" + download_from = Path(self.local_basepath / self.name).absolute() + copy_path = download_from + # from cache + elif cached_repo.exists(): + log_header = "Using cache in tmpdir" + download_from = cached_repo + copy_path = cached_repo self.status.using_cache = True - download_url = str(dest_dir.absolute()) + # from remote + elif not cached_repo.exists(): + log_header = "Downloading from remote" + download_from = download_from_gh_main( + DOWNLOAD_CACHE_DIR / self.name, self.owner, self.name, self.branch + ) + copy_path = DOWNLOAD_CACHE_DIR / self.name + + # copy from source/cache to pulp-docs workdir + log.info(f"{log_header}: source={download_from}, copied_from={copy_path}") + shutil.copytree( + copy_path, + dest_dir, + ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"), + ) - # Return url used - self.status.download_source = str(download_url) + self.status.download_source = str(download_from) return self.status.download_source @@ -118,13 +131,12 @@ def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str): log.info("Downloading from Github with:\n{}".format(" ".join(cmd))) try: subprocess.run(cmd, check=True) - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: log.error( - "An error ocurred while trying to download '{name}' source-code:".format( - name=name - ) + f"An error ocurred while trying to download '{name}' source-code:\n{e}" ) raise + log.info("Done.") return url