Add option --clear-cache to wipe the cached repos

src/pulp_docs/main.py

@@ -3,8 +3,8 @@
 
 It defines its interface.
 """
-import subprocess
 import os
+import subprocess
 import sys
 from pathlib import Path
 
@@ -21,16 +21,34 @@ def get_abspath(name: str) -> Path:
 
 class Config:
     """
+    Configuration shared among CLI and mkdocs_macro.py hooks.
+
     Params:
         mkdocs_file: the base mkdocs used in serving/building
         repolist: the configuration repositories (which and how to fetch)
+        clear_cache: whether to clear cache before downloading from remote
     """
 
-    def __init__(self):
-        self.verbose = False
-        self.workdir = Path()
-        self.mkdocs_file = files("pulp_docs").joinpath("data/mkdocs.yml")
-        self.repolist = files("pulp_docs").joinpath("data/repolist.yml")
+    def __init__(self, from_environ: bool = False):
+        if from_environ is False:
+            self.verbose = False
+            self.workdir = Path().absolute()
+            self.mkdocs_file = files("pulp_docs").joinpath("data/mkdocs.yml").absolute()
+            self.repolist = files("pulp_docs").joinpath("data/repolist.yml").absolute()
+            self.clear_cache = False
+        else:
+            self.verbose = bool(os.environ["PULPDOCS_VERBOSE"])
+            self.workdir = Path(os.environ["PULPDOCS_WORKDIR"])
+            self.mkdocs_file = Path(os.environ["PULPDOCS_MKDOCS_FILE"])
+            self.repolist = Path(os.environ["PULPDOCS_REPOLIST"])
+            self.clear_cache = (
+                False
+                if os.environ["PULPDOCS_CLEAR_CACHE"].lower() in ("f", "false")
+                else True
+            )
+
+    def get_environ_dict(self):
+        return {f"PULPDOCS_{k.upper()}": str(v) for k, v in self.__dict__.items()}
 
 
 pass_config = click.make_pass_decorator(Config, ensure=True)
@@ -45,11 +63,18 @@ def main(config: Config):
 
 
 @main.command()
+@click.option(
+    "--clear-cache",
+    default=False,
+    is_flag=True,
+    help="Whether to clear the cache before serving (default=False).",
+)
 @pass_config
-def serve(config: Config):
-    """Run mkdocs server"""
+def serve(config: Config, clear_cache: bool):
+    """Run mkdocs server."""
     env = os.environ.copy()
-    env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())})
+    config.clear_cache = clear_cache
+    env.update(config.get_environ_dict())
 
     options = (("--config-file", config.mkdocs_file),)
     cmd = ["mkdocs", "serve"]
@@ -63,7 +88,7 @@ def serve(config: Config):
 @main.command()
 @pass_config
 def build(config: Config):
-    """Build mkdocs site"""
+    """Build mkdocs site."""
     env = os.environ.copy()
     env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())})
 
@@ -80,16 +105,8 @@ def build(config: Config):
 
 @main.command()
 @pass_config
-def pull(config: Config):
-    """Pull repositories source from remote into WORKDIR"""
-    repolist = [
-        LocalRepo("Pulp Rpm", get_abspath("new_repo1")),
-        LocalRepo("Pulp Rpm", get_abspath("new_repo2")),
-        LocalRepo("Pulp Rpm", get_abspath("new_repo3")),
-    ]
-
-    repo_paths = download_repos(repolist, TMP_DIR)
-    print(repo_paths)
+def status(config: Config):
+    """Print relevant information about repositories that will be used."""
 
 
 if __name__ == "__main__":

src/pulp_docs/mkdocs_macros.py

@@ -23,6 +23,7 @@
 
 from importlib_resources import as_file, files
 
+from pulp_docs.main import Config
 from pulp_docs.plugin_repos import Repos
 
 # the name of the docs in the source repositories
@@ -47,7 +48,7 @@ def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None, use_cache: bool
     return tmpdir
 
 
-def prepare_repositories(TMPDIR: Path, repos: Repos):
+def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     """
     Download repositories into tmpdir and organize them in a convenient way
     to mkdocs and its plugins.
@@ -99,7 +100,7 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
 
         # 2. Download repo (copy locally or fetch from GH)
         this_src_dir = repo_sources / repo.name
-        repo.download(dest_dir=this_src_dir)
+        repo.download(dest_dir=this_src_dir, clear_cache=config.clear_cache)
 
         # 3. Isolate docs dir from codebase (makes mkdocs happy)
         this_docs_dir = repo_docs / repo.name
@@ -145,23 +146,30 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
     return (repo_docs, repo_sources)
 
 
-def log_local_checkout(repos: Repos):
-    """Emit log.info about local checkout being used or warn if none."""
-    local_checkouts = [
-        repo.name for repo in repos.all if repo.status.use_local_checkout is True
-    ]
-
-    log.info(f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)}")
-    log.info(f"[pulp-docs] Local checkouts in use: {local_checkouts}")
+def print_user_repo(repos: Repos, config: Config):
+    """Emit report  about local checkout being used or warn if none."""
+    local_checkouts = []
+    cached_repos = []
+    downloaded_repos = []
+    for repo in repos.all:
+        record = {repo.name: repo.status.download_source}
+        if repo.status.use_local_checkout is True:
+            local_checkouts.append(record)
+        elif repo.status.using_cache is True:
+            cached_repos.append(record)
+        else:
+            downloaded_repos.append(record)
+
+    # log.info(
+    #     f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)} (where pulp-docs is looking for local checkouts)"
+    # )
+    log.info(f"[pulp-docs] Config: {config.get_environ_dict()}")
+    log.info(f"[pulp-docs] Cached repos: {cached_repos}")
+    log.info(f"[pulp-docs] Downloaded repos: {downloaded_repos}")
     if len(local_checkouts) == 0:
         log.warning("[pulp-docs] No local checkouts found. Serving in read-only mode.")
-
-
-def create_no_content_page(tmpdir: Path):
-    """Create placeholder.md file to be used when section is empty"""
-    placeholder_page = Path(tmpdir / "placeholder.md")
-    placeholder_page.write_text("# Placeholder Page\n\nNo content here yet.")
-    return placeholder_page
+    else:
+        log.info(f"[pulp-docs] Local checkouts: {local_checkouts}")
 
 
 def get_navigation(tmpdir: Path, repos: Repos):
@@ -264,10 +272,10 @@ def define_env(env):
     """The mkdocs-marcros 'on_configuration' hook. Used to setup the project."""
     # Load configuration from environment
     log.info("[pulp-docs] Loading configuration from environ")
-    base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None)
+    config = Config(from_environ=True)
 
-    if base_repolist:
-        repos = Repos.from_yaml(Path(base_repolist))
+    if config.repolist:
+        repos = Repos.from_yaml(config.repolist)
     else:
         repos = (
             Repos.test_fixtures()
@@ -277,7 +285,7 @@ def define_env(env):
     # Download and organize repository files
     log.info("[pulp-docs] Preparing repositories")
     TMPDIR = create_clean_tmpdir()
-    docs_dir, source_dir = prepare_repositories(TMPDIR, repos)
+    docs_dir, source_dir = prepare_repositories(TMPDIR, repos, config)
 
     # Configure mkdocstrings
     log.info("[pulp-docs] Configuring mkdocstrings")
@@ -293,10 +301,11 @@ def define_env(env):
 
     log.info("[pulp-docs] Done with pulp-docs.")
     env.conf["pulp_repos"] = repos
+    env.conf["pulp_config"] = config
 
 
 def on_post_build(env):
     # Log relevant most useful information for end-user
     log.info("*" * 79)
-    log_local_checkout(repos=env.conf["pulp_repos"])
+    print_user_repo(repos=env.conf["pulp_repos"], config=env.conf["pulp_config"])
     log.info("*" * 79)

src/pulp_docs/plugin_repos.py

@@ -22,6 +22,7 @@
 log = logging.getLogger("mkdocs")
 
 FIXTURE_WORKDIR = Path("tests/fixtures").absolute()
+DOWNLOAD_CACHE_DIR = Path(tempfile.gettempdir()) / "repo_downloads"
 RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html"
 
 
@@ -55,16 +56,11 @@ class Repo:
     status: RepoStatus = RepoStatus()
     type: t.Optional[str] = None
 
-    @property
-    def local_url(self):
-        """Return local url for respository as {self.local_basepath}/{self.name}"""
-        return self.local_basepath / self.name
-
     @property
     def rest_api_link(self):
         return RESTAPI_TEMPLATE.format(self.name)
 
-    def download(self, dest_dir: Path) -> str:
+    def download(self, dest_dir: Path, clear_cache: bool = False) -> str:
         """
         Download repository source from url into the {dest_dir} Path.
 
@@ -77,33 +73,50 @@ def download(self, dest_dir: Path) -> str:
         Args:
             dest: The destination directory where source files will be saved.
                 e.g /tmp/pulp-tmp/repo_sources/pulpcore
+            clear_cache: Whether the cache should be cleared before downloading.
         Returns:
             The download url used
         """
         log.info("Downloading '{}' to '{}'".format(self.name, dest_dir.absolute()))
 
-        # Download from local filesystem
-        download_url = None
+        if clear_cache is True:
+            log.info("Clearing cache dir")
+            shutil.rmtree(DOWNLOAD_CACHE_DIR, ignore_errors=True)
+            DOWNLOAD_CACHE_DIR.mkdir()
+
+        cached_repo = Path(DOWNLOAD_CACHE_DIR / self.name).absolute()
+        download_from = cached_repo
+        copy_path = cached_repo
+        log_header = ""
+
+        # from local filesystem
         if self.local_basepath is not None:
-            log.warning(f"Using local checkout: {str(self.local_url)}")
-            download_url = self.local_url.absolute()
-            shutil.copytree(
-                self.local_url,
-                dest_dir,
-                ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"),
-            )
-        # Download from remote
-        elif not dest_dir.exists():
-            download_url = download_from_gh_main(
-                dest_dir, self.owner, self.name, self.branch
-            )
-        else:
-            log.warning(f"Using cache: {str(dest_dir.absolute())}")
+            log_header = "Using local checkout"
+            download_from = Path(self.local_basepath / self.name).absolute()
+            copy_path = download_from
+        # from cache
+        elif cached_repo.exists():
+            log_header = "Using cache in tmpdir"
+            download_from = cached_repo
+            copy_path = cached_repo
             self.status.using_cache = True
-            download_url = str(dest_dir.absolute())
+        # from remote
+        elif not cached_repo.exists():
+            log_header = "Downloading from remote"
+            download_from = download_from_gh_main(
+                DOWNLOAD_CACHE_DIR / self.name, self.owner, self.name, self.branch
+            )
+            copy_path = DOWNLOAD_CACHE_DIR / self.name
+
+        # copy from source/cache to pulp-docs workdir
+        log.info(f"{log_header}: source={download_from}, copied_from={copy_path}")
+        shutil.copytree(
+            copy_path,
+            dest_dir,
+            ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"),
+        )
 
-        # Return url used
-        self.status.download_source = str(download_url)
+        self.status.download_source = str(download_from)
         return self.status.download_source
 
 
@@ -118,13 +131,12 @@ def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str):
     log.info("Downloading from Github with:\n{}".format(" ".join(cmd)))
     try:
         subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError:
+    except subprocess.CalledProcessError as e:
         log.error(
-            "An error ocurred while trying to download '{name}' source-code:".format(
-                name=name
-            )
+            f"An error ocurred while trying to download '{name}' source-code:\n{e}"
         )
         raise
+
     log.info("Done.")
     return url