Skip to content

Commit

Permalink
Add basic local checkout discovery
Browse files Browse the repository at this point in the history
  • Loading branch information
@pedro-psb
pedro-psb committed Jan 9, 2024
1 parent bfd1dbc commit 625e4d4
Show file tree
Hide file tree
Showing 2 changed files with 141 additions and 48 deletions.
93 changes: 65 additions & 28 deletions src/pulp_docs/mkdocs_macros.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,22 @@
# the name of the docs in the source repositories
SRC_DOCS_DIRNAME = "staging_docs"

# the dir to lookup for local repo checkouts
CHECKOUT_WORKDIR = Path().absolute().parent

log = logging.getLogger("mkdocs")


def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None):
def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None, use_cache: bool = True):
tmpdir_basepath = (
Path(custom_tmpdir) if custom_tmpdir else Path(tempfile.gettempdir())
)
tmpdir = tmpdir_basepath / "pulp-docs-tmp"
if tmpdir.exists():

# Clean tmpdir only if not using cache
if tmpdir.exists() and use_cache is False:
shutil.rmtree(tmpdir)
tmpdir.mkdir()
tmpdir.mkdir()
return tmpdir


Expand Down Expand Up @@ -82,15 +87,21 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
# Download/copy source code to tmpdir
repo_sources = TMPDIR / "repo_sources"
repo_docs = TMPDIR / "repo_docs"
shutil.rmtree(repo_sources, ignore_errors=True)
shutil.rmtree(repo_docs, ignore_errors=True)

for repo in repos.all:
# 1. Download repo (copy locally or fetch from GH)
# 1. Check for local checkout if config doesnt specify a local_basepath
checkout_dir = Path().absolute().parent / repo.name
if repo.local_basepath is None and checkout_dir.exists():
repo.status.use_local_checkout = True
repo.local_basepath = Path().absolute().parent

# 2. Download repo (copy locally or fetch from GH)
this_src_dir = repo_sources / repo.name
log.info(
"[pulp-docs] Downloading '{}', at '{}'".format(repo.name, this_src_dir)
)
repo.download(dest_dir=this_src_dir)

# 2. Isolate docs dir from codebase (makes mkdocs happy)
# 3. Isolate docs dir from codebase (makes mkdocs happy)
this_docs_dir = repo_docs / repo.name
log.info(
"Moving doc files:\nfrom '{}'\nto '{}'".format(this_src_dir, this_docs_dir)
Expand All @@ -100,20 +111,21 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
try:
shutil.copy(this_src_dir / "CHANGELOG.md", this_docs_dir / "CHANGELOG.md")
except FileNotFoundError:
log.warn("CHANGELOG.md does not exist. Keep going")
repo.status.has_changelog = False

try:
shutil.copy(this_src_dir / "README.md", this_docs_dir / "README.md")
except FileNotFoundError:
log.warn("README.md does not exist. Keep going")
repo.status.has_readme = False

# 3. Generate REST Api pages (workaround)
log.info("Generating REST_API page")
rest_api_page = this_docs_dir / "docs" / "rest_api.md"
rest_api_page.touch()
md_title = f"# {repo.title} REST Api"
md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})"
rest_api_page.write_text(f"{md_title}\n\n{md_body}")
# 4. Generate REST Api pages (workaround)
if repo.type == "content":
log.info("Generating REST_API page")
rest_api_page = this_docs_dir / "docs" / "rest_api.md"
rest_api_page.touch()
md_title = f"# {repo.title} REST Api"
md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})"
rest_api_page.write_text(f"{md_title}\n\n{md_body}")

# Copy template-files (from this plugin) to tmpdir
log.info("[pulp-docs] Moving pulp-docs /docs to final destination")
Expand All @@ -124,10 +136,27 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
repo_sources / repos.core.name / SRC_DOCS_DIRNAME / "index.md",
repo_docs / "index.md",
)
log.info("[pulp-docs] Done downloading sources.")

# Log
log.info("[pulp-docs] Done downloading sources. Here are the sources used:")
for repo in repos.all:
log.info({repo.name: repo.status})

return (repo_docs, repo_sources)


def log_local_checkout(repos: Repos):
"""Emit log.info about local checkout being used or warn if none."""
local_checkouts = [
repo.name for repo in repos.all if repo.status.use_local_checkout is True
]

log.info(f"[pulp-docs] CHECKOUT_WORKDIR={str(CHECKOUT_WORKDIR)}")
log.info(f"[pulp-docs] Local checkouts in use: {local_checkouts}")
if len(local_checkouts) == 0:
log.warning("[pulp-docs] No local checkouts found. Serving in read-only mode.")


def create_no_content_page(tmpdir: Path):
"""Create placeholder.md file to be used when section is empty"""
placeholder_page = Path(tmpdir / "placeholder.md")
Expand Down Expand Up @@ -233,33 +262,41 @@ def from_core(url: str):

def define_env(env):
"""The mkdocs-marcros 'on_configuration' hook. Used to setup the project."""
# ===
# Load configuration from environment
log.info("[pulp-docs] Loading configuration from environ")
base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None)
log.info(f"{base_repolist=}\n")

log.info("[pulp-docs] Loading repolist file")
if base_repolist == "testing":
repos = Repos.test_fixtures()
else:
if base_repolist:
repos = Repos.from_yaml(Path(base_repolist))
log.info(f"{repos}")
else:
repos = (
Repos.test_fixtures()
) # try to use fixtures if there is no BASE_REPOLIST
log.info(f"Repository configurations loaded: {[repo.name for repo in repos.all]}")

# ===
# Download and organize repository files
log.info("[pulp-docs] Preparing repositories")
TMPDIR = create_clean_tmpdir()
docs_dir, source_dir = prepare_repositories(TMPDIR, repos)

# ===
# Configure mkdocstrings
log.info("[pulp-docs] Configuring mkdocstrings")
code_sources = [str(source_dir / repo.name) for repo in repos.all]
env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"][
"paths"
] = code_sources

# ===
# Configure navigation
log.info("[pulp-docs] Configuring navigation")
env.conf["docs_dir"] = docs_dir
env.conf["nav"] = get_navigation(docs_dir, repos)

log.info("[pulp-docs] Done with pulp-docs.")
env.conf["pulp_repos"] = repos


def on_post_build(env):
# Log relevant most useful information for end-user
log.info("*" * 79)
log_local_checkout(repos=env.conf["pulp_repos"])
log.info("*" * 79)
96 changes: 76 additions & 20 deletions src/pulp_docs/plugin_repos.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from __future__ import annotations

import logging
import os
import shutil
import subprocess
import tarfile
Expand All @@ -24,6 +25,20 @@
RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html"


@dataclass
class RepoStatus:
"""
Usefull status information about a downloaded repository.
"""

download_source: t.Optional[str] = None
use_local_checkout: bool = False
has_readme: bool = True
has_changelog: bool = True
has_staging_docs: bool = True
using_cache: bool = False


@dataclass
class Repo:
"""
Expand All @@ -37,6 +52,8 @@ class Repo:
owner: str = "pulp"
branch: str = "main"
local_basepath: t.Optional[Path] = None
status: RepoStatus = RepoStatus()
type: t.Optional[str] = None

@property
def local_url(self):
Expand All @@ -47,53 +64,78 @@ def local_url(self):
def rest_api_link(self):
return RESTAPI_TEMPLATE.format(self.name)

def download(self, dest_dir: Path) -> Path:
def download(self, dest_dir: Path) -> str:
"""
Download repository source from url into the {dest_dir} Path.
Uses local in the following cases and order (else, downloads from github):
- local_basepath is explicitly set
- parent directory contain dir with self.name
For remote download, uses GitHub API to get latest source code:
https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release
Args:
dest: The destination directory where source files will be saved.
e.g /tmp/pulp-tmp/repo_sources/pulpcore
Returns:
The download url used
"""
# Copy from local filesystem
log.info("Downloading '{}' to '{}'".format(self.name, dest_dir.absolute()))

# Download from local filesystem
download_url = None
if self.local_basepath is not None:
log.warning(f"Using local checkout: {str(self.local_url)}")
download_url = self.local_url.absolute()
shutil.copytree(
self.local_url,
dest_dir,
ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"),
)
return self.local_basepath
# Download from remote
elif not dest_dir.exists():
download_url = download_from_gh_main(
dest_dir, self.owner, self.name, self.branch
)
else:
log.warning(f"Using cache: {str(dest_dir.absolute())}")
self.status.using_cache = True
download_url = str(dest_dir.absolute())

# or Download from remote
# download_from_gh_latest(dest_dir, self.owner, self.name)
download_from_gh_main(dest_dir, self.owner, self.name, self.branch)
return dest_dir
# Return url used
self.status.download_source = str(download_url)
return self.status.download_source


def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str):
"""Download repository source-code from main"""
"""
Download repository source-code from main
Returns the download url.
"""
url = f"https://github.com/{owner}/{name}.git"
cmd = ("git", "clone", "--depth", "1", "--branch", branch, url, str(dest_dir))
log.info("Downloading from Github with:\n{}".format(" ".join(cmd)))
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
log.error(
"An error ocurred while trying to download '{name}' source-code:".format(
name=name
)
)
log.error(f"{e}")
raise
log.info("Done.")
return url


def download_from_gh_latest(dest_dir: Path, owner: str, name: str):
"""
Download repository source-code from latest GitHub Release.
Download repository source-code from latest GitHub Release (w/ GitHub API).
See: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release
Uses GitHub API.
Returns the download url.
"""
latest_release_link_url = (
"https://api.github.com/repos/{}/{}/releases/latest".format(owner, name)
Expand All @@ -117,6 +159,7 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str):
# Reference:
# https://www.python-httpx.org/quickstart/#binary-response-content
# https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall
return latest_release_tar_url


@dataclass
Expand Down Expand Up @@ -149,25 +192,38 @@ def from_yaml(cls, path: str):
title: Maven
```
"""
log.info("[pulp-docs] Loading repolist file from repofile.yml")
file = Path(path)
if not file.exists():
raise ValueError("File does not exist:", file)
log.info(f"repofile={str(file.absolute())}")

with open(file, "r") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
repos = data["repos"]
core_repo = Repo(**repos["core"][0])
content_repos = [Repo(**repo) for repo in repos["content"]]
other_repos = [Repo(**repo) for repo in repos["other"]]
core_repo = Repo(**repos["core"][0], type="core")
content_repos = [Repo(**repo, type="content") for repo in repos["content"]]
other_repos = [Repo(**repo, type="other") for repo in repos["other"]]
return Repos(core=core_repo, content=content_repos, other=other_repos)

@classmethod
def test_fixtures(cls):
"""Factory of test Repos. Uses fixtures shipped in package data."""
DEFAULT_CORE = Repo("Pulp Core", "core")
log.info("[pulp-docs] Loading repolist file from fixtures")
DEFAULT_CORE = Repo("Pulp Core", "core", type="core")
DEFAULT_CONTENT_REPOS = [
Repo("Rpm Packages", "new_repo1", local_basepath=FIXTURE_WORKDIR),
Repo("Debian Packages", "new_repo2", local_basepath=FIXTURE_WORKDIR),
Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR),
Repo(
"Rpm Packages",
"new_repo1",
local_basepath=FIXTURE_WORKDIR,
type="content",
),
Repo(
"Debian Packages",
"new_repo2",
local_basepath=FIXTURE_WORKDIR,
type="content",
),
Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR, type="content"),
]
return Repos(core=DEFAULT_CORE, content=DEFAULT_CONTENT_REPOS)

0 comments on commit 625e4d4

Please sign in to comment.