diff --git a/src/pulp_docs/fake_repo_generator.py b/src/pulp_docs/fake_repo_generator.py index 04fb38d..85f556a 100644 --- a/src/pulp_docs/fake_repo_generator.py +++ b/src/pulp_docs/fake_repo_generator.py @@ -16,6 +16,7 @@ @dataclass class Repository: """A fake repository""" + basedir: str name: str owner: str @@ -27,6 +28,7 @@ def generate_documentation_files(self): @dataclass class MultirepoFactory: """Responsible for creating the multi-repo setup.""" + repositories: t.List[Repository] def generate_multirepo_setup(self): @@ -37,13 +39,18 @@ def add_changes_and_bump_versions(self): #: This are the allowed content types -ContentType = t.Union[t.Literal["learn"], t.Literal["guide"], - t.Literal["reference"], t.Literal["tutorial"]] +ContentType = t.Union[ + t.Literal["learn"], + t.Literal["guide"], + t.Literal["reference"], + t.Literal["tutorial"], +] @dataclass class Document: """A markdown document which belongs to some content-type.""" + name: str template_name: str template_vars: dict @@ -71,111 +78,186 @@ def create(self, write: bool = False): @dataclass class ContentFactory: """Creates a Document or Documents list with content-type templates.""" + basepath: t.Optional[Path] = None template_name = "content_templates" templates = ("guide_page", "tutorial_page", "tutorial_main", "learn_page") content = ("learn", "guide", "tutorial") faker = Faker() - def create_guide(self, title: t.Optional[str] = None, repository: t.Optional[str] = None, persona: t.Optional[str] = None) -> Document: + def create_guide( + self, + title: t.Optional[str] = None, + repository: t.Optional[str] = None, + persona: t.Optional[str] = None, + ) -> Document: """Creates a guide""" verbs = ["do", "create", "generate", "bind", "delete", "update"] def gen_paragraph(): - return " ".join( - (self.faker.paragraph()) for _ in range(3)) + return " ".join((self.faker.paragraph()) for _ in range(3)) def gen_imperative(): - return "{} {}".format(self.faker.word( - ext_word_list=verbs), self.faker.sentence(nb_words=3).lower()) + return "{} {}".format( + self.faker.word(ext_word_list=verbs), + self.faker.sentence(nb_words=3).lower(), + ) + # optional kwargs title = title or "How to {}".format(gen_imperative()[:-1]) repository = repository or "default" - persona = persona or self.faker.word( - ext_word_list=["admin", "dev", "norm"]) + persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) # guide generation n_steps = randint(2, 4) steps = [gen_imperative() for _ in range(n_steps)] template_vars = { - "tags": ["guide", persona], - "created": "0.1.0", - "updated": [], - "title": title, + "tags": ["guide", persona], + "created": "0.1.0", + "updated": [], + "title": title, "overview": gen_paragraph(), "steps": steps, - "sections": [("{} - {}".format(i + 1, step), gen_paragraph()) - for i, step in enumerate(steps)] + "sections": [ + ("{} - {}".format(i + 1, step), gen_paragraph()) + for i, step in enumerate(steps) + ], } relative_path = "{}/guides".format(persona) - return Document(title, "guide_page.md.j2", template_vars, relative_path=relative_path, basepath=self.basepath) - - def create_learn(self, title: t.Optional[str] = None, repository: t.Optional[str] = None, persona: t.Optional[str] = None) -> Document: + return Document( + title, + "guide_page.md.j2", + template_vars, + relative_path=relative_path, + basepath=self.basepath, + ) + + def create_learn( + self, + title: t.Optional[str] = None, + repository: t.Optional[str] = None, + persona: t.Optional[str] = None, + ) -> Document: """Creates a learn""" subject = ["database", "model", "content", "artifact", "repository"] - subject_complement = ["configuration", - "deployment", "ecossystem", "setup"] - adjective = ["resilient", "complex", "flexible", - "autonomous", "modular", "scalable"] + subject_complement = ["configuration", "deployment", "ecossystem", "setup"] + adjective = [ + "resilient", + "complex", + "flexible", + "autonomous", + "modular", + "scalable", + ] introductory = ["On", "About", "In perspective:", "Understanding"] def gen_paragraphs(): - return " ".join( - (self.faker.paragraph()) for _ in range(3)) + return " ".join((self.faker.paragraph()) for _ in range(3)) def gen_title(): - return " ".join([self.faker.word(ext_word_list=introductory), - self.faker.word(ext_word_list=adjective), - self.faker.word(ext_word_list=subject).title(), - self.faker.word(ext_word_list=subject_complement)]) + return " ".join( + [ + self.faker.word(ext_word_list=introductory), + self.faker.word(ext_word_list=adjective), + self.faker.word(ext_word_list=subject).title(), + self.faker.word(ext_word_list=subject_complement), + ] + ) def gen_section(): - return (self.faker.sentence()[:-1], "\n\n".join(self.faker.texts(nb_texts=randint(3, 5), max_nb_chars=randint(200, 400)))) + return ( + self.faker.sentence()[:-1], + "\n\n".join( + self.faker.texts( + nb_texts=randint(3, 5), max_nb_chars=randint(200, 400) + ) + ), + ) title = title or gen_title() - persona = persona or self.faker.word( - ext_word_list=["admin", "dev", "norm"]) + persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) repository = repository or "default" template_vars = { - "tags": ["learn", persona], - "created": "0.1.0", - "updated": [], - "title": title, + "tags": ["learn", persona], + "created": "0.1.0", + "updated": [], + "title": title, "overview": gen_paragraphs(), - "sections": [gen_section() for _ in range(randint(2, 4))] + "sections": [gen_section() for _ in range(randint(2, 4))], } relative_path = "{}/learn".format(persona) - return Document(title, "learn_page.md.j2", template_vars, relative_path=relative_path, basepath=self.basepath) - - def create_tutorial(self, title: t.Optional[str] = None, repository: t.Optional[str] = None, persona: t.Optional[str] = None) -> t.List[Document]: + return Document( + title, + "learn_page.md.j2", + template_vars, + relative_path=relative_path, + basepath=self.basepath, + ) + + def create_tutorial( + self, + title: t.Optional[str] = None, + repository: t.Optional[str] = None, + persona: t.Optional[str] = None, + ) -> t.List[Document]: """Creates a tutorial""" documents = [] - subject = ["the database", "the model", - "the content", "the artifact", "the repository"] - verb = ["configuring", "deploying", - "setting", "organizing", "doing", "handling"] - adjective = ["infrastructure", "internals", "contexts", - "dependencie", "relationships", "signals"] + subject = [ + "the database", + "the model", + "the content", + "the artifact", + "the repository", + ] + verb = [ + "configuring", + "deploying", + "setting", + "organizing", + "doing", + "handling", + ] + adjective = [ + "infrastructure", + "internals", + "contexts", + "dependencie", + "relationships", + "signals", + ] topics = ["Content Management", "Development", "System Administration"] def gen_paragraphs(n: t.Optional[int] = None): n = n or 3 - return " ".join( - (self.faker.paragraph()) for _ in range(n)) + return " ".join((self.faker.paragraph()) for _ in range(n)) def gen_title(): - return "Getting started with {}".format(self.faker.word(ext_word_list=topics)) + return "Getting started with {}".format( + self.faker.word(ext_word_list=topics) + ) def gen_section_title(): - return " ".join([self.faker.word(ext_word_list=verb).title(), self.faker.word(ext_word_list=subject), self.faker.word(ext_word_list=adjective)]) + return " ".join( + [ + self.faker.word(ext_word_list=verb).title(), + self.faker.word(ext_word_list=subject), + self.faker.word(ext_word_list=adjective), + ] + ) def gen_section(): - return (self.faker.sentence()[:-1], "\n\n".join(self.faker.texts(nb_texts=randint(3, 5), max_nb_chars=randint(200, 400)))) - - persona = persona or self.faker.word( - ext_word_list=["admin", "dev", "norm"]) + return ( + self.faker.sentence()[:-1], + "\n\n".join( + self.faker.texts( + nb_texts=randint(3, 5), max_nb_chars=randint(200, 400) + ) + ), + ) + + persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) title = title or "Getting started as {}".format(persona) repository = repository or "default" tags = ["tutorial", persona] @@ -184,29 +266,43 @@ def gen_section(): main_title = "00 - {}".format(title) template_vars = { "tags": tags, - "created": "0.1.0", - "updated": [], - "title": main_title, + "created": "0.1.0", + "updated": [], + "title": main_title, "overview": gen_paragraphs(5), - "sections": [gen_section() for _ in range(randint(2, 4))] + "sections": [gen_section() for _ in range(randint(2, 4))], } relative_path = "{}/tutorials/{}".format(persona, title) documents.append( - Document(main_title, "tutorial_main.md.j2", template_vars, relative_path=relative_path, basepath=self.basepath)) + Document( + main_title, + "tutorial_main.md.j2", + template_vars, + relative_path=relative_path, + basepath=self.basepath, + ) + ) # generate each page of the tutorial for i in range(randint(2, 4)): - section_title = "0{} - {}".format(i+1, gen_section_title()) + section_title = "0{} - {}".format(i + 1, gen_section_title()) template_vars = { - "tags": ["tutorial", persona], - "created": "0.1.0", - "updated": [], - "title": section_title, + "tags": ["tutorial", persona], + "created": "0.1.0", + "updated": [], + "title": section_title, "overview": gen_paragraphs(), - "sections": [gen_section() for _ in range(randint(2, 4))] + "sections": [gen_section() for _ in range(randint(2, 4))], } documents.append( - Document(section_title, "tutorial_main.md.j2", template_vars, relative_path=relative_path, basepath=self.basepath)) + Document( + section_title, + "tutorial_main.md.j2", + template_vars, + relative_path=relative_path, + basepath=self.basepath, + ) + ) return documents @@ -214,6 +310,7 @@ def gen_section(): @dataclass class DocumentationFactory: """Generates Diataxis-based documentation for Pulp""" + basepath: Path def create_documentation(self) -> t.List[Document]: @@ -223,12 +320,18 @@ def create_documentation(self) -> t.List[Document]: # create content for one repo content_factory = ContentFactory(self.basepath) for persona in ("developer", "content-manager", "sys-admin"): - [content_factory.create_guide(persona=persona).create( - write=True) for _ in range(randint(1, 5))] - [content_factory.create_learn(persona=persona).create( - write=True) for _ in range(randint(1, 5))] - [tuto.create(write=True) - for tuto in content_factory.create_tutorial(persona=persona)] + [ + content_factory.create_guide(persona=persona).create(write=True) + for _ in range(randint(1, 5)) + ] + [ + content_factory.create_learn(persona=persona).create(write=True) + for _ in range(randint(1, 5)) + ] + [ + tuto.create(write=True) + for tuto in content_factory.create_tutorial(persona=persona) + ] return documents @@ -236,16 +339,20 @@ def create_documentation(self) -> t.List[Document]: def main(): # parse args parser = argparse.ArgumentParser( - "docs_generator", description="Genreate a doc structure populated with fake data") + "docs_generator", + description="Genreate a doc structure populated with fake data", + ) parser.add_argument( - "basepath", help="The basepath where the documentation folder will be created") + "basepath", help="The basepath where the documentation folder will be created" + ) args = parser.parse_args() basepath = Path() / args.basepath # ensure won't unintended override a direcotory if basepath.exists(): overwrite = input( - f"{basepath} exists, do you wanna override? (N/y): ").lower() in ("y", "yes") + f"{basepath} exists, do you wanna override? (N/y): " + ).lower() in ("y", "yes") if overwrite: print(f"Cleaned: {basepath.absolute()}.") shutil.rmtree(basepath) diff --git a/src/pulp_docs/main.py b/src/pulp_docs/main.py index 76f9f29..ac0b9b5 100644 --- a/src/pulp_docs/main.py +++ b/src/pulp_docs/main.py @@ -49,13 +49,9 @@ def main(config: Config): def serve(config: Config): """Run mkdocs server""" env = os.environ.copy() - env.update({ - "PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute()) - }) + env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())}) - options = ( - ("--config-file", config.mkdocs_file), - ) + options = (("--config-file", config.mkdocs_file),) cmd = ["mkdocs", "serve"] for opt in options: @@ -69,9 +65,7 @@ def serve(config: Config): def build(config: Config): """Build mkdocs site""" env = os.environ.copy() - env.update({ - "PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute()) - }) + env.update({"PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())}) options = ( ("--config-file", config.mkdocs_file), diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py index 6118485..980c3ec 100644 --- a/src/pulp_docs/mkdocs_macros.py +++ b/src/pulp_docs/mkdocs_macros.py @@ -28,12 +28,13 @@ # the name of the docs in the source repositories SRC_DOCS_DIRNAME = "staging_docs" -log = logging.getLogger('mkdocs') +log = logging.getLogger("mkdocs") def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None): - tmpdir_basepath = Path(custom_tmpdir) if custom_tmpdir else Path( - tempfile.gettempdir()) + tmpdir_basepath = ( + Path(custom_tmpdir) if custom_tmpdir else Path(tempfile.gettempdir()) + ) tmpdir = tmpdir_basepath / "pulp-docs-tmp" if tmpdir.exists(): shutil.rmtree(tmpdir) @@ -85,25 +86,24 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): # 1. Download repo (copy locally or fetch from GH) this_src_dir = repo_sources / repo.name log.info( - "[pulp-docs] Downloading '{}', at '{}'".format(repo.name, this_src_dir)) + "[pulp-docs] Downloading '{}', at '{}'".format(repo.name, this_src_dir) + ) repo.download(dest_dir=this_src_dir) # 2. Isolate docs dir from codebase (makes mkdocs happy) this_docs_dir = repo_docs / repo.name - log.info("Moving doc files:\nfrom '{}'\nto '{}'".format( - this_src_dir, this_docs_dir)) - shutil.copytree(this_src_dir / SRC_DOCS_DIRNAME, - this_docs_dir / "docs") + log.info( + "Moving doc files:\nfrom '{}'\nto '{}'".format(this_src_dir, this_docs_dir) + ) + shutil.copytree(this_src_dir / SRC_DOCS_DIRNAME, this_docs_dir / "docs") try: - shutil.copy(this_src_dir / "CHANGELOG.md", - this_docs_dir / "CHANGELOG.md") + shutil.copy(this_src_dir / "CHANGELOG.md", this_docs_dir / "CHANGELOG.md") except FileNotFoundError: log.warn("CHANGELOG.md does not exist. Keep going") try: - shutil.copy(this_src_dir / "README.md", - this_docs_dir / "README.md") + shutil.copy(this_src_dir / "README.md", this_docs_dir / "README.md") except FileNotFoundError: log.warn("README.md does not exist. Keep going") @@ -120,8 +120,10 @@ def prepare_repositories(TMPDIR: Path, repos: Repos): data_file_docs = files("pulp_docs").joinpath("data/docs") with as_file(data_file_docs) as _docs: shutil.copytree(_docs, repo_docs / "pulp-docs") - shutil.copy(repo_sources / repos.core.name / SRC_DOCS_DIRNAME / - "index.md", repo_docs / "index.md") + shutil.copy( + repo_sources / repos.core.name / SRC_DOCS_DIRNAME / "index.md", + repo_docs / "index.md", + ) log.info("[pulp-docs] Done downloading sources.") return (repo_docs, repo_sources) @@ -135,12 +137,16 @@ def create_no_content_page(tmpdir: Path): def get_navigation(tmpdir: Path, repos: Repos): """The dynamic generated 'nav' section of mkdocs.yml""" + # {repo}/docs/{persona}/{content-type}/*md # {repo}/docs/reference/*md def get_children(path: t.Union[str, Path]): _path = tmpdir / path if isinstance(path, str) else path - result = [str(file.relative_to(tmpdir)) - for file in _path.glob("*.md") if not file.name.startswith("_")] + result = [ + str(file.relative_to(tmpdir)) + for file in _path.glob("*.md") + if not file.name.startswith("_") + ] return result def expand_repos(template_str: str): @@ -171,41 +177,50 @@ def from_core(url: str): getting_started = [ {"Overview": from_core("docs/sections/getting_started/index.md")}, - {"Quickstart": get_children( - from_core("docs/sections/getting_started/quickstart"))}, - {"Fundamentals": get_children( - from_core("docs/sections/getting_started/fundamentals"))} + { + "Quickstart": get_children( + from_core("docs/sections/getting_started/quickstart") + ) + }, + { + "Fundamentals": get_children( + from_core("docs/sections/getting_started/fundamentals") + ) + }, ] - guides= [ + guides = [ {"Overview": from_core("docs/sections/guides/index.md")}, - {"For Content-Management": expand_repos( - "{repo}/docs/content-manager/guides")}, + {"For Content-Management": expand_repos("{repo}/docs/content-manager/guides")}, {"For Sys-Admins": expand_repos("{repo}/docs/sys-admin/guides")}, ] - learn= [ + learn = [ {"Overview": from_core("docs/sections/learn/index.md")}, - {"For Content-Management": expand_repos( - "{repo}/docs/content-manager/learn")}, + {"For Content-Management": expand_repos("{repo}/docs/content-manager/learn")}, {"For Sys-Admins": expand_repos("{repo}/docs/sys-admin/learn")}, ] - reference= [ + reference = [ {"Overview": from_core("docs/sections/reference/index.md")}, - {"Repository Map": from_core( - "docs/sections/reference/01-repository-map.md")}, + {"Repository Map": from_core("docs/sections/reference/01-repository-map.md")}, {"Glossary": from_core("docs/sections/reference/02-glossary.md")}, {"Repositories": expand_reference("{repo}/docs/reference")}, ] - development= [ + development = [ {"Overview": from_core("docs/sections/development/index.md")}, - {"Quickstart": get_children( - from_core("docs/sections/development/quickstart/"))}, - {"Onboarding": get_children( - from_core("docs/sections/development/onboarding/"))}, + { + "Quickstart": get_children( + from_core("docs/sections/development/quickstart/") + ) + }, + { + "Onboarding": get_children( + from_core("docs/sections/development/onboarding/") + ) + }, {"Guides": get_children("core/docs/sections/development/guides/")}, ] # main navigation - navigation= [ + navigation = [ {"Home": "index.md"}, {"Getting Started": getting_started}, {"Guides": guides}, @@ -220,29 +235,31 @@ def define_env(env): """The mkdocs-marcros 'on_configuration' hook. Used to setup the project.""" # === log.info("[pulp-docs] Loading configuration from environ") - base_repolist= os.environ.get("PULPDOCS_BASE_REPOLIST", None) + base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None) log.info(f"{base_repolist=}\n") log.info("[pulp-docs] Loading repolist file") if base_repolist == "testing": - repos= Repos.test_fixtures() + repos = Repos.test_fixtures() else: - repos= Repos.from_yaml(Path(base_repolist)) + repos = Repos.from_yaml(Path(base_repolist)) log.info(f"{repos}") # === log.info("[pulp-docs] Preparing repositories") - TMPDIR= create_clean_tmpdir() - docs_dir, source_dir= prepare_repositories(TMPDIR, repos) + TMPDIR = create_clean_tmpdir() + docs_dir, source_dir = prepare_repositories(TMPDIR, repos) # === log.info("[pulp-docs] Configuring mkdocstrings") - code_sources= [str(source_dir / repo.name) for repo in repos.all] - env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"]["paths"]= code_sources + code_sources = [str(source_dir / repo.name) for repo in repos.all] + env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"][ + "paths" + ] = code_sources # === log.info("[pulp-docs] Configuring navigation") - env.conf["docs_dir"]= docs_dir - env.conf["nav"]= get_navigation(docs_dir, repos) + env.conf["docs_dir"] = docs_dir + env.conf["nav"] = get_navigation(docs_dir, repos) log.info("[pulp-docs] Done with pulp-docs.") diff --git a/src/pulp_docs/plugin_repos.py b/src/pulp_docs/plugin_repos.py index 0e19271..8ee3450 100644 --- a/src/pulp_docs/plugin_repos.py +++ b/src/pulp_docs/plugin_repos.py @@ -18,7 +18,7 @@ import httpx import yaml -log = logging.getLogger('mkdocs') +log = logging.getLogger("mkdocs") FIXTURE_WORKDIR = Path("tests/fixtures").absolute() RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html" @@ -31,6 +31,7 @@ class Repo: The real repository content is plugin sourcecode and markdown documentation. """ + title: str name: str owner: str = "pulp" @@ -58,8 +59,11 @@ def download(self, dest_dir: Path) -> Path: """ # Copy from local filesystem if self.local_basepath is not None: - shutil.copytree(self.local_url, dest_dir, ignore=shutil.ignore_patterns( - "tests", "*venv*", "__pycache__")) + shutil.copytree( + self.local_url, + dest_dir, + ignore=shutil.ignore_patterns("tests", "*venv*", "__pycache__"), + ) return self.local_basepath # or Download from remote @@ -77,7 +81,10 @@ def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str): subprocess.run(cmd, check=True) except subprocess.CalledProcessError as e: log.error( - "An error ocurred while trying to download '{name}' source-code:".format(name=name)) + "An error ocurred while trying to download '{name}' source-code:".format( + name=name + ) + ) log.error(f"{e}") log.info("Done.") @@ -88,8 +95,9 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str): Uses GitHub API. """ - latest_release_link_url = "https://api.github.com/repos/{}/{}/releases/latest".format( - owner, name) + latest_release_link_url = ( + "https://api.github.com/repos/{}/{}/releases/latest".format(owner, name) + ) print("Fetching latest release with:", latest_release_link_url) response = httpx.get(latest_release_link_url) @@ -114,6 +122,7 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str): @dataclass class Repos: """A collection of Repos""" + core: Repo content: t.List[Repo] = field(default_factory=list) other: t.List[Repo] = field(default_factory=list) @@ -158,8 +167,7 @@ def test_fixtures(cls): DEFAULT_CORE = Repo("Pulp Core", "core") DEFAULT_CONTENT_REPOS = [ Repo("Rpm Packages", "new_repo1", local_basepath=FIXTURE_WORKDIR), - Repo("Debian Packages", "new_repo2", - local_basepath=FIXTURE_WORKDIR), + Repo("Debian Packages", "new_repo2", local_basepath=FIXTURE_WORKDIR), Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR), ] return Repos(core=DEFAULT_CORE, content=DEFAULT_CONTENT_REPOS)