From c6c9f5e2e08c9d3113be746ae8b7829baa4cee16 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Fri, 15 Nov 2024 14:55:33 -0800 Subject: [PATCH 1/8] feat: Added the ability to use insiders documentation features locally if a PAT is provided with access permissions to the private insiders repos --- docs/generate_api_pages.py | 1 + mkdocs.yml | 5 +- pyproject.toml | 8 ++ scripts/install_insiders_packages.py | 133 +++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 scripts/install_insiders_packages.py diff --git a/docs/generate_api_pages.py b/docs/generate_api_pages.py index 4d197a3c..a157607b 100644 --- a/docs/generate_api_pages.py +++ b/docs/generate_api_pages.py @@ -77,6 +77,7 @@ def sort_paths(path_object: Path) -> Tuple[int, str]: " options:\n" " inherited_members: false\n" " merge_init_into_class: false\n" + " show_inheritance_diagram: false\n" " filters: ['!^_']\n" ) if module_path.parts[-2:] == ("drivers", "__init__"): diff --git a/mkdocs.yml b/mkdocs.yml index ad85e3e1..5c30dd04 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -103,7 +103,7 @@ plugins: options: # General options extensions: [docs/griffe_custom_decorator_labels.py] - show_inheritance_diagram: true +# show_inheritance_diagram: true # INSIDERS FEATURE # TODO: insiders docs: enable show_source: false # a link is included at the top of each page # Headings options heading_level: 1 @@ -124,8 +124,11 @@ plugins: trim_doctest_flags: true docstring_section_style: list merge_init_into_class: true +# relative_crossrefs: true # INSIDERS FEATURE # TODO: insiders docs: enable +# scoped_crossrefs: true # INSIDERS FEATURE # TODO: insiders docs: enable # Signature options line_length: 100 +# modernize_annotations: true # INSIDERS FEATURE # TODO: enable after https://github.com/pawamoy-insiders/griffe/issues/2 is resolved show_signature_annotations: true signature_crossrefs: true separate_signature: true diff --git a/pyproject.toml b/pyproject.toml index 0ae8a008..b301148d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,8 +136,10 @@ mkdocs-spellcheck = "^1.1.0" mkdocstrings = "^0.26.0" mkdocstrings-python = "^1.10.2" nodeenv = "^1.9.1" +packaging = "^24.0" pygments = "^2.17.2" pymdown-extensions = "^10.8.1" +requests = "^2.31.0" symspellpy = "^6.7.7" tomli = "^2.0.1" @@ -465,21 +467,27 @@ commands_pre = [testenv:docs] basepython = {env:DOC_PYTHON_VERSION} +passenv = + GITHUB_PAT deps = -rdocs/requirements.txt commands_pre = nodeenv --python-virtualenv --clean-src + python scripts/install_insiders_packages.py commands = python -c "import shutil; shutil.rmtree('.results_{envname}', ignore_errors=True)" mkdocs --verbose build --site-dir .results_{envname} [testenv:doctests] basepython = {env:DOC_PYTHON_VERSION} +passenv = + GITHUB_PAT deps = -rdocs/requirements.txt -rtests/requirements.txt commands_pre = nodeenv --python-virtualenv --clean-src + python scripts/install_insiders_packages.py commands = pytest -v -k "test_docs" --showlocals --junitxml={tox_root}/.results_{envname}/results.xml --self-contained-html --html={tox_root}/.results_{envname}/results.html """ diff --git a/scripts/install_insiders_packages.py b/scripts/install_insiders_packages.py new file mode 100644 index 00000000..d5b19393 --- /dev/null +++ b/scripts/install_insiders_packages.py @@ -0,0 +1,133 @@ +"""Install insiders packages corresponding to currently installed versions of specified packages.""" + +import importlib.metadata +import logging +import os +import subprocess +import sys + +from typing import List + +import requests + +from packaging.version import parse as parse_version + +logging.basicConfig( + level=getattr(logging, os.getenv("LOGGING_LEVEL", "INFO")), + format="[%(asctime)s] [%(levelname)8s] %(message)s", +) +logger = logging.getLogger(__name__) + +PACKAGE_LIST = { + "mkdocstrings-python": "https://github.com/pawamoy-insiders/mkdocstrings-python", + "griffe": "https://github.com/pawamoy-insiders/griffe", +} + + +def get_github_tags(repo_url_str: str, github_token: str) -> List[str]: + """Get the tags for a GitHub repository. + + Args: + repo_url_str: The URL of the GitHub repository. + github_token: The GitHub token to use for authentication. + + Returns: + A list of tag names for the repository. + """ + # Extract the owner and repository name from the URL + parts = repo_url_str.rstrip("/").split("/") + owner, repo = parts[-2], parts[-1] + + try: + response = requests.get( + f"https://api.github.com/repos/{owner}/{repo}/tags", + headers={"Authorization": f"token {github_token}"}, + timeout=30, + ) + response.raise_for_status() # Raise an exception for HTTP errors + + tags = response.json() + return [tag["name"] for tag in tags] + + except requests.exceptions.RequestException: + logger.error("Error getting tags for %s", repo_url_str) # noqa: TRY400 + return [] + + +def get_newest_matching_tag(current_version: str, tags: List[str]) -> str: + """Find the newest tag that starts with the given current version. + + Args: + current_version: The current version of the package. + tags: A list of tag names. + + Returns: + The newest tag that starts with the current version, or an empty string if none are found. + """ + # Filter tags that start with the current version + matching_tags = [tag for tag in tags if tag.startswith(current_version)] + + # Sort matching tags as version numbers in descending order + matching_tags.sort(key=parse_version, reverse=True) + + return matching_tags[0] if matching_tags else "" + + +def install_package(package_name: str, repo_url_str: str, tag: str, github_token: str) -> None: + """Install a package from a GitHub repository using a specific tag. + + Args: + package_name: The name of the package to install. + repo_url_str: The HTTPS URL of the GitHub repository. + tag: The tag to install. + github_token: The GitHub token to use for authentication. + """ + # Insert the token into the URL for authentication + repo_url_str = repo_url_str.replace("https://", f"https://{github_token}@") + + install_url = f"{repo_url_str}.git@{tag}" + try: + logger.info("Installing %s from tag %s", package_name, tag) + subprocess.check_call( # noqa: S603 + [sys.executable, "-m", "pip", "install", f"git+{install_url}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + logger.info("Successfully installed %s from tag %s", package_name, tag) + except subprocess.CalledProcessError: + logger.error("Failed to install %s from tag %s", package_name, tag) # noqa: TRY400 + + +def main() -> None: + """Install insiders packages.""" + if not (github_token := os.environ.get("GITHUB_PAT")): + logger.info( + "GITHUB_PAT environment variable is not set, no insiders packages will be installed." + ) + return + for package, repo_url in PACKAGE_LIST.items(): + logger.info("Processing %s", package) + try: + version = importlib.metadata.version(package) + logger.info("%s installed version is %s", package, version) + except importlib.metadata.PackageNotFoundError: + logger.warning("%s is not installed.", package) + version = None + + if version and (package_tags := get_github_tags(repo_url, github_token)): + logger.info("Tags for %s: %s", package, package_tags) + + if newest_tag := get_newest_matching_tag(version, package_tags): + logger.info( + 'Newest matching tag for %s version %s is "%s"', + package, + version, + newest_tag, + ) + install_package(package, repo_url, newest_tag, github_token) + else: + logger.info("No matching tags found for %s with version %s", package, version) + + +if __name__ == "__main__": + main() From 7e95af0c3e7978b5206f7aadbfe1b8aee39c1ca3 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Mon, 18 Nov 2024 09:16:41 -0800 Subject: [PATCH 2/8] chore: Added a new tox env to export requirements for tests/docs --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ae64959c..97c8238a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -489,4 +489,10 @@ commands_pre = python scripts/install_insiders_packages.py commands = pytest -v -k "test_docs" --showlocals --junitxml={tox_root}/.results_{envname}/results.xml --self-contained-html --html={tox_root}/.results_{envname}/results.html + +[testenv:export-reqs] +commands = + poetry export --without-hashes --without-urls --all-extras --only=docs --output=docs/requirements.txt + poetry export --without-hashes --without-urls --all-extras --only=tests --output=tests/requirements.txt + - pre-commit run -a requirements-txt-fixer """ From ed733fb16372c3b0af00d4d426746ee483a1f401 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Mon, 18 Nov 2024 10:11:08 -0800 Subject: [PATCH 3/8] chore: Added a new tox env to export requirements for tests/docs --- .../mkdocstrings/python/readthedocs/children.html.jinja | 6 ++---- docs/known_words.txt | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/_templates/mkdocstrings/python/readthedocs/children.html.jinja b/docs/_templates/mkdocstrings/python/readthedocs/children.html.jinja index 324e1819..11d86192 100644 --- a/docs/_templates/mkdocstrings/python/readthedocs/children.html.jinja +++ b/docs/_templates/mkdocstrings/python/readthedocs/children.html.jinja @@ -16,13 +16,11 @@ Context: {%- for item in item_list %} {%- if item.docstring %} - {%- filter stash_crossref(length=item.canonical_path|length) -%} - + {{ item.name }} - + - {%- endfilter -%} {{ item.docstring.parsed[0].value.split("\n")[0]|convert_markdown(heading_level, html_id) }} {%- endif %} diff --git a/docs/known_words.txt b/docs/known_words.txt index b90d6485..0d86ce28 100644 --- a/docs/known_words.txt +++ b/docs/known_words.txt @@ -104,6 +104,7 @@ tm_devices tmdevicessupport toml usb +venv walkthrough www yaml From 214203e79d53963c8c3883c119571b409d98b103 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Tue, 19 Nov 2024 10:50:38 -0800 Subject: [PATCH 4/8] docs: Add template for mermaid diagrams --- .../python/readthedocs/class.html.jinja | 49 +++++++++++++++++++ mkdocs.yml | 8 +-- 2 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 docs/_templates/mkdocstrings/python/readthedocs/class.html.jinja diff --git a/docs/_templates/mkdocstrings/python/readthedocs/class.html.jinja b/docs/_templates/mkdocstrings/python/readthedocs/class.html.jinja new file mode 100644 index 00000000..bcdccc4f --- /dev/null +++ b/docs/_templates/mkdocstrings/python/readthedocs/class.html.jinja @@ -0,0 +1,49 @@ +{# TODO: Drop Python 3.8: remove this file after updating to newer versions of mkdocstrings-python #} +{% extends "_base/class.html.jinja" %} + {% block inheritance_diagram %} + {#- Inheritance diagram block. + + This block renders the inheritance diagram for the class, + using Mermaid syntax and a bit of JavaScript to make the nodes clickable, + linking to the corresponding class documentation. + -#} + {% if config.show_inheritance_diagram and class.bases %} + {% macro edges(class) %} + {% for base in class.resolved_bases %} + {{ base.path }} --> {{ class.path }} + {{ edges(base) }} + {% endfor %} + {% endmacro %} +
+ + {% for base in class.mro() %} + + {% endfor %} +
+
+ flowchart LR + {{ class.path }}[{{ class.name }}] + {% for base in class.mro() %} + {{ base.path }}[{{ base.name }}] + {% endfor %} + + {{ edges(class) | safe }} + + click {{ class.path }} href "" "{{ class.path }}" + {% for base in class.mro() %} + click {{ base.path }} href "" "{{ base.path }}" + {% endfor %} +
+ + {% endif %} + {% endblock inheritance_diagram %} diff --git a/mkdocs.yml b/mkdocs.yml index 5c30dd04..bd115d36 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -103,7 +103,7 @@ plugins: options: # General options extensions: [docs/griffe_custom_decorator_labels.py] -# show_inheritance_diagram: true # INSIDERS FEATURE # TODO: insiders docs: enable + show_inheritance_diagram: true # INSIDERS FEATURE show_source: false # a link is included at the top of each page # Headings options heading_level: 1 @@ -124,11 +124,11 @@ plugins: trim_doctest_flags: true docstring_section_style: list merge_init_into_class: true -# relative_crossrefs: true # INSIDERS FEATURE # TODO: insiders docs: enable -# scoped_crossrefs: true # INSIDERS FEATURE # TODO: insiders docs: enable + relative_crossrefs: true # INSIDERS FEATURE + scoped_crossrefs: true # INSIDERS FEATURE # Signature options line_length: 100 -# modernize_annotations: true # INSIDERS FEATURE # TODO: enable after https://github.com/pawamoy-insiders/griffe/issues/2 is resolved + modernize_annotations: false # INSIDERS FEATURE show_signature_annotations: true signature_crossrefs: true separate_signature: true From b6da2b14caa48abf91cdaf605ec4077c11e792c6 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Wed, 20 Nov 2024 16:43:25 -0800 Subject: [PATCH 5/8] docs: Add insiders package installation script to official documentation build process --- .readthedocs.yml | 3 +++ scripts/install_insiders_packages.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 14947713..74007c29 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,9 @@ build: tools: python: '3.11' # This needs to stay in sync with the tox config in pyproject.toml and any CI scripts nodejs: '20' # This needs to stay in sync with any CI scripts + jobs: + post_install: + - FAIL_ON_ACCESS_ERROR=true python scripts/install_insiders_packages.py python: install: - requirements: docs/requirements.txt diff --git a/scripts/install_insiders_packages.py b/scripts/install_insiders_packages.py index d5b19393..ece6a625 100644 --- a/scripts/install_insiders_packages.py +++ b/scripts/install_insiders_packages.py @@ -22,6 +22,7 @@ "mkdocstrings-python": "https://github.com/pawamoy-insiders/mkdocstrings-python", "griffe": "https://github.com/pawamoy-insiders/griffe", } +FAIL_ON_ACCESS_ERROR = bool(os.getenv("FAIL_ON_ACCESS_ERROR")) def get_github_tags(repo_url_str: str, github_token: str) -> List[str]: @@ -127,6 +128,9 @@ def main() -> None: install_package(package, repo_url, newest_tag, github_token) else: logger.info("No matching tags found for %s with version %s", package, version) + elif FAIL_ON_ACCESS_ERROR: + msg = f"\nFailed to access {repo_url}, exiting with error" + raise SystemExit(msg) if __name__ == "__main__": From a7ffdd1270c75b90e2d7d8fa7e179d96564d2408 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Thu, 21 Nov 2024 07:40:36 -0800 Subject: [PATCH 6/8] chore: Update comment in mkdocs.yml file --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index bd115d36..e8f4ec13 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,7 +128,7 @@ plugins: scoped_crossrefs: true # INSIDERS FEATURE # Signature options line_length: 100 - modernize_annotations: false # INSIDERS FEATURE + modernize_annotations: false # INSIDERS FEATURE (if the source code annotations format changes, update this to true) show_signature_annotations: true signature_crossrefs: true separate_signature: true From 7da8041ba457260334b858754b1060ffa4b8367b Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Thu, 21 Nov 2024 07:42:06 -0800 Subject: [PATCH 7/8] fix: Update script to raise an error if no PAT is available --- scripts/install_insiders_packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/install_insiders_packages.py b/scripts/install_insiders_packages.py index ece6a625..9e4de234 100644 --- a/scripts/install_insiders_packages.py +++ b/scripts/install_insiders_packages.py @@ -102,6 +102,9 @@ def install_package(package_name: str, repo_url_str: str, tag: str, github_token def main() -> None: """Install insiders packages.""" if not (github_token := os.environ.get("GITHUB_PAT")): + if FAIL_ON_ACCESS_ERROR: + msg = "\nGITHUB_PAT environment variable is not set, exiting with error" + raise SystemExit(msg) logger.info( "GITHUB_PAT environment variable is not set, no insiders packages will be installed." ) From 7237e3df1b1e2f3acdf23de0c1683a2b9a137fa9 Mon Sep 17 00:00:00 2001 From: "Felt, Nicholas" Date: Thu, 21 Nov 2024 10:45:31 -0800 Subject: [PATCH 8/8] docs: Specify the javascript to ensure locally building docs doesn't have any errors. Also, removed the FAIL_ON_ACCESS_ERROR env var from the insiders installation script, since the token isn't available for PR builds on ReadtheDocs. --- mkdocs.yml | 2 +- scripts/install_insiders_packages.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index e8f4ec13..151807b5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,7 +91,7 @@ plugins: inline_refs: none markdown_links: true - mermaid2: - version: 11.3.0 + javascript: https://unpkg.com/mermaid@11.3.0/dist/mermaid.min.js - mkdocstrings: # additional customization takes place in docs/generate_api_pages.py # noinspection YAMLIncompatibleTypes enabled: !ENV [TM_DEVICES_API_GENERATION, true] diff --git a/scripts/install_insiders_packages.py b/scripts/install_insiders_packages.py index 9e4de234..d5b19393 100644 --- a/scripts/install_insiders_packages.py +++ b/scripts/install_insiders_packages.py @@ -22,7 +22,6 @@ "mkdocstrings-python": "https://github.com/pawamoy-insiders/mkdocstrings-python", "griffe": "https://github.com/pawamoy-insiders/griffe", } -FAIL_ON_ACCESS_ERROR = bool(os.getenv("FAIL_ON_ACCESS_ERROR")) def get_github_tags(repo_url_str: str, github_token: str) -> List[str]: @@ -102,9 +101,6 @@ def install_package(package_name: str, repo_url_str: str, tag: str, github_token def main() -> None: """Install insiders packages.""" if not (github_token := os.environ.get("GITHUB_PAT")): - if FAIL_ON_ACCESS_ERROR: - msg = "\nGITHUB_PAT environment variable is not set, exiting with error" - raise SystemExit(msg) logger.info( "GITHUB_PAT environment variable is not set, no insiders packages will be installed." ) @@ -131,9 +127,6 @@ def main() -> None: install_package(package, repo_url, newest_tag, github_token) else: logger.info("No matching tags found for %s with version %s", package, version) - elif FAIL_ON_ACCESS_ERROR: - msg = f"\nFailed to access {repo_url}, exiting with error" - raise SystemExit(msg) if __name__ == "__main__":