diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e98ae308..a4162349d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Noop run: "true" checks: - name: tox -e format-check,lint-check,typecheck,vendor-check,package + name: tox -e format-check,lint-check,typecheck,vendor-check,package,docs needs: org-check runs-on: ubuntu-22.04 steps: @@ -39,10 +39,23 @@ jobs: with: # We need to keep Python 3.8 for consistent vendoring with tox. python-version: "3.8" - - name: Check Formatting, Lints, Types, Vendoring and Packaging + - name: Check Formatting, Lints and Types uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 with: - tox-env: format-check,lint-check,typecheck,vendor-check,package -- --additional-format sdist --additional-format wheel + tox-env: format-check,lint-check,typecheck + - name: Check Vendoring + uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 + with: + tox-env: vendor-check + - name: Check Packaging + uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 + with: + tox-env: package -- --additional-format sdist --additional-format wheel --embed-docs + - name: Check Docs + uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 + with: + tox-env: docs -- --linkcheck --pdf --clean-html + # N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In # particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs. linux-tests: diff --git a/.github/workflows/doc-site.yml b/.github/workflows/doc-site.yml new file mode 100644 index 000000000..50bca6813 --- /dev/null +++ b/.github/workflows/doc-site.yml @@ -0,0 +1,42 @@ +name: Deploy Doc Site +on: + push: + tags: + - v[0-9]+.[0-9]+.[0-9]+ + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout Pex + uses: actions/checkout@v3 + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Build Doc Site + uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 + with: + tox-env: docs -- --linkcheck --pdf --clean-html + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + path: "dist/docs/html/" + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 302aee84a..e70eb2e94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: - name: Build sdist and wheel uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 with: - tox-env: package -- --no-pex --additional-format sdist --additional-format wheel + tox-env: package -- --no-pex --additional-format sdist --additional-format wheel --embed-docs - name: Publish Pex ${{ needs.determine-tag.outputs.release-tag }} uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -83,7 +83,11 @@ jobs: - name: Package Pex ${{ needs.determine-tag.outputs.release-tag }} PEX uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 with: - tox-env: package + tox-env: package -- --embed-docs + - name: Generate Pex ${{ needs.determine-tag.outputs.release-tag }} PDF + uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 + with: + tox-env: docs -- --no-html --pdf - name: Prepare Changelog id: prepare-changelog uses: a-scie/actions/changelog@v1.5 @@ -100,6 +104,8 @@ jobs: body_path: ${{ steps.prepare-changelog.outputs.changelog-file }} draft: false prerelease: false - files: dist/pex + files: | + dist/pex + dist/docs/pdf/pex.pdf fail_on_unmatched_files: true discussion_category_name: Announcements diff --git a/.gitignore b/.gitignore index 96f732fee..c7a905086 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,5 @@ /.mypy_cache/ /.tox/ /dist/ -/docs/_build +/docs/_static_dynamic/ diff --git a/CHANGES.md b/CHANGES.md index 827ba5da9..f04414489 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,18 @@ # Release Notes +## 2.1.164 + +This release moves Pex documentation from https://pex.readthedocs.io to +https://docs.pex-tool.org. While legacy versioned docs will remain +available at RTD in perpetuity, going forward only the latest Pex +release docs will be available online at the https://docs.pex-tool.org +site. If you want to see the Pex docs for the version you are currently +using, Pex now supports the `pex3 docs` command which will serve the +docs for your Pex version locally, offline, but with full functionality, +including search. + +* Re-work Pex documentation. (#2362) + ## 2.1.163 This release fixes Pex to work in certain OS / SSL environments where it @@ -567,7 +580,7 @@ In addition, you can now "inject" runtime environment variables and arguments into PEX files such that, when run, the PEX runtime ensures those environment variables and command line arguments are passed to the PEXed application. See [PEX Recipes]( -https://pex.readthedocs.io/en/latest/recipes.html#uvicorn-and-other-customizable-application-servers +https://docs.pex-tool.org/recipes.html#uvicorn-and-other-customizable-application-servers ) for more information. @@ -1021,7 +1034,7 @@ PEX venvs (More on additional data files as well as a new venv install `--scope` that can be used to create fully optimized container images with PEXed applications (See how to use this feature -[here](https://pex.readthedocs.io/en/latest/recipes.html#pex-app-in-a-container)). +[here](https://docs.pex-tool.org/recipes.html#pex-app-in-a-container)). * Support splitting venv creation into deps & srcs. (#1634) * Fix handling of data files when creating venvs. (#1632) @@ -1052,7 +1065,7 @@ pre-built wheels are available for that foreign platform. Additionally, PEXes now know how to set a usable process name when the PEX contains the `setproctitle` distribution. See -[here](https://pex.readthedocs.io/en/v2.1.66/recipes.html#long-running-pex-applications-and-daemons) +[here](https://docs.pex-tool.org/recipes.html#long-running-pex-applications-and-daemons) for more information. * Add support for `--complete-platform`. (#1609) @@ -1148,7 +1161,7 @@ creating optimized container images from PEX files. ## 2.1.55 This release brings official support for Python 3.10 as well as fixing - doc generation and fixing help for + doc generation and fixing help for `pex-tools` / `PEX_TOOLS=1 ./my.pex` pex tools invocations that have too few arguments. @@ -1547,7 +1560,7 @@ This release improves interpreter discovery to prefer more recent patch versions, e.g. preferring Python 3.6.10 over 3.6.8. We recently regained access to the docsite, and - is now up-to-date. + is now up-to-date. * Prefer more recent patch versions in interpreter discovery. (#1088) * Fix `--pex-python` when it's the same as the current interpreter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ad6da2d6..f1eb6bdc5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,12 @@ lead to a better end result. Before sending off changes you should run `tox -e fmt,lint,check`. This formats, lints and type checks the code. -In addition you should run tests, which are divided into integration tests (those under +If you've made `docs/` changes, you should run `tox -e docs -- --linkcheck --pdf --serve` which +will build the full doc site including its downloadable PDF version as well as the search index. +You can browse to the URL printed out at the end of the output to view the site on your local +machine. + +In addition, you should run tests, which are divided into integration tests (those under `tests/integration/`) and unit tests (those under `tests/` but not under `tests/integration/`). Unit tests have a tox environment name that matches the desired interpreter to test against. So, to run unit tests against CPython 3.11 (which you must have installed), use `tox -e py311`. For @@ -56,3 +61,14 @@ without having to actually install PyPy 2.7 on your machine. When you're ready to get additional eyes on your changes, submit a [pull request]( https://github.com/pex-tool/pex/pulls). +If you've made documentation changes you can render the site in the fork you used for the pull +request by navigating to the "Deploy Doc Site" action in your fork and running the workflow +manually. You do this using the "Run workflow" widget in the top row of the workflow run list, +selecting your PR branch and clicking "Run workflow". This will fail your first time doing this due +to a branch protection rule the "Deploy Doc Site" action automatically establishes to restrict doc +site deployments to the main branch. To fix this, navigate to "Environments" in your fork settings +and edit the "github-pages" branch protection rule, changing "Deployment Branches" from +"Selected branches" to "All branches" and then save the protection rules. You can now re-run the +workflow and should be able to browse to https://.github.io/pex to browse the +deployed site with your changes incorporated. N.B.: The site will be destroyed when you delete your +PR branch. diff --git a/README.rst b/README.rst index 836a21f78..95da09d62 100644 --- a/README.rst +++ b/README.rst @@ -138,7 +138,7 @@ Documentation ============= More documentation about Pex, building .pex files, and how .pex files work -is available at https://pex.readthedocs.io. +is available at https://docs.pex-tool.org. Development diff --git a/assets/download.svg b/assets/download.svg new file mode 100644 index 000000000..f06ce1fc4 --- /dev/null +++ b/assets/download.svg @@ -0,0 +1,4 @@ + + + + diff --git a/assets/github.svg b/assets/github.svg new file mode 100644 index 000000000..5e7b2e0a9 --- /dev/null +++ b/assets/github.svg @@ -0,0 +1,4 @@ + + + + diff --git a/assets/pdf.svg b/assets/pdf.svg new file mode 100644 index 000000000..54a4d78ad --- /dev/null +++ b/assets/pdf.svg @@ -0,0 +1,4 @@ + + + + diff --git a/assets/pex-full-dark.png b/assets/pex-full-dark.png new file mode 100644 index 000000000..d3a4bd0dd Binary files /dev/null and b/assets/pex-full-dark.png differ diff --git a/assets/pex-full-light.png b/assets/pex-full-light.png new file mode 100644 index 000000000..539c2fe82 Binary files /dev/null and b/assets/pex-full-light.png differ diff --git a/assets/pex-full.svg b/assets/pex-full.svg new file mode 100644 index 000000000..3494c02e7 --- /dev/null +++ b/assets/pex-full.svg @@ -0,0 +1,73 @@ + + + + + + + + ex + + + + diff --git a/assets/pex-icon-512.png b/assets/pex-icon-512.png new file mode 100644 index 000000000..98a164557 Binary files /dev/null and b/assets/pex-icon-512.png differ diff --git a/assets/pex-icon-512x512.png b/assets/pex-icon-512x512.png new file mode 100644 index 000000000..7b504884b Binary files /dev/null and b/assets/pex-icon-512x512.png differ diff --git a/assets/pex.ico b/assets/pex.ico new file mode 100644 index 000000000..bcd0c3ab9 Binary files /dev/null and b/assets/pex.ico differ diff --git a/assets/python.svg b/assets/python.svg new file mode 100644 index 000000000..3d35fb51b --- /dev/null +++ b/assets/python.svg @@ -0,0 +1,4 @@ + + + + diff --git a/build-backend/pex_build/__init__.py b/build-backend/pex_build/__init__.py index 87fb2ed9a..05794947f 100644 --- a/build-backend/pex_build/__init__.py +++ b/build-backend/pex_build/__init__.py @@ -1,2 +1,11 @@ # Copyright 2024 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os + +# When running under MyPy, this will be set to True for us automatically; so we can use it as a typing module import +# guard to protect Python 2 imports of typing - which is not normally available in Python 2. +TYPE_CHECKING = False + + +INCLUDE_DOCS = os.environ.get("__PEX_BUILD_INCLUDE_DOCS__", "False").lower() in ("1", "true") diff --git a/build-backend/pex_build/hatchling/build.py b/build-backend/pex_build/hatchling/build.py index cf00f622e..25985bbe1 100644 --- a/build-backend/pex_build/hatchling/build.py +++ b/build-backend/pex_build/hatchling/build.py @@ -1,7 +1,29 @@ # Copyright 2024 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import +from __future__ import absolute_import, print_function + +import hatchling.build +import pex_build # We re-export all hatchling's PEP-517 build backend hooks here for the build frontend to call. from hatchling.build import * # NOQA + +if pex_build.TYPE_CHECKING: + from typing import Any, Dict, List, Optional + + +def get_requires_for_build_wheel(config_settings=None): + # type: (Optional[Dict[str, Any]]) -> List[str] + + reqs = hatchling.build.get_requires_for_build_wheel( + config_settings=config_settings + ) # type: List[str] + if pex_build.INCLUDE_DOCS: + with open("docs-requirements.txt") as fp: + for raw_req in fp.readlines(): + req = raw_req.strip() + if not req or req.startswith("#"): + continue + reqs.append(req) + return reqs diff --git a/build-backend/pex_build/hatchling/build_hook.py b/build-backend/pex_build/hatchling/build_hook.py new file mode 100644 index 000000000..1b89faeda --- /dev/null +++ b/build-backend/pex_build/hatchling/build_hook.py @@ -0,0 +1,38 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, print_function + +import os.path +import subprocess +import sys + +import pex_build +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +if pex_build.TYPE_CHECKING: + from typing import Any, Dict + + +class AdjustBuild(BuildHookInterface): + """Allows alteration of the build process.""" + + PLUGIN_NAME = "pex-adjust-build" + + def initialize( + self, + version, # type: str + build_data, # type: Dict[str, Any] + ): + # type: (...) -> None + if pex_build.INCLUDE_DOCS: + out_dir = os.path.join(self.root, "dist", "docs") + subprocess.check_call( + args=[ + sys.executable, + os.path.join(self.root, "scripts", "build_docs.py"), + "--clean-html", + out_dir, + ] + ) + build_data["force_include"][out_dir] = os.path.join("pex", "docs") diff --git a/build-backend/pex_build/hatchling/dynamic_requires_python.py b/build-backend/pex_build/hatchling/dynamic_requires_python.py deleted file mode 100644 index 666581ad2..000000000 --- a/build-backend/pex_build/hatchling/dynamic_requires_python.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Pex project contributors. -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, print_function - -import os -import sys -from typing import Any, Dict - -from hatchling.metadata.plugin.interface import MetadataHookInterface - - -class DynamicRequiresPythonHook(MetadataHookInterface): - """Allows dynamically specifying requires-python metadata via _PEX_REQUIRES_PYTHON env var.""" - - PLUGIN_NAME = "pex-dynamic-requires-python" - - def update(self, metadata): - # type: (Dict[str, Any]) -> None - requires_python = os.environ.get("_PEX_REQUIRES_PYTHON") - if requires_python: - print( - "pex_build: Dynamically modifying pyproject.toml requires-python of {original} to " - "{dynamic}".format(original=metadata["requires-python"], dynamic=requires_python), - file=sys.stderr, - ) - metadata["requires-python"] = requires_python diff --git a/build-backend/pex_build/hatchling/hooks.py b/build-backend/pex_build/hatchling/hooks.py index 2a90079c4..ea8305c24 100644 --- a/build-backend/pex_build/hatchling/hooks.py +++ b/build-backend/pex_build/hatchling/hooks.py @@ -3,13 +3,22 @@ from __future__ import absolute_import -from typing import Type - +import pex_build from hatchling.plugin import hookimpl -from pex_build.hatchling.dynamic_requires_python import DynamicRequiresPythonHook +from pex_build.hatchling.build_hook import AdjustBuild +from pex_build.hatchling.metadata_hook import AdjustMetadata + +if pex_build.TYPE_CHECKING: + from typing import Type @hookimpl def hatch_register_metadata_hook(): - # type: () -> Type[DynamicRequiresPythonHook] - return DynamicRequiresPythonHook + # type: () -> Type[AdjustMetadata] + return AdjustMetadata + + +@hookimpl +def hatch_register_build_hook(): + # type: () -> Type[AdjustBuild] + return AdjustBuild diff --git a/build-backend/pex_build/hatchling/metadata_hook.py b/build-backend/pex_build/hatchling/metadata_hook.py new file mode 100644 index 000000000..196f9d4d4 --- /dev/null +++ b/build-backend/pex_build/hatchling/metadata_hook.py @@ -0,0 +1,61 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, print_function + +import os +import sys + +import pex_build +from hatchling.metadata.plugin.interface import MetadataHookInterface + +if pex_build.TYPE_CHECKING: + from typing import Any, Dict + + +def expand_value( + value, # type: Any + **fmt # type: str +): + # type: (...) -> Any + if isinstance(value, str): + return value.format(**fmt) + if isinstance(value, list): + return [expand_value(val) for val in value] + if isinstance(value, dict): + return {key: expand_value(value, **fmt) for key, value in value.items()} + return value + + +class AdjustMetadata(MetadataHookInterface): + """Allows modifying project metadata. + + The following mutations are supported: + + Specifying alternate requires-python metadata via _PEX_REQUIRES_PYTHON env var. + + Expanding format string placeholder (`{name}`) with metadata values via the `expand` mapping of placeholder name + to metadata value. + """ + + PLUGIN_NAME = "pex-adjust-metadata" + + def update(self, metadata): + # type: (Dict[str, Any]) -> None + requires_python = os.environ.get("_PEX_REQUIRES_PYTHON") + if requires_python: + print( + "pex_build: Dynamically modifying pyproject.toml requires-python of {original} to " + "{dynamic}".format(original=metadata["requires-python"], dynamic=requires_python), + file=sys.stderr, + ) + metadata["requires-python"] = requires_python + + expand = self.config.get("expand") + if expand: + metadata.update( + ( + key, + expand_value(value, **{key: metadata[value] for key, value in expand.items()}), + ) + for key, value in metadata.items() + if key != "version" + ) diff --git a/docs-requirements.txt b/docs-requirements.txt new file mode 100644 index 000000000..6c95b3098 --- /dev/null +++ b/docs-requirements.txt @@ -0,0 +1,5 @@ +furo +httpx +myst-parser[linkify] +sphinx +sphinx-simplepdf \ No newline at end of file diff --git a/docs/_ext/sphinx_pex/__init__.py b/docs/_ext/sphinx_pex/__init__.py new file mode 100644 index 000000000..dd6b8b944 --- /dev/null +++ b/docs/_ext/sphinx_pex/__init__.py @@ -0,0 +1,71 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path, PurePath +from typing import Any, Dict, List, Optional + +from sphinx.application import Sphinx +from sphinx_pex.vars import Vars + +PROJECT_ROOT = Path(__file__).parent.parent.parent.parent +ICON_ASSETS = PROJECT_ROOT / "assets" +DOC_ROOT = PROJECT_ROOT / "docs" +STATIC_ASSETS = [DOC_ROOT / "_static", DOC_ROOT / "_static_dynamic"] + + +def html_static_path() -> List[str]: + return [ + static_asset_root.relative_to(DOC_ROOT).as_posix() for static_asset_root in STATIC_ASSETS + ] + + +@dataclass(frozen=True) +class SVGIcon: + @classmethod + def load(cls, name: str, icon_asset: PurePath, url: str, css_class: str = "") -> SVGIcon: + return cls( + name=name, url=url, html=(ICON_ASSETS / icon_asset).read_text(), css_class=css_class + ) + + @classmethod + def load_if_static_asset_exists( + cls, name: str, icon_asset: PurePath, static_asset: PurePath, css_class: str = "" + ) -> Optional[SVGIcon]: + for static_asset_root in STATIC_ASSETS: + static_asset_file = static_asset_root / static_asset + if static_asset_file.is_file(): + return cls.load( + name=name, + icon_asset=icon_asset, + # N.B.: No matter where in the `html_static_path` the static asset comes from, its destination in + # the generated doc site will be the `_static/` dir. + url=(PurePath("_static") / static_asset).as_posix(), + css_class=css_class, + ) + return None + + name: str + url: str + html: str + css_class: str = "" + + def as_furo_footer_icon(self) -> Dict[str, str]: + return { + "name": self.name, + "url": self.url, + "html": self.html, + "class": self.css_class, + } + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.add_directive("vars", Vars) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/_ext/vars.py b/docs/_ext/sphinx_pex/vars.py similarity index 78% rename from docs/_ext/vars.py rename to docs/_ext/sphinx_pex/vars.py index c39a2c282..433bbd271 100644 --- a/docs/_ext/vars.py +++ b/docs/_ext/sphinx_pex/vars.py @@ -1,6 +1,8 @@ # Copyright 2020 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). +from typing import List + from docutils import nodes, statemachine from docutils.parsers.rst import Directive from sphinx import addnodes @@ -10,7 +12,7 @@ class Vars(Directive): - def convert_rst_to_nodes(self, rst_source): + def convert_rst_to_nodes(self, rst_source: str) -> List[nodes.Node]: """Turn an RST string into a node that can be used in the document.""" node = nodes.Element() node.document = self.state.document @@ -19,8 +21,8 @@ def convert_rst_to_nodes(self, rst_source): ) return node.children - def run(self): - def make_nodes(var_name): + def run(self) -> List[nodes.Node]: + def make_nodes(var_name: str) -> List[nodes.Node]: var_obj = Variables.__dict__[var_name] if isinstance(var_obj, DefaultedProperty): desc_str = var_obj._func.__doc__ @@ -33,18 +35,10 @@ def make_nodes(var_name): sig.append(nodes.target("", "", ids=[var_name])) sig.append(addnodes.desc_signature(var_name, var_name)) - return [sig] + self.convert_rst_to_nodes(desc_str) + var_nodes = [sig] # type: List[nodes.Node] + var_nodes.extend(self.convert_rst_to_nodes(desc_str)) + return var_nodes return [ node for var in dir(Variables) if var.startswith("PEX_") for node in make_nodes(var) ] - - -def setup(app): - app.add_directive("vars", Vars) - - return { - "version": "0.1", - "parallel_read_safe": True, - "parallel_write_safe": True, - } diff --git a/docs/_static/pex-cover.png b/docs/_static/pex-cover.png new file mode 120000 index 000000000..22dcab8fb --- /dev/null +++ b/docs/_static/pex-cover.png @@ -0,0 +1 @@ +../../assets/pex-icon-512.png \ No newline at end of file diff --git a/docs/_static/pex-full-dark.png b/docs/_static/pex-full-dark.png new file mode 120000 index 000000000..b2d8376da --- /dev/null +++ b/docs/_static/pex-full-dark.png @@ -0,0 +1 @@ +../../assets/pex-full-dark.png \ No newline at end of file diff --git a/docs/_static/pex-full-light.png b/docs/_static/pex-full-light.png new file mode 120000 index 000000000..d0e6eca9f --- /dev/null +++ b/docs/_static/pex-full-light.png @@ -0,0 +1 @@ +../../assets/pex-full-light.png \ No newline at end of file diff --git a/docs/_static/pex.ico b/docs/_static/pex.ico new file mode 120000 index 000000000..069050e0c --- /dev/null +++ b/docs/_static/pex.ico @@ -0,0 +1 @@ +../../assets/pex.ico \ No newline at end of file diff --git a/docs/_templates/search.html b/docs/_templates/search.html new file mode 100644 index 000000000..629f7c8f8 --- /dev/null +++ b/docs/_templates/search.html @@ -0,0 +1,54 @@ +{% extends "page.html" %} + +{%- block htmltitle -%} +{{ _("Search") }} - {{ docstitle }} +{%- endblock htmltitle -%} + +{% block content %} +

{{ _("Search") }}

+ +{% endblock %} + +{% block scripts -%} +{{ super() }} + + +{%- endblock scripts %} + +{% block extra_styles -%} +{{ super() }} + +{%- endblock extra_styles %} + diff --git a/docs/api/vars.md b/docs/api/vars.md new file mode 100644 index 000000000..f3092cb0e --- /dev/null +++ b/docs/api/vars.md @@ -0,0 +1,5 @@ +(vars)= +# PEX runtime environment variables + +```{vars} +``` diff --git a/docs/api/vars.rst b/docs/api/vars.rst deleted file mode 100644 index c4d5e5f50..000000000 --- a/docs/api/vars.rst +++ /dev/null @@ -1,4 +0,0 @@ -PEX runtime environment variables -================================= - -.. vars:: diff --git a/docs/buildingpex.rst b/docs/buildingpex.rst index 10682462e..40a11103e 100644 --- a/docs/buildingpex.rst +++ b/docs/buildingpex.rst @@ -435,9 +435,7 @@ Tailoring PEX execution at runtime ---------------------------------- Tailoring of PEX execution can be done at runtime by setting various environment variables. -The source of truth for these environment variables can be found in the -`pex.variables API `_. - +See :ref:`vars`. Using ``bdist_pex`` =================== diff --git a/docs/conf.py b/docs/conf.py index 779bff35b..c6462cf92 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,279 +1,110 @@ -# -*- coding: utf-8 -*- -# -# pex documentation build configuration file, created by -# sphinx-quickstart on Fri Jul 25 15:16:37 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# Configuration file for the Sphinx documentation builder. # -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -import os import sys from datetime import datetime +from pathlib import PurePath + +sys.path.insert(0, str(PurePath(__file__).parent.parent)) -sys.path.insert(0, os.path.abspath("..")) -sys.path.append(os.path.abspath("./_ext")) +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information # Note: must come after the sys.path manipulation above. from pex.version import __version__ as PEX_VERSION # isort:skip +project = "pex" +version = ".".join(PEX_VERSION.split(".")[:2]) +release = PEX_VERSION +copyright = f"{datetime.now().year}, Pex project contributors" +author = "Pex project contributors" -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. +# Note: the vars extension is housed in _ext. +sys.path.insert(0, str(PurePath(__file__).parent / "_ext")) extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.viewcode", - "vars", + "myst_parser", + "sphinx_pex", ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"pex" -copyright = u"%s, Pants project contributors" % datetime.now().year - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = ".".join(PEX_VERSION.split(".")[0:2]) - -# The full version, including alpha/beta/rc tags. -release = PEX_VERSION - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. - -try: - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except ImportError: - html_theme = "default" - sys.stderr.write("Failed to import sphinx_rtd_theme!") - - -html_domain_indices = True -html_show_sourcelink = True - - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "pexdoc" - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - #'preamble': '', +source_suffix = { + ".md": "markdown", + ".rst": "restructuredtext", } -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ("index", "pex.tex", u"pex Documentation", u"Brian Wickman", "manual"), +suppress_warnings = [ + # Otherwise epub warns (and we treat warinings as errors) when it finds .doctrees/ files, which it should not + # consider anyhow. + "epub.unknown_project_files" ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [("index", "pex", u"pex Documentation", [u"Brian Wickman"], 1)] - -# If true, show URL addresses after external links. -# man_show_urls = False - +templates_path = [ + "_templates", +] -# -- Options for Texinfo output ------------------------------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output +# https://myst-parser.readthedocs.io/en/latest/configuration.html +# https://pradyunsg.me/furo/customisation/ -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - "index", - "pex", - u"pex Documentation", - u"Brian Wickman", - "pex", - "One line description of project.", - "Miscellaneous", - ), +myst_enable_extensions = [ + "linkify", ] -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True +import sphinx_pex +from sphinx_pex import SVGIcon + +html_title = f"Pex Docs (v{release})" +html_theme = "furo" +html_favicon = "_static/pex.ico" +html_static_path = sphinx_pex.html_static_path() + + +html_theme_options = { + "light_logo": "pex-full-light.png", + "dark_logo": "pex-full-dark.png", + "sidebar_hide_name": True, + "source_repository": "https://github.com/pex-tool/pex/", + "source_branch": "main", + "source_directory": "docs/", + "footer_icons": [ + icon.as_furo_footer_icon() + for icon in [ + SVGIcon.load_if_static_asset_exists( + name="PDF", icon_asset=PurePath("pdf.svg"), static_asset=PurePath("pex.pdf") + ), + SVGIcon.load( + name="PyPI", + icon_asset=PurePath("python.svg"), + url=f"https://pypi.org/project/pex/{PEX_VERSION}/", + ), + SVGIcon.load( + name="Download", + icon_asset=PurePath("download.svg"), + url=f"https://github.com/pex-tool/pex/releases/download/v{PEX_VERSION}/pex", + ), + SVGIcon.load( + name="Source", + icon_asset=PurePath("github.svg"), + url="https://github.com/pex-tool/pex", + ), + ] + if icon + ], +} -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' +# -- Options for Sphinx-SimplePDF output ------------------------------------------------- +# https://sphinx-simplepdf.readthedocs.io/en/latest/configuration.html -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False +simplepdf_vars = { + "primary": "#ffcc00", + "cover": "black", + "cover-bg": "url(pex-cover.png) no-repeat center", +} diff --git a/pex/cli/commands/__init__.py b/pex/cli/commands/__init__.py index 9ea8b48ad..25e40b32c 100644 --- a/pex/cli/commands/__init__.py +++ b/pex/cli/commands/__init__.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pex.cli.command import BuildTimeCommand +from pex.cli.commands.docs import Docs from pex.cli.commands.interpreter import Interpreter from pex.cli.commands.lock import Lock from pex.cli.commands.venv import Venv @@ -13,4 +14,4 @@ def all_commands(): # type: () -> Iterable[Type[BuildTimeCommand]] - return Interpreter, Lock, Venv + return Docs, Interpreter, Lock, Venv diff --git a/pex/cli/commands/docs.py b/pex/cli/commands/docs.py new file mode 100644 index 000000000..89b9e1df0 --- /dev/null +++ b/pex/cli/commands/docs.py @@ -0,0 +1,220 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import errno +import json +import logging +import os +import re +import signal +import subprocess +import sys +import time +from textwrap import dedent + +from pex import docs +from pex.cli.command import BuildTimeCommand +from pex.commands.command import try_open_file +from pex.common import safe_open +from pex.result import Error, Result +from pex.typing import TYPE_CHECKING +from pex.variables import ENV +from pex.version import __version__ + +if TYPE_CHECKING: + from typing import Optional, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +logger = logging.getLogger(__name__) + + +SERVER_NAME = "Pex v{version} docs HTTP server".format(version=__version__) +SERVER_DIR = os.path.join(ENV.PEX_ROOT, "docs", "server", __version__) + + +@attr.s(frozen=True) +class Pidfile(object): + _PIDFILE = os.path.join(SERVER_DIR, "pidfile") + + @classmethod + def load(cls): + # type: () -> Optional[Pidfile] + try: + with open(cls._PIDFILE) as fp: + data = json.load(fp) + return cls(url=data["url"], pid=data["pid"]) + except (OSError, IOError, ValueError, KeyError) as e: + logger.warning( + "Failed to load {server} pid file from {path}: {err}".format( + server=SERVER_NAME, path=cls._PIDFILE, err=e + ) + ) + return None + + @staticmethod + def _read_url( + server_log, # type: str + timeout, # type: float + ): + # type: (...) -> Optional[str] + + # The permutations of Python versions, simple http server module and the output it provides: + # 2.7: Serving HTTP on 0.0.0.0 port 46399 ... -mSimpleHttpServer + # 3.5: Serving HTTP on 0.0.0.0 port 45577 ... -mhttp.server + # 3.6+: Serving HTTP on 0.0.0.0 port 33539 (http://0.0.0.0:33539/) ... -mhttp.server + + start = time.time() + while time.time() - start < timeout: + with open(server_log) as fp: + for line in fp: + if line.endswith(os.linesep): + match = re.search(r"Serving HTTP on 0.0.0.0 port (?P\d+)", line) + if match: + port = match.group("port") + return "http://localhost:{port}".format(port=port) + return None + + @classmethod + def record( + cls, + server_log, # type: str + pid, # type: int + timeout=5.0, # type: float + ): + # type: (...) -> Optional[Pidfile] + url = cls._read_url(server_log, timeout) + if not url: + return None + + with safe_open(cls._PIDFILE, "w") as fp: + json.dump(dict(url=url, pid=pid), fp, indent=2, sort_keys=True) + return cls(url=url, pid=pid) + + url = attr.ib() # type: str + pid = attr.ib() # type: int + + def alive(self): + # type: () -> bool + # TODO(John Sirois): Handle pid rollover + try: + os.kill(self.pid, 0) + return True + except OSError as e: + if e.errno == errno.ESRCH: # No such process. + return False + raise + + def kill(self): + # type: () -> None + os.kill(self.pid, signal.SIGTERM) + + +@attr.s(frozen=True) +class LaunchResult(object): + url = attr.ib() # type: str + already_running = attr.ib() # type: bool + + +def launch_docs_server( + document_root, # type: str + port, # type: int + timeout=5.0, # type: float +): + # type: (...) -> Union[str, LaunchResult] + + pidfile = Pidfile.load() + if pidfile and pidfile.alive(): + return LaunchResult(url=pidfile.url, already_running=True) + + # Not proper daemonization, but good enough. + log = os.path.join(SERVER_DIR, "log.txt") + http_server_module = "http.server" if sys.version_info[0] == 3 else "SimpleHttpServer" + env = os.environ.copy() + # N.B.: We set up line buffering for the process pipes as well as the underlying Python running + # the http server to ensure we can observe the `Serving HTTP on ...` line we need to grab the + # ephemeral port chosen. + env.update(PYTHONUNBUFFERED="1") + with safe_open(log, "w") as fp: + process = subprocess.Popen( + args=[sys.executable, "-m", http_server_module, str(port)], + env=env, + cwd=document_root, + preexec_fn=os.setsid, + bufsize=1, + stdout=fp.fileno(), + stderr=subprocess.STDOUT, + ) + + pidfile = Pidfile.record(server_log=log, pid=process.pid, timeout=timeout) + if not pidfile: + try: + os.kill(process.pid, signal.SIGKILL) + except OSError as e: + if e.errno != errno.ESRCH: # No such process. + raise + return log + + return LaunchResult(url=pidfile.url, already_running=False) + + +def shutdown_docs_server(): + # type: () -> bool + + pidfile = Pidfile.load() + if not pidfile: + return False + + logger.info( + "Killing {server} {url} @ {pid}".format( + server=SERVER_NAME, url=pidfile.url, pid=pidfile.pid + ) + ) + pidfile.kill() + return True + + +class Docs(BuildTimeCommand): + """Interact with the Pex documentation.""" + + def run(self): + # type: () -> Result + html_docs = docs.root(doc_type="html") + if not html_docs: + # TODO(John Sirois): Evaluate if this should fall back to opening latest html docs instead of just + # displaying links. + return Error( + dedent( + """\ + This Pex distribution does not include embedded docs. + + You can find the latest docs here: + HTML: https://docs.pex-tool.org + PDF: https://github.com/pex-tool/pex/releases/latest/download/pex.pdf + """ + ).rstrip() + ) + + # TODO(John Sirois): Consider trying a standard pex docs port, then fall back to ephemeral if: + # 2.7: socket.error: [Errno 98] Address already in use + # 3.x: OSError: [Errno 98] Address already in use + # This would allow for cookie stickiness for light / dark mode although the furo theme default of detecting + # the system default mode works well in practice to get you what you probably wanted anyhow. + result = launch_docs_server(html_docs, port=0) + if isinstance(result, str): + with open(result) as fp: + for line in fp: + logger.log(logging.ERROR, line.rstrip()) + return Error("Failed to launch {server}.".format(server=SERVER_NAME)) + + logger.info( + ( + "{server} already running at {url}" + if result.already_running + else "Launched {server} at {url}" + ).format(server=SERVER_NAME, url=result.url) + ) + return try_open_file(result.url) diff --git a/pex/docs/__init__.py b/pex/docs/__init__.py new file mode 100644 index 000000000..9fc271824 --- /dev/null +++ b/pex/docs/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path + +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional + + +def root(doc_type="html"): + # type: (str) -> Optional[str] + + doc_root = os.path.join(os.path.dirname(__file__), doc_type) + return doc_root if os.path.isdir(doc_root) else None diff --git a/pex/platforms.py b/pex/platforms.py index 961f8d592..f43950e87 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -82,7 +82,7 @@ def create(cls, platform, cause=None): You may be forced to specify this form when resolves encounter environment markers that use `python_full_version`. See the `--complete-platform` help as well as: - + https://pex.readthedocs.io/en/latest/buildingpex.html#complete-platform + + https://docs.pex-tool.org/buildingpex.html#complete-platform + https://www.python.org/dev/peps/pep-0508/#environment-markers """ ) diff --git a/pex/variables.py b/pex/variables.py index 3cea85320..41ff3b287 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -337,7 +337,9 @@ def _maybe_get_path_tuple( def strip(self): # type: () -> Variables stripped_environ = { - k: v for k, v in self.copy().items() if not k.startswith(("PEX_", "__PEX_")) + k: v + for k, v in self.copy().items() + if k.startswith("__PEX_BUILD_") or not k.startswith(("PEX_", "__PEX_")) } return Variables(environ=stripped_environ) diff --git a/pex/venv/installer_options.py b/pex/venv/installer_options.py index c178cf32c..a5201de91 100644 --- a/pex/venv/installer_options.py +++ b/pex/venv/installer_options.py @@ -27,7 +27,7 @@ def register( "situations it's beneficial to split the venv installation into {deps} and " "{sources} steps. This is particularly useful when installing a PEX in a container " "image. See " - "https://pex.readthedocs.io/en/latest/recipes.html#pex-app-in-a-container for more " + "https://docs.pex-tool.org/recipes.html#pex-app-in-a-container for more " "information.".format( all=InstallScope.ALL, deps=InstallScope.DEPS_ONLY, diff --git a/pex/version.py b/pex/version.py index 2188eae44..2ae99045d 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.1.163" +__version__ = "2.1.164" diff --git a/pyproject.toml b/pyproject.toml index 3fe90a7c8..1b47ea741 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,10 @@ backend-path = ["build-backend"] build-backend = "pex_build.hatchling.build" requires = ["hatchling"] -[tool.hatch.metadata.hooks.pex-dynamic-requires-python] +[tool.hatch.metadata.hooks.pex-adjust-metadata] +expand = {"pex_version" = "version"} + +[tool.hatch.build.targets.wheel.hooks.pex-adjust-build] # We need this empty table to enable our hook. [project] @@ -62,11 +65,11 @@ pex-tools = "pex.tools.main:main" bdist_pex = "pex.distutils.commands.bdist_pex:bdist_pex" [project.urls] +Changelog = "https://github.com/pex-tool/pex/blob/v{pex_version}/CHANGES.md" +Documentation = "https://docs.pex-tool.org/" +Download = "https://github.com/pex-tool/pex/releases/download/v{pex_version}/pex" Homepage = "https://github.com/pex-tool/pex" -Download = "https://github.com/pex-tool/pex/releases/latest/download/pex" -Changelog = "https://github.com/pex-tool/pex/blob/main/CHANGES.md" -Documentation = "https://pex.readthedocs.io/en/latest/" -Source = "https://github.com/pex-tool/pex" +Source = "https://github.com/pex-tool/pex/tree/v{pex_version}" [tool.hatch.version] path = "pex/version.py" diff --git a/scripts/build_docs.py b/scripts/build_docs.py new file mode 100644 index 000000000..4b867e9b9 --- /dev/null +++ b/scripts/build_docs.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import atexit +import os.path +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +from enum import Enum +from pathlib import Path, PurePath +from typing import Iterable, Optional +from urllib.parse import unquote, urlparse + +import httpx + +PEX_DEV_DIR = Path("~/.pex_dev").expanduser() + +PAGEFIND_NAME = "pagefind" +PAGEFIND_VERSION = "1.0.4" + + +class Platform(Enum): + Linux_aarch64 = "aarch64-unknown-linux-musl" + Linux_x86_64 = "x86_64-unknown-linux-musl" + Macos_aarch64 = "aarch64-apple-darwin" + Macos_x86_64 = "x86_64-apple-darwin" + Windows_x86_64 = "x86_64-pc-windows-msvc" + + @classmethod + def parse(cls, value: str) -> Platform: + return Platform.current() if "current" == value else Platform(value) + + @classmethod + def current(cls) -> Platform: + system = platform.system().lower() + machine = platform.machine().lower() + if system == "linux": + if machine in ("aarch64", "arm64"): + return cls.Linux_aarch64 + elif machine in ("amd64", "x86_64"): + return cls.Linux_x86_64 + elif system == "darwin": + if machine in ("aarch64", "arm64"): + return cls.Macos_aarch64 + elif machine in ("amd64", "x86_64"): + return cls.Macos_x86_64 + elif system == "windows" and machine in ("amd64", "x86_64"): + return cls.Windows_x86_64 + + raise ValueError( + f"The current operating system / machine pair is not supported!: {system} / {machine}" + ) + + @property + def extension(self): + return ".exe" if self is self.Windows_x86_64 else "" + + def binary_name(self, binary_name: str) -> str: + return f"{binary_name}{self.extension}" + + +CURRENT_PLATFORM = Platform.current() + + +def target_triple() -> str: + return CURRENT_PLATFORM.value + + +def pagefind_executable() -> str: + return CURRENT_PLATFORM.binary_name( + binary_name="{name}-{version}".format(name=PAGEFIND_NAME, version=PAGEFIND_VERSION) + ) + + +def ensure_pagefind() -> PurePath: + pagefind_exe = pagefind_executable() + pagefind_exe_path = PEX_DEV_DIR / pagefind_exe + if pagefind_exe_path.is_file() and os.access(pagefind_exe_path, os.R_OK | os.X_OK): + return pagefind_exe_path + + tmp_dir = PEX_DEV_DIR / ".tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + download_dir = Path(tempfile.mktemp(prefix="download-pagefind.", dir=tmp_dir)) + atexit.register(shutil.rmtree, str(download_dir), ignore_errors=True) + + tarball_url = ( + f"https://github.com/CloudCannon/pagefind/releases/download/v{PAGEFIND_VERSION}/" + f"{PAGEFIND_NAME}-v{PAGEFIND_VERSION}-{target_triple()}.tar.gz" + ) + out_path = download_dir / PurePath(unquote(urlparse(tarball_url).path)).name + out_path.parent.mkdir(parents=True, exist_ok=True) + with httpx.stream("GET", tarball_url, follow_redirects=True) as response, out_path.open( + "wb" + ) as out_fp: + for chunk in response.iter_bytes(): + out_fp.write(chunk) + with tarfile.open(out_path) as tf: + tf.extract(PAGEFIND_NAME, path=str(download_dir)) + (download_dir / PAGEFIND_NAME).rename(pagefind_exe_path) + + return pagefind_exe_path + + +def execute_pagefind(args: Iterable[str]) -> None: + subprocess.run(args=[str(ensure_pagefind()), *args], check=True) + + +def execute_sphinx_build(out_base_dir: Path, builder: str, out_dir: Optional[str] = None) -> Path: + gen_dir = (out_base_dir / (out_dir or builder)).absolute() + shutil.rmtree(gen_dir, ignore_errors=True) + subprocess.run( + args=[sys.executable, "-m", "sphinx", "-b", builder, "-aEW", "docs", str(gen_dir)], + check=True, + ) + return gen_dir + + +def main( + out_dir: Path, + linkcheck: bool = False, + pdf: bool = False, + html: bool = True, + clean_html: bool = False, + serve: bool = False, +) -> None: + static_dynamic_dir = (Path("docs") / "_static_dynamic").absolute() + + def clean_static_dynmaic_dir() -> None: + shutil.rmtree(static_dynamic_dir, ignore_errors=True) + + clean_static_dynmaic_dir() + static_dynamic_dir.mkdir(parents=True, exist_ok=True) + atexit.register(clean_static_dynmaic_dir) + + if linkcheck: + execute_sphinx_build(out_dir, "linkcheck") + + if pdf: + pdf_dir = execute_sphinx_build(out_dir, "simplepdf", out_dir="pdf") + (static_dynamic_dir / "pex.pdf").symlink_to(pdf_dir / "pex.pdf") + + if html: + html_dir = execute_sphinx_build(out_dir, "html") + if clean_html: + shutil.rmtree(html_dir / ".doctrees", ignore_errors=True) + (html_dir / ".buildinfo").unlink(missing_ok=True) + (html_dir / "objects.inv").unlink(missing_ok=True) + + page_find_args = ["--site", str(html_dir), "--output-subdir", "_pagefind"] + if serve: + page_find_args.append("--serve") + execute_pagefind(page_find_args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--linkcheck", action="store_true") + parser.add_argument("--pdf", action="store_true") + parser.add_argument("--no-html", action="store_true") + parser.add_argument("--clean-html", action="store_true") + parser.add_argument("--serve", action="store_true") + parser.add_argument("out_dir", type=Path, default=Path("dist") / "docs", nargs="?") + options = parser.parse_args() + try: + main( + out_dir=options.out_dir, + linkcheck=options.linkcheck, + pdf=options.pdf, + html=not options.no_html, + clean_html=options.clean_html, + serve=options.serve, + ) + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) diff --git a/scripts/format.py b/scripts/format.py index ba7629894..e25169845 100755 --- a/scripts/format.py +++ b/scripts/format.py @@ -35,7 +35,17 @@ def run_black(*args: str) -> None: dest=sys.stdout, ) as out_fd: subprocess.run( - args=["black", "--color", *args, "build-backend", "pex", "scripts", "testing", "tests"], + args=[ + "black", + "--color", + *args, + "build-backend", + "docs", + "pex", + "scripts", + "testing", + "tests", + ], stdout=out_fd, stderr=subprocess.STDOUT, check=True, @@ -44,7 +54,8 @@ def run_black(*args: str) -> None: def run_isort(*args: str) -> None: subprocess.run( - args=["isort", *args, "build-backend", "pex", "scripts", "testing", "tests"], check=True + args=["isort", *args, "build-backend", "docs", "pex", "scripts", "testing", "tests"], + check=True, ) diff --git a/scripts/lint.py b/scripts/lint.py index 41e01ec20..a17b0caa0 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -25,6 +25,7 @@ def run_autoflake(*args: str) -> None: ",".join(excludes), "--recursive", "build-backend", + "docs", "pex", "scripts", "testing", diff --git a/scripts/package.py b/scripts/package.py index b160f78bc..9944c5c71 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -1,21 +1,26 @@ #!/usr/bin/env python3 +import atexit import hashlib import io import os +import shutil import subprocess import sys +import tempfile from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from email.parser import Parser from enum import Enum, unique from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path, PurePath -from typing import Optional, Tuple, cast +from typing import Dict, Iterator, Optional, Tuple, cast DIST_DIR = Path("dist") -def build_pex_pex(output_file: PurePath, verbosity: int = 0) -> None: +def build_pex_pex( + output_file: PurePath, verbosity: int = 0, env: Optional[Dict[str, str]] = None +) -> PurePath: # NB: We do not include the subprocess extra (which would be spelled: `.[subprocess]`) since we # would then produce a pex that would not be consumable by all python interpreters otherwise # meeting `python_requires`; ie: we'd need to then come up with a deploy environment / deploy @@ -44,7 +49,8 @@ def build_pex_pex(output_file: PurePath, verbosity: int = 0) -> None: "pex", pex_requirement, ] - subprocess.run(args, check=True) + subprocess.run(args=args, env=env, check=True) + return output_file def describe_rev() -> str: @@ -82,32 +88,54 @@ def build_arg(self) -> str: return f"--{self.value}" -def build_pex_dists(dist_fmt: Format, *additional_dist_fmts: Format, verbose: bool = False) -> None: +def build_pex_dists( + dist_fmt: Format, + *additional_dist_fmts: Format, + verbose: bool = False, + env: Optional[Dict[str, str]] = None +) -> Iterator[PurePath]: + tmp_dir = DIST_DIR / ".tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + out_dir = tempfile.mkdtemp(dir=tmp_dir) + atexit.register(shutil.rmtree, out_dir, ignore_errors=True) + output = None if verbose else subprocess.DEVNULL + subprocess.run( - [ + args=[ sys.executable, "-m", "build", "--outdir", - str(DIST_DIR), + out_dir, *[fmt.build_arg() for fmt in [dist_fmt, *additional_dist_fmts]], ], + env=env, stdout=output, stderr=output, check=True, ) + for dist in os.listdir(out_dir): + built = DIST_DIR / dist + shutil.move(os.path.join(out_dir, dist), built) + yield built + def main( *additional_dist_formats: Format, verbosity: int = 0, + embed_docs: bool = False, pex_output_file: Optional[Path] = DIST_DIR / "pex", serve: bool = False ) -> None: + env = os.environ.copy() + if embed_docs: + env.update(__PEX_BUILD_INCLUDE_DOCS__="1") + if pex_output_file: print(f"Building Pex PEX to `{pex_output_file}` ...") - build_pex_pex(pex_output_file, verbosity) + build_pex_pex(pex_output_file, verbosity, env=env) rev = describe_rev() sha256, size = describe_file(pex_output_file) @@ -120,16 +148,17 @@ def main( f"Building additional distribution formats to `{DIST_DIR}`: " f'{", ".join(f"{i + 1}.) {fmt}" for i, fmt in enumerate(additional_dist_formats))} ...' ) - build_pex_dists( - additional_dist_formats[0], *additional_dist_formats[1:], verbose=verbosity > 0 + built = list( + build_pex_dists( + additional_dist_formats[0], + *additional_dist_formats[1:], + verbose=verbosity > 0, + env=env + ) ) print("Built:") - for root, _, files in os.walk(DIST_DIR): - root_path = Path(root) - for f in files: - dist_path = root_path / f - if dist_path != pex_output_file: - print(f" {dist_path}") + for dist_path in built: + print(f" {dist_path}") if serve: server = HTTPServer(("", 0), SimpleHTTPRequestHandler) @@ -149,6 +178,12 @@ def main( parser.add_argument( "-v", dest="verbosity", action="count", default=0, help="Increase output verbosity level." ) + parser.add_argument( + "--embed-docs", + default=False, + action="store_true", + help="Embed offline docs in the built binary distributions.", + ) parser.add_argument( "--additional-format", dest="additional_formats", @@ -180,6 +215,7 @@ def main( main( *(args.additional_formats or ()), verbosity=args.verbosity, + embed_docs=args.embed_docs, pex_output_file=None if args.no_pex else args.pex_output_file, serve=args.serve ) diff --git a/scripts/typecheck.py b/scripts/typecheck.py index f3e2ad93c..da98a200b 100755 --- a/scripts/typecheck.py +++ b/scripts/typecheck.py @@ -41,6 +41,11 @@ def main() -> None: run_mypy( "2.7", files=sorted(find_files_to_check(include=["build-backend"])), subject="build-backend" ) + run_mypy( + "3.8", + files=sorted(find_files_to_check(include=["docs"])), + subject="sphinx_pex", + ) run_mypy("3.8", files=sorted(find_files_to_check(include=["scripts"])), subject="scripts") source_and_tests = sorted( diff --git a/tox.ini b/tox.ini index c28b741ac..b563116a8 100644 --- a/tox.ini +++ b/tox.ini @@ -74,8 +74,7 @@ setenv = # line buffering (which is what setting PYTHONUNBUFFERED nets you) so that tests can rely on # stderr lines being observable. py{py35,py36,py37,py38,py39,35,36,37,38}: PYTHONUNBUFFERED=1 -whitelist_externals = - open +allowlist_externals = bash git @@ -136,6 +135,8 @@ deps = types-setuptools types-toml==0.10.5 httpx==0.23.0 + sphinx + types-docutils commands = python scripts/typecheck.py @@ -168,13 +169,11 @@ commands = git diff --exit-code [testenv:docs] -changedir = docs +basepython = python3 deps = - sphinx - sphinx-rtd-theme + -r docs-requirements.txt commands = - sphinx-build -b html -d {envtmpdir}/doctrees . _build/html - open _build/html/index.html + python scripts/build_docs.py {posargs} [_package] basepython = python3