From c610139225d2d74553ac7ab79b8b2bf6ceab95ff Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 15 Jun 2023 17:51:20 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20Move=20packaging=20to=20PEP=2051?= =?UTF-8?q?7=20in-tree=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This essentially allows the cythonization be controlled by the `--build-c-extensions` PEP 517 config setting that can be passed to the corresponding build frontends via their respective CLIs. --- .github/workflows/ci-cd.yml | 39 ++- Makefile | 4 +- packaging/pep517_backend/__init__.py | 1 + packaging/pep517_backend/_backend.py | 302 ++++++++++++++++++++++ packaging/pep517_backend/_compat.py | 23 ++ packaging/pep517_backend/_transformers.py | 41 +++ packaging/pep517_backend/hooks.py | 12 + pyproject.toml | 67 ++++- requirements/ci.txt | 1 - setup.cfg | 2 +- setup.py | 51 ---- 11 files changed, 462 insertions(+), 81 deletions(-) create mode 100644 packaging/pep517_backend/__init__.py create mode 100644 packaging/pep517_backend/_backend.py create mode 100644 packaging/pep517_backend/_compat.py create mode 100644 packaging/pep517_backend/_transformers.py create mode 100644 packaging/pep517_backend/hooks.py delete mode 100644 setup.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 64339471a..58db0ebcd 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -38,11 +38,9 @@ jobs: uses: py-actions/py-dependency-install@v4 with: path: requirements/lint.txt - - name: Install itself + - name: Self-install run: | - python setup.py install - env: - YARL_NO_EXTENSIONS: 1 + pip install . - name: Run linters run: | make lint @@ -55,10 +53,8 @@ jobs: make doc-spelling - name: Prepare twine checker run: | - pip install -U twine wheel - python setup.py sdist bdist_wheel - env: - YARL_NO_EXTENSIONS: 1 + pip install -U build twine + python -m build - name: Run twine checker run: | twine check dist/* @@ -111,16 +107,18 @@ jobs: uses: py-actions/py-dependency-install@v4 with: path: requirements/cython.txt - - name: Cythonize - if: ${{ matrix.no-extensions == '' }} - run: | - make cythonize - name: Install dependencies uses: py-actions/py-dependency-install@v4 with: path: requirements/ci.txt - env: - YARL_NO_EXTENSIONS: ${{ matrix.no-extensions }} + - name: Self-install + run: >- + python -m pip install -e . + ${{ + matrix.no-extensions == '' + && '--config-settings=--build-c-extensions=' + || '' + }} - name: Run unittests env: COLOR: 'yes' @@ -167,16 +165,12 @@ jobs: uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 - - name: Install cython - uses: py-actions/py-dependency-install@v4 - with: - path: requirements/cython.txt - - name: Cythonize + - name: Install pypa/build run: | - make cythonize + python -Im pip install build - name: Make sdist run: - python setup.py sdist + python -Im build --sdist - name: Upload artifacts uses: actions/upload-artifact@v3 with: @@ -222,9 +216,6 @@ jobs: uses: py-actions/py-dependency-install@v4 with: path: requirements/cython.txt - - name: Cythonize - run: | - make cythonize - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 env: diff --git a/Makefile b/Makefile index e0e0c7883..1af0db9d9 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ yarl/%.c: yarl/%.pyx cythonize: .cythonize -.develop: .install-deps $(shell find yarl -type f) .cythonize - @pip install -e . +.develop: .install-deps $(shell find yarl -type f) + @pip install -e . --config-settings=--build-c-extensions= @touch .develop fmt: diff --git a/packaging/pep517_backend/__init__.py b/packaging/pep517_backend/__init__.py new file mode 100644 index 000000000..74ae43697 --- /dev/null +++ b/packaging/pep517_backend/__init__.py @@ -0,0 +1 @@ +"""PEP 517 build backend for optionally pre-building Cython.""" diff --git a/packaging/pep517_backend/_backend.py b/packaging/pep517_backend/_backend.py new file mode 100644 index 000000000..b3014fb14 --- /dev/null +++ b/packaging/pep517_backend/_backend.py @@ -0,0 +1,302 @@ +"""PEP 517 build backend wrapper for pre-building Cython for wheel.""" + +from __future__ import annotations + +import os +import typing as t +from contextlib import contextmanager, suppress +from pathlib import Path +from shutil import copytree +from tempfile import TemporaryDirectory +from warnings import warn as _warn_that +from sys import ( + implementation as _system_implementation, + stderr as _standard_error_stream, + version_info as _python_version_tuple +) + +try: + from tomllib import loads as _load_toml_from_string +except ImportError: + from tomli import loads as _load_toml_from_string + +from expandvars import expandvars + +from setuptools.build_meta import ( + build_sdist as _setuptools_build_sdist, + build_wheel as _setuptools_build_wheel, + get_requires_for_build_wheel as _setuptools_get_requires_for_build_wheel, + prepare_metadata_for_build_wheel as _setuptools_prepare_metadata_for_build_wheel, +) + + +# isort: split +from distutils.command.install import install as _distutils_install_cmd +from distutils.core import Distribution as _DistutilsDistribution +from distutils.dist import DistributionMetadata as _DistutilsDistributionMetadata + +with suppress(ImportError): + # NOTE: Only available for wheel builds that bundle C-extensions. Declared + # NOTE: by `get_requires_for_build_wheel()` when `--build-c-extensions` is + # NOTE: passed. + from Cython.Build.Cythonize import main as _cythonize_cli_cmd + +from ._compat import chdir_cm +from ._transformers import ( # noqa: WPS436 + get_cli_kwargs_from_config, + get_enabled_cli_flags_from_config, + sanitize_rst_roles, +) + +__all__ = ( # noqa: WPS317, WPS410 + 'build_sdist', + 'build_wheel', + 'get_requires_for_build_editable', + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_editable', + 'prepare_metadata_for_build_wheel', +) + + +BUILD_C_EXT_CONFIG_SETTING = '--build-c-extensions' +"""Config setting name toggle that is used to request C-ext in wheels.""" + +IS_PY3_12_PLUS = _python_version_tuple[:2] >= (3, 12) +"""A flag meaning that the current runtime is Python 3.12 or higher.""" + +IS_CPYTHON = _system_implementation.name == "cpython" +"""A flag meaning that the current interpreter implementation is CPython.""" + + +def _get_local_cython_config(): + """Grab optional build dependencies from pyproject.toml config. + + :returns: config section from ``pyproject.toml`` + :rtype: dict + + This basically reads entries from:: + + [tool.local.cythonize] + # Env vars provisioned during cythonize call + src = ["src/**/*.pyx"] + + [tool.local.cythonize.env] + # Env vars provisioned during cythonize call + LDFLAGS = "-lssh" + + [tool.local.cythonize.flags] + # This section can contain the following booleans: + # * annotate — generate annotated HTML page for source files + # * build — build extension modules using distutils + # * inplace — build extension modules in place using distutils (implies -b) + # * force — force recompilation + # * quiet — be less verbose during compilation + # * lenient — increase Python compat by ignoring some compile time errors + # * keep-going — compile as much as possible, ignore compilation failures + annotate = false + build = false + inplace = true + force = true + quiet = false + lenient = false + keep-going = false + + [tool.local.cythonize.kwargs] + # This section can contain args tha have values: + # * exclude=PATTERN exclude certain file patterns from the compilation + # * parallel=N run builds in N parallel jobs (default: calculated per system) + exclude = "**.py" + parallel = 12 + + [tool.local.cythonize.kwargs.directives] + # This section can contain compiler directives + # NAME = "VALUE" + + [tool.local.cythonize.kwargs.compile-time-env] + # This section can contain compile time env vars + # NAME = "VALUE" + + [tool.local.cythonize.kwargs.options] + # This section can contain cythonize options + # NAME = "VALUE" + """ + config_toml_txt = (Path.cwd().resolve() / 'pyproject.toml').read_text() + config_mapping = _load_toml_from_string(config_toml_txt) + return config_mapping['tool']['local']['cythonize'] + + +@contextmanager +def patched_distutils_cmd_install(): + """Make `install_lib` of `install` cmd always use `platlib`. + + :yields: None + """ + # Without this, build_lib puts stuff under `*.data/purelib/` folder + orig_finalize = _distutils_install_cmd.finalize_options + + def new_finalize_options(self): # noqa: WPS430 + self.install_lib = self.install_platlib + orig_finalize(self) + + _distutils_install_cmd.finalize_options = new_finalize_options + try: # noqa: WPS501 + yield + finally: + _distutils_install_cmd.finalize_options = orig_finalize + + +@contextmanager +def patched_dist_has_ext_modules(): + """Make `has_ext_modules` of `Distribution` always return `True`. + + :yields: None + """ + # Without this, build_lib puts stuff under `*.data/platlib/` folder + orig_func = _DistutilsDistribution.has_ext_modules + + _DistutilsDistribution.has_ext_modules = lambda *args, **kwargs: True + try: # noqa: WPS501 + yield + finally: + _DistutilsDistribution.has_ext_modules = orig_func + + +@contextmanager +def patched_dist_get_long_description(): + """Make `has_ext_modules` of `Distribution` always return `True`. + + :yields: None + """ + # Without this, build_lib puts stuff under `*.data/platlib/` folder + _orig_func = _DistutilsDistributionMetadata.get_long_description + + def _get_sanitized_long_description(self): + return sanitize_rst_roles(self.long_description) + + _DistutilsDistributionMetadata.get_long_description = ( + _get_sanitized_long_description + ) + try: # noqa: WPS501 + yield + finally: + _DistutilsDistributionMetadata.get_long_description = _orig_func + + +@contextmanager +def patched_env(env): + """Temporary set given env vars. + + :param env: tmp env vars to set + :type env: dict + + :yields: None + """ + orig_env = os.environ.copy() + expanded_env = {name: expandvars(var_val) for name, var_val in env.items()} + os.environ.update(expanded_env) + if os.getenv('YARL_CYTHON_TRACING') == '1': + os.environ['CFLAGS'] = ' '.join(( + os.getenv('CFLAGS', ''), + '-DCYTHON_TRACE=1', + '-DCYTHON_TRACE_NOGIL=1', + )).strip() + try: # noqa: WPS501 + yield + finally: + os.environ.clear() + os.environ.update(orig_env) + + +@contextmanager +def _run_in_temporary_directory() -> t.Iterator[Path]: + with TemporaryDirectory(prefix='.tmp-yarl-pep517-') as tmp_dir: + with chdir_cm(tmp_dir): + yield Path(tmp_dir) + + +@patched_dist_get_long_description() +def build_wheel( # noqa: WPS210, WPS430 + wheel_directory: str, + config_settings: dict[str, str] | None = None, + metadata_directory: str | None = None, +) -> str: + build_c_ext_requested = BUILD_C_EXT_CONFIG_SETTING in ( + config_settings or {} + ) + if not build_c_ext_requested: + print("*********************", file=_standard_error_stream) + print("* Pure Python build *", file=_standard_error_stream) + print("*********************", file=_standard_error_stream) + return _setuptools_build_wheel( + wheel_directory=wheel_directory, + config_settings=config_settings, + metadata_directory=metadata_directory, + ) + + print("**********************", file=_standard_error_stream) + print("* Accelerated build *", file=_standard_error_stream) + print("**********************", file=_standard_error_stream) + if not IS_CPYTHON: + _warn_that( + 'Building C-extensions under the runtimes other than CPython is ' + 'unsupported and will likely fail. Consider not passing the ' + f'`{BUILD_C_EXT_CONFIG_SETTING !s}` PEP 517 config setting.', + RuntimeWarning, + stacklevel=999, + ) + + original_src_dir = Path.cwd().resolve() + with _run_in_temporary_directory() as tmp_dir: + tmp_src_dir = Path(tmp_dir) / 'src' + copytree(original_src_dir, tmp_src_dir, symlinks=True) + os.chdir(tmp_src_dir) + + config = _get_local_cython_config() + + py_ver_arg = f'-{_python_version_tuple.major!s}' + + cli_flags = get_enabled_cli_flags_from_config(config['flags']) + cli_kwargs = get_cli_kwargs_from_config(config['kwargs']) + + cythonize_args = cli_flags + [py_ver_arg] + cli_kwargs + config['src'] + with patched_env(config['env']): + _cythonize_cli_cmd(cythonize_args) + with patched_distutils_cmd_install(): + with patched_dist_has_ext_modules(): + return _setuptools_build_wheel( + wheel_directory=wheel_directory, + config_settings=config_settings, + metadata_directory=metadata_directory, + ) + + +def get_requires_for_build_wheel( + config_settings: dict[str, str] | None = None, +) -> list[str]: + build_c_ext_requested = BUILD_C_EXT_CONFIG_SETTING in ( + config_settings or {} + ) + + if build_c_ext_requested and not IS_CPYTHON: + _warn_that( + 'Building C-extensions under the runtimes other than CPython is ' + 'unsupported and will likely fail. Consider not passing the ' + f'`{BUILD_C_EXT_CONFIG_SETTING !s}` PEP 517 config setting.', + RuntimeWarning, + stacklevel=999, + ) + + c_ext_build_deps = [ + 'Cython >= 3.0.0b3' if IS_PY3_12_PLUS # Only Cython 3+ is compatible + else 'Cython', + ] if build_c_ext_requested else [] + + return _setuptools_get_requires_for_build_wheel( + config_settings=config_settings, + ) + c_ext_build_deps + + +build_sdist = patched_dist_get_long_description()(_setuptools_build_sdist) +get_requires_for_build_editable = get_requires_for_build_wheel +prepare_metadata_for_build_wheel = patched_dist_get_long_description()(_setuptools_prepare_metadata_for_build_wheel) +prepare_metadata_for_build_editable = prepare_metadata_for_build_wheel diff --git a/packaging/pep517_backend/_compat.py b/packaging/pep517_backend/_compat.py new file mode 100644 index 000000000..900ae80a4 --- /dev/null +++ b/packaging/pep517_backend/_compat.py @@ -0,0 +1,23 @@ +"""Cross-python stdlib shims.""" + +import os +import typing as t +from contextlib import contextmanager +from pathlib import Path + + +try: + from contextlib import chdir as chdir_cm +except ImportError: + @contextmanager + def chdir_cm(path: os.PathLike) -> t.Iterator[None]: + """Temporarily change the current directory, recovering on exit.""" + original_wd = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(original_wd) + + +__all__ = ('chdir_cm', ) # noqa: WPS317, WPS410 diff --git a/packaging/pep517_backend/_transformers.py b/packaging/pep517_backend/_transformers.py new file mode 100644 index 000000000..71a4fc06c --- /dev/null +++ b/packaging/pep517_backend/_transformers.py @@ -0,0 +1,41 @@ +"""Data conversion helpers for the in-tree PEP 517 build backend.""" + +from itertools import chain +from re import sub as _substitute_with_regexp + + +def _emit_opt_pairs(opt_pair): + flag, flag_value = opt_pair + flag_opt = '--{name!s}'.format(name=flag) + if isinstance(flag_value, dict): + sub_pairs = flag_value.items() + else: + sub_pairs = ((flag_value,),) + + yield from ( + '='.join(map(str, (flag_opt,) + pair)) + for pair in sub_pairs + ) + + +def get_cli_kwargs_from_config(kwargs_map): + """Make a list of options with values from config.""" + return list(chain.from_iterable(map(_emit_opt_pairs, kwargs_map.items()))) + + +def get_enabled_cli_flags_from_config(flags_map): + """Make a list of enabled boolean flags from config.""" + return [ + '--{flag}'.format(flag=flag) + for flag, is_enabled in flags_map.items() + if is_enabled + ] + + +def sanitize_rst_roles(rst_source_text: str) -> str: + """Replace RST roles with inline highlighting.""" + role_regex = r"(?::\w+)?:\w+:`(?P[^`]+)(?:\s+(.*))?`" + substitution_pattern = r"``\g``" + return _substitute_with_regexp( + role_regex, substitution_pattern, rst_source_text, + ) diff --git a/packaging/pep517_backend/hooks.py b/packaging/pep517_backend/hooks.py new file mode 100644 index 000000000..3126e2bb8 --- /dev/null +++ b/packaging/pep517_backend/hooks.py @@ -0,0 +1,12 @@ +"""PEP 517 build backend for optionally pre-building Cython.""" + +from setuptools.build_meta import * # Re-exporting PEP 517 hooks # pylint: disable=unused-wildcard-import,wildcard-import + +from ._backend import ( # noqa: WPS436 # Re-exporting PEP 517 hooks + build_sdist, + build_wheel, + get_requires_for_build_editable, + get_requires_for_build_wheel, + prepare_metadata_for_build_editable, + prepare_metadata_for_build_wheel, +) diff --git a/pyproject.toml b/pyproject.toml index 870be55c8..4e62a9cee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,67 @@ [build-system] requires = [ - "setuptools>=40", + # NOTE: The following build dependencies are necessary for initial + # NOTE: provisioning of the in-tree build backend located under + # NOTE: `packaging/pep517_backend/`. + "expandvars", + "setuptools >= 45", + "tomli; python_version < '3.11'", ] -build-backend = "setuptools.build_meta" +backend-path = ["packaging"] # requires `pip >= 20` or `pep517 >= 0.6.0` +build-backend = "pep517_backend.hooks" # wraps `setuptools.build_meta` + +[tool.local.cythonize] +# This attr can contain multiple globs +src = ["yarl/*.pyx"] + +[tool.local.cythonize.env] +# Env vars provisioned during cythonize call +#CFLAGS = "-DCYTHON_TRACE=1 ${CFLAGS}" +#LDFLAGS = "${LDFLAGS}" + +[tool.local.cythonize.flags] +# This section can contain the following booleans: +# * annotate — generate annotated HTML page for source files +# * build — build extension modules using distutils +# * inplace — build extension modules in place using distutils (implies -b) +# * force — force recompilation +# * quiet — be less verbose during compilation +# * lenient — increase Python compat by ignoring some compile time errors +# * keep-going — compile as much as possible, ignore compilation failures +annotate = false +build = false +inplace = true +force = true +quiet = false +lenient = false +keep-going = false + +[tool.local.cythonize.kwargs] +# This section can contain args that have values: +# * exclude=PATTERN exclude certain file patterns from the compilation +# * parallel=N run builds in N parallel jobs (default: calculated per system) +# exclude = "**.py" +# parallel = 12 + +[tool.local.cythonize.kwargs.directive] +# This section can contain compiler directives +# Ref: https://github.com/cython/cython/blob/d6e6de9/Cython/Compiler/Options.py#L170-L242 +embedsignature = "True" +emit_code_comments = "True" +linetrace = "True" +profile = "True" + +[tool.local.cythonize.kwargs.compile-time-env] +# This section can contain compile time env vars + +[tool.local.cythonize.kwargs.option] +# This section can contain cythonize options +# Ref: https://github.com/cython/cython/blob/d6e6de9/Cython/Compiler/Options.py#L694-L730 +#docstrings = "True" +#embed_pos_in_docstring = "True" +#warning_errors = "True" +#error_on_unknown_names = "True" +#error_on_uninitialized = "True" [tool.towncrier] package = "yarl" @@ -13,7 +72,11 @@ issue_format = "`#{issue} `_" [tool.cibuildwheel] +build-frontend = "build" test-requires = "-r requirements/ci.txt" test-command = "pytest {project}/tests" # don't build PyPy wheels, install from source instead skip = "pp*" + +[tool.cibuildwheel.config-settings] +--build-c-extensions = "" diff --git a/requirements/ci.txt b/requirements/ci.txt index 421b0594e..4e95b36ec 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,3 +1,2 @@ --e . -r test.txt -r lint.txt diff --git a/setup.cfg b/setup.cfg index f9a506234..f93cc1e56 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ project_urls = GitHub: issues = https://github.com/aio-libs/yarl/issues GitHub: repo = https://github.com/aio-libs/yarl description = Yet another URL library -# long_description = file: README.rst, CHANGES.rst +long_description = file: README.rst, CHANGES.rst long_description_content_type = text/x-rst author = Andrew Svetlov author_email = andrew.svetlov@gmail.com diff --git a/setup.py b/setup.py deleted file mode 100644 index e9334a2e7..000000000 --- a/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import pathlib -import re -import sys - -from setuptools import Extension, setup - -NO_EXTENSIONS = bool(os.environ.get("YARL_NO_EXTENSIONS")) # type: bool - -if sys.implementation.name != "cpython": - NO_EXTENSIONS = True - - -extensions = [Extension("yarl._quoting_c", ["yarl/_quoting_c.c"])] - - -here = pathlib.Path(__file__).parent - - -def read(name): - fname = here / name - with fname.open(encoding="utf8") as f: - return f.read() - - -# $ echo ':something:`test `' | sed 's/:\w\+:`\(\w\+\)\(\s\+\(.*\)\)\?`/``\1``/g' -# ``test`` -def sanitize_rst_roles(rst_source_text: str) -> str: - """Replace RST roles with inline highlighting.""" - role_regex = r":\w+:`(?P[^`]+)(\s+(.*))?`" - substitution_pattern = r"``(?P=rendered_text)``" - return re.sub(role_regex, substitution_pattern, rst_source_text) - - -args = dict( - long_description="\n\n".join( - [read("README.rst"), sanitize_rst_roles(read("CHANGES.rst"))] - ), -) - - -if not NO_EXTENSIONS: - print("**********************") - print("* Accelerated build *") - print("**********************") - setup(ext_modules=extensions, **args) -else: - print("*********************") - print("* Pure Python build *") - print("*********************") - setup(**args)