📦 Move packaging to PEP 517 in-tree backend

This essentially allows the cythonization be controlled by the
`--build-c-extensions` PEP 517 config setting that can be passed to
the corresponding build frontends via their respective CLIs.

.github/workflows/ci-cd.yml

@@ -38,11 +38,9 @@ jobs:
       uses: py-actions/py-dependency-install@v4
       with:
         path: requirements/lint.txt
-    - name: Install itself
+    - name: Self-install
       run: |
-        python setup.py install
-      env:
-        YARL_NO_EXTENSIONS: 1
+        pip install .
     - name: Run linters
       run: |
         make lint
@@ -55,10 +53,8 @@ jobs:
         make doc-spelling
     - name: Prepare twine checker
       run: |
-        pip install -U twine wheel
-        python setup.py sdist bdist_wheel
-      env:
-        YARL_NO_EXTENSIONS: 1
+        pip install -U build twine
+        python -m build
     - name: Run twine checker
       run: |
         twine check dist/*
@@ -111,16 +107,18 @@ jobs:
       uses: py-actions/py-dependency-install@v4
       with:
         path: requirements/cython.txt
-    - name: Cythonize
-      if: ${{ matrix.no-extensions == '' }}
-      run: |
-        make cythonize
     - name: Install dependencies
       uses: py-actions/py-dependency-install@v4
       with:
         path: requirements/ci.txt
-      env:
-        YARL_NO_EXTENSIONS: ${{ matrix.no-extensions }}
+    - name: Self-install
+      run: >-
+        python -m pip install -e .
+        ${{
+          matrix.no-extensions == ''
+          && '--config-settings=--build-c-extensions='
+          || ''
+        }}
     - name: Run unittests
       env:
         COLOR: 'yes'
@@ -167,16 +165,12 @@ jobs:
       uses: actions/checkout@v4
     - name: Setup Python
       uses: actions/setup-python@v4
-    - name: Install cython
-      uses: py-actions/py-dependency-install@v4
-      with:
-        path: requirements/cython.txt
-    - name: Cythonize
+    - name: Install pypa/build
       run: |
-        make cythonize
+        python -Im pip install build
     - name: Make sdist
       run:
-        python setup.py sdist
+        python -Im build --sdist
     - name: Upload artifacts
       uses: actions/upload-artifact@v3
       with:
@@ -222,9 +216,6 @@ jobs:
       uses: py-actions/py-dependency-install@v4
       with:
         path: requirements/cython.txt
-    - name: Cythonize
-      run: |
-        make cythonize
     - name: Build wheels
       uses: pypa/[email protected]
       env:

Makefile

@@ -25,8 +25,8 @@ yarl/%.c: yarl/%.pyx
 cythonize: .cythonize
 
 
-.develop: .install-deps $(shell find yarl -type f) .cythonize
-	@pip install -e .
+.develop: .install-deps $(shell find yarl -type f)
+	@pip install -e . --config-settings=--build-c-extensions=
 	@touch .develop
 
 fmt:

packaging/pep517_backend/__init__.py

@@ -0,0 +1 @@
+"""PEP 517 build backend for optionally pre-building Cython."""

packaging/pep517_backend/_backend.py

@@ -0,0 +1,302 @@
+"""PEP 517 build backend wrapper for pre-building Cython for wheel."""
+
+from __future__ import annotations
+
+import os
+import typing as t
+from contextlib import contextmanager, suppress
+from pathlib import Path
+from shutil import copytree
+from tempfile import TemporaryDirectory
+from warnings import warn as _warn_that
+from sys import (
+    implementation as _system_implementation,
+    stderr as _standard_error_stream,
+    version_info as _python_version_tuple
+)
+
+try:
+    from tomllib import loads as _load_toml_from_string
+except ImportError:
+    from tomli import loads as _load_toml_from_string
+
+from expandvars import expandvars
+
+from setuptools.build_meta import (
+    build_sdist as _setuptools_build_sdist,
+    build_wheel as _setuptools_build_wheel,
+    get_requires_for_build_wheel as _setuptools_get_requires_for_build_wheel,
+    prepare_metadata_for_build_wheel as _setuptools_prepare_metadata_for_build_wheel,
+)
+
+
+# isort: split
+from distutils.command.install import install as _distutils_install_cmd
+from distutils.core import Distribution as _DistutilsDistribution
+from distutils.dist import DistributionMetadata as _DistutilsDistributionMetadata
+
+with suppress(ImportError):
+    # NOTE: Only available for wheel builds that bundle C-extensions. Declared
+    # NOTE: by `get_requires_for_build_wheel()` when `--build-c-extensions` is
+    # NOTE: passed.
+    from Cython.Build.Cythonize import main as _cythonize_cli_cmd
+
+from ._compat import chdir_cm
+from ._transformers import (  # noqa: WPS436
+    get_cli_kwargs_from_config,
+    get_enabled_cli_flags_from_config,
+    sanitize_rst_roles,
+)
+
+__all__ = (  # noqa: WPS317, WPS410
+    'build_sdist',
+    'build_wheel',
+    'get_requires_for_build_editable',
+    'get_requires_for_build_wheel',
+    'prepare_metadata_for_build_editable',
+    'prepare_metadata_for_build_wheel',
+)
+
+
+BUILD_C_EXT_CONFIG_SETTING = '--build-c-extensions'
+"""Config setting name toggle that is used to request C-ext in wheels."""
+
+IS_PY3_12_PLUS = _python_version_tuple[:2] >= (3, 12)
+"""A flag meaning that the current runtime is Python 3.12 or higher."""
+
+IS_CPYTHON = _system_implementation.name == "cpython"
+"""A flag meaning that the current interpreter implementation is CPython."""
+
+
+def _get_local_cython_config():
+    """Grab optional build dependencies from pyproject.toml config.
+
+    :returns: config section from ``pyproject.toml``
+    :rtype: dict
+
+    This basically reads entries from::
+
+        [tool.local.cythonize]
+        # Env vars provisioned during cythonize call
+        src = ["src/**/*.pyx"]
+
+        [tool.local.cythonize.env]
+        # Env vars provisioned during cythonize call
+        LDFLAGS = "-lssh"
+
+        [tool.local.cythonize.flags]
+        # This section can contain the following booleans:
+        # * annotate — generate annotated HTML page for source files
+        # * build — build extension modules using distutils
+        # * inplace — build extension modules in place using distutils (implies -b)
+        # * force — force recompilation
+        # * quiet — be less verbose during compilation
+        # * lenient — increase Python compat by ignoring some compile time errors
+        # * keep-going — compile as much as possible, ignore compilation failures
+        annotate = false
+        build = false
+        inplace = true
+        force = true
+        quiet = false
+        lenient = false
+        keep-going = false
+
+        [tool.local.cythonize.kwargs]
+        # This section can contain args tha have values:
+        # * exclude=PATTERN      exclude certain file patterns from the compilation
+        # * parallel=N    run builds in N parallel jobs (default: calculated per system)
+        exclude = "**.py"
+        parallel = 12
+
+        [tool.local.cythonize.kwargs.directives]
+        # This section can contain compiler directives
+        # NAME = "VALUE"
+
+        [tool.local.cythonize.kwargs.compile-time-env]
+        # This section can contain compile time env vars
+        # NAME = "VALUE"
+
+        [tool.local.cythonize.kwargs.options]
+        # This section can contain cythonize options
+        # NAME = "VALUE"
+    """
+    config_toml_txt = (Path.cwd().resolve() / 'pyproject.toml').read_text()
+    config_mapping = _load_toml_from_string(config_toml_txt)
+    return config_mapping['tool']['local']['cythonize']
+
+
+@contextmanager
+def patched_distutils_cmd_install():
+    """Make `install_lib` of `install` cmd always use `platlib`.
+
+    :yields: None
+    """
+    # Without this, build_lib puts stuff under `*.data/purelib/` folder
+    orig_finalize = _distutils_install_cmd.finalize_options
+
+    def new_finalize_options(self):  # noqa: WPS430
+        self.install_lib = self.install_platlib
+        orig_finalize(self)
+
+    _distutils_install_cmd.finalize_options = new_finalize_options
+    try:  # noqa: WPS501
+        yield
+    finally:
+        _distutils_install_cmd.finalize_options = orig_finalize
+
+
+@contextmanager
+def patched_dist_has_ext_modules():
+    """Make `has_ext_modules` of `Distribution` always return `True`.
+
+    :yields: None
+    """
+    # Without this, build_lib puts stuff under `*.data/platlib/` folder
+    orig_func = _DistutilsDistribution.has_ext_modules
+
+    _DistutilsDistribution.has_ext_modules = lambda *args, **kwargs: True
+    try:  # noqa: WPS501
+        yield
+    finally:
+        _DistutilsDistribution.has_ext_modules = orig_func
+
+
+@contextmanager
+def patched_dist_get_long_description():
+    """Make `has_ext_modules` of `Distribution` always return `True`.
+
+    :yields: None
+    """
+    # Without this, build_lib puts stuff under `*.data/platlib/` folder
+    _orig_func = _DistutilsDistributionMetadata.get_long_description
+
+    def _get_sanitized_long_description(self):
+        return sanitize_rst_roles(self.long_description)
+
+    _DistutilsDistributionMetadata.get_long_description = (
+        _get_sanitized_long_description
+    )
+    try:  # noqa: WPS501
+        yield
+    finally:
+        _DistutilsDistributionMetadata.get_long_description = _orig_func
+
+
+@contextmanager
+def patched_env(env):
+    """Temporary set given env vars.
+
+    :param env: tmp env vars to set
+    :type env: dict
+
+    :yields: None
+    """
+    orig_env = os.environ.copy()
+    expanded_env = {name: expandvars(var_val) for name, var_val in env.items()}
+    os.environ.update(expanded_env)
+    if os.getenv('YARL_CYTHON_TRACING') == '1':
+        os.environ['CFLAGS'] = ' '.join((
+            os.getenv('CFLAGS', ''),
+            '-DCYTHON_TRACE=1',
+            '-DCYTHON_TRACE_NOGIL=1',
+        )).strip()
+    try:  # noqa: WPS501
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(orig_env)
+
+
+@contextmanager
+def _run_in_temporary_directory() -> t.Iterator[Path]:
+    with TemporaryDirectory(prefix='.tmp-yarl-pep517-') as tmp_dir:
+        with chdir_cm(tmp_dir):
+            yield Path(tmp_dir)
+
+
+@patched_dist_get_long_description()
+def build_wheel(  # noqa: WPS210, WPS430
+         wheel_directory: str,
+         config_settings: dict[str, str] | None = None,
+         metadata_directory: str | None = None,
+) -> str:
+    build_c_ext_requested = BUILD_C_EXT_CONFIG_SETTING in (
+        config_settings or {}
+    )
+    if not build_c_ext_requested:
+        print("*********************", file=_standard_error_stream)
+        print("* Pure Python build *", file=_standard_error_stream)
+        print("*********************", file=_standard_error_stream)
+        return _setuptools_build_wheel(
+            wheel_directory=wheel_directory,
+            config_settings=config_settings,
+            metadata_directory=metadata_directory,
+        )
+
+    print("**********************", file=_standard_error_stream)
+    print("* Accelerated build *", file=_standard_error_stream)
+    print("**********************", file=_standard_error_stream)
+    if not IS_CPYTHON:
+        _warn_that(
+            'Building C-extensions under the runtimes other than CPython is '
+            'unsupported and will likely fail. Consider not passing the '
+            f'`{BUILD_C_EXT_CONFIG_SETTING !s}` PEP 517 config setting.',
+            RuntimeWarning,
+            stacklevel=999,
+        )
+
+    original_src_dir = Path.cwd().resolve()
+    with _run_in_temporary_directory() as tmp_dir:
+        tmp_src_dir = Path(tmp_dir) / 'src'
+        copytree(original_src_dir, tmp_src_dir, symlinks=True)
+        os.chdir(tmp_src_dir)
+
+        config = _get_local_cython_config()
+
+        py_ver_arg = f'-{_python_version_tuple.major!s}'
+
+        cli_flags = get_enabled_cli_flags_from_config(config['flags'])
+        cli_kwargs = get_cli_kwargs_from_config(config['kwargs'])
+
+        cythonize_args = cli_flags + [py_ver_arg] + cli_kwargs + config['src']
+        with patched_env(config['env']):
+            _cythonize_cli_cmd(cythonize_args)
+        with patched_distutils_cmd_install():
+            with patched_dist_has_ext_modules():
+                return _setuptools_build_wheel(
+                    wheel_directory=wheel_directory,
+                    config_settings=config_settings,
+                    metadata_directory=metadata_directory,
+                )
+
+
+def get_requires_for_build_wheel(
+        config_settings: dict[str, str] | None = None,
+) -> list[str]:
+    build_c_ext_requested = BUILD_C_EXT_CONFIG_SETTING in (
+        config_settings or {}
+    )
+
+    if build_c_ext_requested and not IS_CPYTHON:
+        _warn_that(
+            'Building C-extensions under the runtimes other than CPython is '
+            'unsupported and will likely fail. Consider not passing the '
+            f'`{BUILD_C_EXT_CONFIG_SETTING !s}` PEP 517 config setting.',
+            RuntimeWarning,
+            stacklevel=999,
+        )
+
+    c_ext_build_deps = [
+        'Cython >= 3.0.0b3' if IS_PY3_12_PLUS  # Only Cython 3+ is compatible
+        else 'Cython',
+    ] if build_c_ext_requested else []
+
+    return _setuptools_get_requires_for_build_wheel(
+        config_settings=config_settings,
+    ) + c_ext_build_deps
+
+
+build_sdist = patched_dist_get_long_description()(_setuptools_build_sdist)
+get_requires_for_build_editable = get_requires_for_build_wheel
+prepare_metadata_for_build_wheel = patched_dist_get_long_description()(_setuptools_prepare_metadata_for_build_wheel)
+prepare_metadata_for_build_editable = prepare_metadata_for_build_wheel