diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 5f48070f6..343c5cc9f 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -24,6 +24,7 @@ Unbuffered, detect_ci_provider, resources_dir, + read_python_configs, ) @@ -219,6 +220,8 @@ def main() -> None: manylinux_images[build_platform] = image + python_configs = read_python_configs(resources_dir / f"{platform}.toml") + build_options = BuildOptions( architectures=archs, package_dir=package_dir, @@ -235,6 +238,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images, + python_configs=python_configs, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' @@ -306,13 +310,16 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: def print_build_identifiers( platform: str, build_selector: BuildSelector, architectures: List[Architecture] ) -> None: + + python_configs = read_python_configs(resources_dir / f"{platform}.toml") + python_configurations: List[Any] = [] if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures) + python_configurations = cibuildwheel.linux.get_python_configurations(python_configs, build_selector, architectures) elif platform == 'windows': - python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) + python_configurations = cibuildwheel.windows.get_python_configurations(python_configs, build_selector) elif platform == 'macos': - python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) + python_configurations = cibuildwheel.macos.get_python_configurations(python_configs, build_selector) for config in python_configurations: print(config.identifier) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6a553ef73..067129f54 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -3,7 +3,7 @@ import textwrap from os import PathLike from pathlib import Path, PurePath -from typing import List, NamedTuple, Union +from typing import List, NamedTuple, Union, Dict from .docker_container import DockerContainer from .logger import log @@ -24,42 +24,13 @@ def path(self): def get_python_configurations( - build_selector: BuildSelector, architectures: List[Architecture] + python_configs: List[Dict[str, str]], + build_selector: BuildSelector, + architectures: List[Architecture] ) -> List[PythonConfiguration]: - python_configurations = [ - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_x86_64', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_i686', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path_str='/opt/python/pp27-pypy_73'), - PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path_str='/opt/python/pp36-pypy36_pp73'), - PythonConfiguration(version='3.7', identifier='pp37-manylinux_x86_64', path_str='/opt/python/pp37-pypy37_pp73'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_aarch64', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_ppc64le', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), - ] + + python_configurations = [PythonConfiguration(**item) for item in python_configs] + # return all configurations whose arch is in our `architectures` list, # and match the build/skip rules @@ -81,7 +52,7 @@ def build(options: BuildOptions) -> None: exit(2) assert options.manylinux_images is not None - python_configurations = get_python_configurations(options.build_selector, options.architectures) + python_configurations = get_python_configurations(options.python_configs, options.build_selector, options.architectures) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), ('cp', 'manylinux_i686', options.manylinux_images['i686']), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2db1fb4c9..128394f0d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -32,20 +32,11 @@ class PythonConfiguration(NamedTuple): url: str -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: - python_configurations = [ - # CPython - PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), - PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), - PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), - PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg'), - # PyPy - PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), - ] +def get_python_configurations( + python_configs: List[Dict[str, str]], + build_selector: BuildSelector) -> List[PythonConfiguration]: + + python_configurations = [PythonConfiguration(**item) for item in python_configs] # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] @@ -204,7 +195,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.python_configs, options.build_selector) for config in python_configurations: log.build_start(config.identifier) diff --git a/cibuildwheel/resources/linux.toml b/cibuildwheel/resources/linux.toml new file mode 100644 index 000000000..34ed9d4f2 --- /dev/null +++ b/cibuildwheel/resources/linux.toml @@ -0,0 +1,119 @@ +[cp27-manylinux_x86_64] +version="2.7" +path_str="/opt/python/cp27-cp27m" + +[cp35-manylinux_x86_64] +version="3.5" +path_str="/opt/python/cp35-cp35m" + +[cp36-manylinux_x86_64] +version="3.6" +path_str="/opt/python/cp36-cp36m" + +[cp37-manylinux_x86_64] +version="3.7" +path_str="/opt/python/cp37-cp37m" + +[cp38-manylinux_x86_64] +version="3.8" +path_str="/opt/python/cp38-cp38" + +[cp39-manylinux_x86_64] +version="3.9" +path_str="/opt/python/cp39-cp39" + +[cp27-manylinux_i686] +version="2.7" +path_str="/opt/python/cp27-cp27m" + +[cp35-manylinux_i686] +version="3.5" +path_str="/opt/python/cp35-cp35m" + +[cp36-manylinux_i686] +version="3.6" +path_str="/opt/python/cp36-cp36m" + +[cp37-manylinux_i686] +version="3.7" +path_str="/opt/python/cp37-cp37m" + +[cp38-manylinux_i686] +version="3.8" +path_str="/opt/python/cp38-cp38" + +[cp39-manylinux_i686] +version="3.9" +path_str="/opt/python/cp39-cp39" + +[pp27-manylinux_x86_64] +version="2.7" +path_str="/opt/python/pp27-pypy_73" + +[pp36-manylinux_x86_64] +version="3.6" +path_str="/opt/python/pp36-pypy36_pp73" + +[pp37-manylinux_x86_64] +version="3.7" +path_str="/opt/python/pp37-pypy37_pp73" + +[cp35-manylinux_aarch64] +version="3.5" +path_str="/opt/python/cp35-cp35m" + +[cp36-manylinux_aarch64] +version="3.6" +path_str="/opt/python/cp36-cp36m" + +[cp37-manylinux_aarch64] +version="3.7" +path_str="/opt/python/cp37-cp37m" + +[cp38-manylinux_aarch64] +version="3.8" +path_str="/opt/python/cp38-cp38" + +[cp39-manylinux_aarch64] +version="3.9" +path_str="/opt/python/cp39-cp39" + +[cp35-manylinux_ppc64le] +version="3.5" +path_str="/opt/python/cp35-cp35m" + +[cp36-manylinux_ppc64le] +version="3.6" +path_str="/opt/python/cp36-cp36m" + +[cp37-manylinux_ppc64le] +version="3.7" +path_str="/opt/python/cp37-cp37m" + +[cp38-manylinux_ppc64le] +version="3.8" +path_str="/opt/python/cp38-cp38" + +[cp39-manylinux_ppc64le] +version="3.9" +path_str="/opt/python/cp39-cp39" + +[cp35-manylinux_s390x] +version="3.5" +path_str="/opt/python/cp35-cp35m" + +[cp36-manylinux_s390x] +version="3.6" +path_str="/opt/python/cp36-cp36m" + +[cp37-manylinux_s390x] +version="3.7" +path_str="/opt/python/cp37-cp37m" + +[cp38-manylinux_s390x] +version="3.8" +path_str="/opt/python/cp38-cp38" + +[cp39-manylinux_s390x] +version="3.9" +path_str="/opt/python/cp39-cp39" diff --git a/cibuildwheel/resources/macos.toml b/cibuildwheel/resources/macos.toml new file mode 100644 index 000000000..17cb9776f --- /dev/null +++ b/cibuildwheel/resources/macos.toml @@ -0,0 +1,35 @@ +[cp27-macosx_x86_64] +version="2.7" +url="https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" + +[cp35-macosx_x86_64] +version="3.5" +url="https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg" + +[cp36-macosx_x86_64] +version="3.6" +url="https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" + +[cp37-macosx_x86_64] +version="3.7" +url="https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" + +[cp38-macosx_x86_64] +version="3.8" +url="https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg" + +[cp39-macosx_x86_64] +version="3.9" +url="https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg" + +[pp27-macosx_x86_64] +version="2.7" +url="https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" + +[pp36-macosx_x86_64] +version="3.6" +url="https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" + +[pp37-macosx_x86_64] +version="3.7" +url="https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" diff --git a/cibuildwheel/resources/windows.toml b/cibuildwheel/resources/windows.toml new file mode 100644 index 000000000..4196ebfe5 --- /dev/null +++ b/cibuildwheel/resources/windows.toml @@ -0,0 +1,62 @@ +[cp27-win32] +version="2.7.18" +arch="32" + +[cp27-win_amd64] +version="2.7.18" +arch="64" + +[cp35-win32] +version="3.5.4" +arch="32" + +[cp35-win_amd64] +version="3.5.4" +arch="64" + +[cp36-win32] +version="3.6.8" +arch="32" + +[cp36-win_amd64] +version="3.6.8" +arch="64" + +[cp37-win32] +version="3.7.9" +arch="32" + +[cp37-win_amd64] +version="3.7.9" +arch="64" + +[cp38-win32] +version="3.8.7" +arch="32" + +[cp38-win_amd64] +version="3.8.7" +arch="64" + +[cp39-win32] +version="3.9.1" +arch="32" + +[cp39-win_amd64] +version="3.9.1" +arch="64" + +[pp27-win32] +version="2.7" +arch="32" +url="https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip" + +[pp36-win32] +version="3.6" +arch="32" +url="https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip" + +[pp37-win32] +version="3.7" +arch="32" +url="https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 70eaa44d1..42d11867e 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -11,6 +11,7 @@ from typing import Dict, List, NamedTuple, Optional, Union import certifi +import toml from .environment import ParsedEnvironment @@ -34,6 +35,18 @@ def get_build_verbosity_extra_flags(level: int) -> List[str]: return [] +def read_python_configs(input_path: Union[str, os.PathLike]) -> List[Dict[str, str]]: + loaded_file = toml.load(input_path) + results: List[Dict[str, str]] = [] + + for k, v in loaded_file.items()] + results.append({'identifier': k, **v}) + if 'cp27-manylinux_' in k: + results.append({'identifier': k, 'version':v['version'], 'path':f'{v["path_str"]}u'} + + return results + + class BuildSelector: def __init__(self, build_config: str, skip_config: str): self.build_patterns = build_config.split() @@ -173,6 +186,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int + python_configs: List[Dict[str, str]] resources_dir = Path(__file__).resolve().parent / 'resources' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3abb61dde..79276b581 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -48,26 +48,11 @@ class PythonConfiguration(NamedTuple): url: Optional[str] -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: - python_configurations = [ - # CPython - PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), - PythonConfiguration(version='2.7.18', arch='64', identifier='cp27-win_amd64', url=None), - PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None), - PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None), - PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None), - PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), - PythonConfiguration(version='3.7.9', arch='32', identifier='cp37-win32', url=None), - PythonConfiguration(version='3.7.9', arch='64', identifier='cp37-win_amd64', url=None), - PythonConfiguration(version='3.8.7', arch='32', identifier='cp38-win32', url=None), - PythonConfiguration(version='3.8.7', arch='64', identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='3.9.1', arch='32', identifier='cp39-win32', url=None), - PythonConfiguration(version='3.9.1', arch='64', identifier='cp39-win_amd64', url=None), - # PyPy - PythonConfiguration(version='2.7', arch='32', identifier='pp27-win32', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip'), - PythonConfiguration(version='3.6', arch='32', identifier='pp36-win32', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip'), - PythonConfiguration(version='3.7', arch='32', identifier='pp37-win32', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip'), - ] +def get_python_configurations( + python_configs: List[Dict[str, str]], + build_selector: BuildSelector) -> List[PythonConfiguration]: + + python_configurations = [PythonConfiguration(**item) for item in python_configs] if IS_RUNNING_ON_TRAVIS: # cannot install VCForPython27.msi which is needed for compiling C software @@ -220,7 +205,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) shell(before_all_prepared, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.python_configs, options.build_selector) for config in python_configurations: log.build_start(config.identifier)