diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8f76c6a3a..a770a2569 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -20,6 +20,7 @@ Unbuffered, detect_ci_provider, resources_dir, + read_python_configs, ) @@ -120,6 +121,10 @@ def main() -> None: file=sys.stderr) exit(2) + if platform not in {"linux", "macos", "windows"}: + print(f'cibuildwheel: Unsupported platform: {platform}', file=sys.stderr) + exit(2) + package_dir = Path(args.package_dir) output_dir = Path(args.output_dir) @@ -215,6 +220,8 @@ def main() -> None: manylinux_images[build_platform] = image + python_configs = read_python_configs(resources_dir / 'build-platforms.toml', platform) + build_options = BuildOptions( architectures=archs, package_dir=package_dir, @@ -231,6 +238,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images, + python_configs=python_configs, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' @@ -247,9 +255,6 @@ def main() -> None: cibuildwheel.windows.build(build_options) elif platform == 'macos': cibuildwheel.macos.build(build_options) - else: - print(f'cibuildwheel: Unsupported platform: {platform}', file=sys.stderr) - exit(2) def detect_obsolete_options() -> None: @@ -302,13 +307,16 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: def print_build_identifiers( platform: str, build_selector: BuildSelector, architectures: Set[Architecture] ) -> None: + + python_configs = read_python_configs(resources_dir / "build-platforms.toml", platform) + python_configurations: List[Any] = [] if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures) + python_configurations = cibuildwheel.linux.get_python_configurations(python_configs, build_selector, architectures) elif platform == 'windows': - python_configurations = cibuildwheel.windows.get_python_configurations(build_selector, architectures) + python_configurations = cibuildwheel.windows.get_python_configurations(python_configs, build_selector, architectures) elif platform == 'macos': - python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) + python_configurations = cibuildwheel.macos.get_python_configurations(python_configs, build_selector) for config in python_configurations: print(config.identifier) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 7da89aeea..1db959d3c 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -2,7 +2,7 @@ import sys import textwrap from pathlib import Path, PurePath -from typing import List, NamedTuple, Set +from typing import List, NamedTuple, Dict, Set from .docker_container import DockerContainer from .logger import log @@ -29,42 +29,12 @@ def path(self) -> PurePath: def get_python_configurations( - build_selector: BuildSelector, architectures: Set[Architecture] + python_configs: List[Dict[str, str]], + build_selector: BuildSelector, + architectures: Set[Architecture] ) -> List[PythonConfiguration]: - python_configurations = [ - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_x86_64', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_i686', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path_str='/opt/python/pp27-pypy_73'), - PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path_str='/opt/python/pp36-pypy36_pp73'), - PythonConfiguration(version='3.7', identifier='pp37-manylinux_x86_64', path_str='/opt/python/pp37-pypy37_pp73'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_aarch64', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_ppc64le', path_str='/opt/python/cp39-cp39'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path_str='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path_str='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path_str='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), - ] + + python_configurations = [PythonConfiguration(**item) for item in python_configs] # return all configurations whose arch is in our `architectures` set, # and match the build/skip rules @@ -88,7 +58,7 @@ def build(options: BuildOptions) -> None: exit(2) assert options.manylinux_images is not None - python_configurations = get_python_configurations(options.build_selector, options.architectures) + python_configurations = get_python_configurations(options.python_configs, options.build_selector, options.architectures) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), ('cp', 'manylinux_i686', options.manylinux_images['i686']), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 519c8bc16..c62e90ad4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -39,20 +39,11 @@ class PythonConfiguration(NamedTuple): url: str -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: - python_configurations = [ - # CPython - PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), - PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), - PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), - PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg'), - # PyPy - PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), - ] +def get_python_configurations( + python_configs: List[Dict[str, str]], + build_selector: BuildSelector) -> List[PythonConfiguration]: + + python_configurations = [PythonConfiguration(**item) for item in python_configs] # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] @@ -207,7 +198,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.python_configs, options.build_selector) for config in python_configurations: log.build_start(config.identifier) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml new file mode 100644 index 000000000..2298e3f2d --- /dev/null +++ b/cibuildwheel/resources/build-platforms.toml @@ -0,0 +1,63 @@ +[tool.cibw.build-platforms] +linux = [ + { identifier = "cp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/cp27-cp27m" }, + { identifier = "cp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/cp27-cp27mu" }, + { identifier = "cp35-manylinux_x86_64", version = "3.5", path_str = "/opt/python/cp35-cp35m" }, + { identifier = "cp36-manylinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp27-manylinux_i686", version = "2.7", path_str = "/opt/python/cp27-cp27m" }, + { identifier = "cp27-manylinux_i686", version = "2.7", path_str = "/opt/python/cp27-cp27mu" }, + { identifier = "cp35-manylinux_i686", version = "3.5", path_str = "/opt/python/cp35-cp35m" }, + { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "pp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/pp27-pypy_73" }, + { identifier = "pp36-manylinux_x86_64", version = "3.6", path_str = "/opt/python/pp36-pypy36_pp73" }, + { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, + { identifier = "cp35-manylinux_aarch64", version = "3.5", path_str = "/opt/python/cp35-cp35m" }, + { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp35-manylinux_ppc64le", version = "3.5", path_str = "/opt/python/cp35-cp35m" }, + { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp35-manylinux_s390x", version = "3.5", path_str = "/opt/python/cp35-cp35m" }, + { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" }, +] +macos = [ + { identifier = "cp27-macosx_x86_64", version = "2.7", url = "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" }, + { identifier = "cp35-macosx_x86_64", version = "3.5", url = "https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg" }, + { identifier = "cp36-macosx_x86_64", version = "3.6", url = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" }, + { identifier = "cp37-macosx_x86_64", version = "3.7", url = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" }, + { identifier = "cp38-macosx_x86_64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg" }, + { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg" }, + { identifier = "pp27-macosx_x86_64", version = "2.7", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" }, + { identifier = "pp36-macosx_x86_64", version = "3.6", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" }, + { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" }, +] +windows = [ + { identifier = "cp27-win32", version = "2.7.18", arch = "32" }, + { identifier = "cp27-win_amd64", version = "2.7.18", arch = "64" }, + { identifier = "cp35-win32", version = "3.5.4", arch = "32" }, + { identifier = "cp35-win_amd64", version = "3.5.4", arch = "64" }, + { identifier = "cp36-win32", version = "3.6.8", arch = "32" }, + { identifier = "cp36-win_amd64", version = "3.6.8", arch = "64" }, + { identifier = "cp37-win32", version = "3.7.9", arch = "32" }, + { identifier = "cp37-win_amd64", version = "3.7.9", arch = "64" }, + { identifier = "cp38-win32", version = "3.8.7", arch = "32" }, + { identifier = "cp38-win_amd64", version = "3.8.7", arch = "64" }, + { identifier = "cp39-win32", version = "3.9.1", arch = "32" }, + { identifier = "cp39-win_amd64", version = "3.9.1", arch = "64" }, + { identifier = "pp27-win32", version = "2.7", arch = "32", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip" }, + { identifier = "pp36-win32", version = "3.6", arch = "32", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip" }, + { identifier = "pp37-win32", version = "3.7", arch = "32", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip" }, +] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 6435252f1..3129d9228 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -13,6 +13,7 @@ from typing import Dict, List, NamedTuple, Optional, Set import certifi +import toml from .environment import ParsedEnvironment from .typing import PathOrStr @@ -42,6 +43,12 @@ def get_build_verbosity_extra_flags(level: int) -> List[str]: return [] +def read_python_configs(input_path: PathOrStr, config: str) -> List[Dict[str, str]]: + loaded_file = toml.load(input_path) + results: List[Dict[str, str]] = list(loaded_file["tool"]["cibw"]["build-platforms"][config]) + return results + + class BuildSelector: def __init__(self, build_config: str, skip_config: str): self.build_patterns = build_config.split() @@ -188,6 +195,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int + python_configs: List[Dict[str, str]] resources_dir = Path(__file__).resolve().parent / 'resources' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 355fe0554..96a80ffec 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -52,35 +52,22 @@ class PythonConfiguration(NamedTuple): version: str arch: str identifier: str - url: Optional[str] + url: Optional[str] = None -def get_python_configurations(build_selector: BuildSelector, architectures: Set[Architecture]) -> List[PythonConfiguration]: +def get_python_configurations( + python_configs: List[Dict[str, str]], + build_selector: BuildSelector, + architectures: Set[Architecture], +) -> List[PythonConfiguration]: + + python_configurations = [PythonConfiguration(**item) for item in python_configs] + map_arch = { '32': Architecture.x86, '64': Architecture.AMD64, } - python_configurations = [ - # CPython - PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), - PythonConfiguration(version='2.7.18', arch='64', identifier='cp27-win_amd64', url=None), - PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None), - PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None), - PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None), - PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), - PythonConfiguration(version='3.7.9', arch='32', identifier='cp37-win32', url=None), - PythonConfiguration(version='3.7.9', arch='64', identifier='cp37-win_amd64', url=None), - PythonConfiguration(version='3.8.7', arch='32', identifier='cp38-win32', url=None), - PythonConfiguration(version='3.8.7', arch='64', identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='3.9.1', arch='32', identifier='cp39-win32', url=None), - PythonConfiguration(version='3.9.1', arch='64', identifier='cp39-win_amd64', url=None), - # PyPy - PythonConfiguration(version='2.7', arch='32', identifier='pp27-win32', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip'), - PythonConfiguration(version='3.6', arch='32', identifier='pp36-win32', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip'), - PythonConfiguration(version='3.7', arch='32', identifier='pp37-win32', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip'), - ] - if IS_RUNNING_ON_TRAVIS: # cannot install VCForPython27.msi which is needed for compiling C software # try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive @@ -230,7 +217,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) shell(before_all_prepared, env=env) - python_configurations = get_python_configurations(options.build_selector, options.architectures) + python_configurations = get_python_configurations(options.python_configs, options.build_selector, options.architectures) for config in python_configurations: log.build_start(config.identifier) diff --git a/unit_test/build_ids_test.py b/unit_test/build_ids_test.py new file mode 100644 index 000000000..99c5b57ed --- /dev/null +++ b/unit_test/build_ids_test.py @@ -0,0 +1,28 @@ +import toml +from toml.encoder import TomlEncoder + +from cibuildwheel.util import resources_dir + + +class InlineArrayDictEncoder(TomlEncoder): + def dump_sections(self, o: dict, sup: str): + if all(isinstance(a, list) for a in o.values()): + val = "" + for k, v in o.items(): + inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) + val += f"{k} = [\n {inner},\n]\n" + return val, self._dict() + else: + return super().dump_sections(o, sup) + + +def test_compare_configs(): + with open(resources_dir / "build-platforms.toml") as f: + txt = f.read() + + dict_txt = toml.loads(txt) + + new_txt = toml.dumps(dict_txt, encoder=InlineArrayDictEncoder()) + print(new_txt) + + assert new_txt == txt