From 3c91348693e82b30b13089f551aad47bf486d0c6 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 15:38:30 -0800 Subject: [PATCH 1/6] pythonbuild: make download_to_path() concurrently safe I'm about to enable support for parallel CPython builds. This function was currently not safe if called in parallel. This was causing setuptools / pip downloads to race writing to the same file and failing builds. --- pythonbuild/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 1c1ebef0..3108670f 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -12,7 +12,9 @@ import os import pathlib import platform +import random import stat +import string import subprocess import sys import tarfile @@ -269,7 +271,15 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): path.unlink() - tmp = path.with_name("%s.tmp" % path.name) + # Need to write to random path to avoid race conditions. If there is a + # race, worst case we'll download the same file N>1 times. Meh. + tmp = path.with_name( + "%s.tmp%s" + % ( + path.name, + "".join(random.choices(string.ascii_uppercase + string.digits, k=8)), + ) + ) for attempt in range(5): try: From a3b21789bf5f83d5b199c02f96bd45dd389fcbd3 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:09:25 -0800 Subject: [PATCH 2/6] unix: move Python major minor version parsing to Makefile This is less ergonomic (oh Make). But it will make it easier to build multiple Python versions at the same time in a future commit. --- cpython-unix/Makefile | 9 ++++----- cpython-unix/build-main.py | 8 +++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 2827fc88..d1c9a86b 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -4,6 +4,7 @@ OUTDIR := $(ROOT)/build BUILD := $(HERE)/build.py NULL := +SPACE := $(subst ,, ) ifndef PYBUILD_TARGET_TRIPLE $(error PYBUILD_TARGET_TRIPLE not defined) @@ -25,9 +26,7 @@ ifndef PYBUILD_PYTHON_VERSION $(error PYBUILD_PYTHON_VERSION not defined) endif -ifndef PYBUILD_PYTHON_MAJOR_VERSION - $(error PYBUILD_PYTHON_MAJOR_VERSION not defined) -endif +PYTHON_MAJOR_VERSION := $(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(PYBUILD_PYTHON_VERSION)))) TARGET_TRIPLE := $(PYBUILD_TARGET_TRIPLE) HOST_PLATFORM := $(PYBUILD_HOST_PLATFORM) @@ -47,7 +46,7 @@ endif # Always write out settings files. $(shell $(RUN_BUILD) placeholder_archive makefiles) -include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYBUILD_PYTHON_MAJOR_VERSION) +include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYTHON_MAJOR_VERSION) include $(OUTDIR)/versions/VERSION.* # Always write out expanded Dockerfiles. @@ -274,7 +273,7 @@ PYTHON_DEPENDS := \ $(PYTHON_SUPPORT_FILES) \ $(OUTDIR)/versions/VERSION.pip \ $(OUTDIR)/versions/VERSION.setuptools \ - $(OUTDIR)/cpython-$(PYBUILD_PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ + $(OUTDIR)/cpython-$(PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \ $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \ $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \ diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 1e310b22..e837dd5d 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -156,7 +156,7 @@ def main(): return 1 cpython_version = env["PYBUILD_PYTHON_VERSION"] - env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2]) + python_majmin = ".".join(cpython_version.split(".")[0:2]) if "PYBUILD_RELEASE_TAG" in os.environ: release_tag = os.environ["PYBUILD_RELEASE_TAG"] @@ -164,12 +164,10 @@ def main(): release_tag = release_tag_from_git() # Guard against accidental misuse of the free-threaded flag with older versions - if "freethreaded" in args.options and env["PYBUILD_PYTHON_MAJOR_VERSION"] not in ( - "3.13" - ): + if "freethreaded" in args.options and python_majmin not in ("3.13",): print( "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+ (got %s)" - % env["PYBUILD_PYTHON_MAJOR_VERSION"] + % cpython_version ) return 1 From 64592149e689928722a0871ad49d427a5497f7e0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:58:45 -0800 Subject: [PATCH 3/6] unix: only generate 1 Makefile per build configuration Previously we generated the same Makefile for every Python version. We can simplify things by only writing 1 Makefile per target configuration. --- cpython-unix/Makefile | 2 +- pythonbuild/utils.py | 42 +++++++++++++++++++----------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index d1c9a86b..42dd31ca 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -46,7 +46,7 @@ endif # Always write out settings files. $(shell $(RUN_BUILD) placeholder_archive makefiles) -include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYTHON_MAJOR_VERSION) +include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE) include $(OUTDIR)/versions/VERSION.* # Always write out expanded Dockerfiles. diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 3108670f..d8c61abc 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -143,35 +143,31 @@ def write_triples_makefiles( for triple, settings in targets.items(): for host_platform in settings["host_platforms"]: - for python in settings["pythons_supported"]: - makefile_path = dest_dir / ( - "Makefile.%s.%s.%s" % (host_platform, triple, python) - ) + makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple)) - lines = [] - for need in settings.get("needs", []): - lines.append( - "NEED_%s := 1\n" - % need.upper().replace("-", "_").replace(".", "_") - ) + lines = [] + for need in settings.get("needs", []): + lines.append( + "NEED_%s := 1\n" % need.upper().replace("-", "_").replace(".", "_") + ) - image_suffix = settings.get("docker_image_suffix", "") + image_suffix = settings.get("docker_image_suffix", "") - lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) - lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) - entry = clang_toolchain(host_platform, triple) - lines.append( - "CLANG_FILENAME := %s-%s-%s.tar\n" - % (entry, DOWNLOADS[entry]["version"], host_platform) - ) + entry = clang_toolchain(host_platform, triple) + lines.append( + "CLANG_FILENAME := %s-%s-%s.tar\n" + % (entry, DOWNLOADS[entry]["version"], host_platform) + ) - lines.append( - "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" - % (support_search_dir / "extension-modules.yml") - ) + lines.append( + "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" + % (support_search_dir / "extension-modules.yml") + ) - write_if_different(makefile_path, "".join(lines).encode("ascii")) + write_if_different(makefile_path, "".join(lines).encode("ascii")) def write_package_versions(dest_path: pathlib.Path): From 881e4c1861afc569d5f593dce5e6684ea79a6f45 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:42:16 -0800 Subject: [PATCH 4/6] unix: templatize per version Makefile variables and targets This refactor allows targets for each major Python version to be defined simultaneously. This opens the door to a single `make` invocation building multiple versions in parallel. --- cpython-unix/Makefile | 111 ++++++++++++++++++------------------------ pythonbuild/utils.py | 6 +++ 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 42dd31ca..779bc12e 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -6,6 +6,8 @@ BUILD := $(HERE)/build.py NULL := SPACE := $(subst ,, ) +ALL_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 3.13 + ifndef PYBUILD_TARGET_TRIPLE $(error PYBUILD_TARGET_TRIPLE not defined) endif @@ -68,7 +70,7 @@ PYTHON_DEP_DEPENDS := \ $(TOOLCHAIN_DEPENDS) \ $(NULL) -default: $(OUTDIR)/cpython-$(PYBUILD_PYTHON_VERSION)-$(PACKAGE_SUFFIX).tar +default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar ifndef PYBUILD_NO_DOCKER $(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile @@ -254,65 +256,48 @@ PYTHON_HOST_DEPENDS := \ $(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar \ $(NULL) -$(OUTDIR)/cpython-3.9-$(CPYTHON_3.9_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9-host - -$(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10-host - -$(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host - -$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host - -$(OUTDIR)/cpython-3.13-$(CPYTHON_3.13_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13-host - -PYTHON_DEPENDS := \ - $(PYTHON_SUPPORT_FILES) \ - $(OUTDIR)/versions/VERSION.pip \ - $(OUTDIR)/versions/VERSION.setuptools \ - $(OUTDIR)/cpython-$(PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ - $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_EXPAT),$(OUTDIR)/expat-$(EXPAT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBEDIT),$(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI_3_3),$(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI),$(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_m4),$(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_MPDECIMAL),$(OUTDIR)/mpdecimal-$(MPDECIMAL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_NCURSES),$(OUTDIR)/ncurses-$(NCURSES_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_1_1),$(OUTDIR)/openssl-1.1-$(OPENSSL_1.1_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_3_0),$(OUTDIR)/openssl-3.0-$(OPENSSL_3.0_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_PATCHELF),$(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_SQLITE),$(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TCL),$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TK),$(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TIX),$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_UUID),$(OUTDIR)/uuid-$(UUID_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_XZ),$(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_ZLIB),$(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(NULL) - -ALL_PYTHON_DEPENDS = \ - $(PYTHON_DEP_DEPENDS) \ - $(HERE)/build-cpython.sh \ - $(PYTHON_DEPENDS) \ - $(NULL) - -$(OUTDIR)/cpython-$(CPYTHON_3.9_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9 - -$(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10 - -$(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11 - -$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12 - -$(OUTDIR)/cpython-$(CPYTHON_3.13_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13 +# Each X.Y Python version has its own set of variables and targets. This independent +# definition allows multiple Python versions to be built using the same Makefile +# invocation. +define python_version_template +PYTHON_DEPENDS_$(1) := \ + $$(PYTHON_SUPPORT_FILES) \ + $$(OUTDIR)/versions/VERSION.pip \ + $$(OUTDIR)/versions/VERSION.setuptools \ + $$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar \ + $$(if$$(NEED_AUTOCONF),$$(OUTDIR)/autoconf-$$(AUTOCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BDB),$$(OUTDIR)/bdb-$$(BDB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BZIP2),$$(OUTDIR)/bzip2-$$(BZIP2_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_EXPAT),$$(OUTDIR)/expat-$$(EXPAT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBEDIT),$$(OUTDIR)/libedit-$$(LIBEDIT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI_3_3),$$(OUTDIR)/libffi-3.3-$$(LIBFFI_3.3_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI),$$(OUTDIR)/libffi-$$(LIBFFI_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_m4),$$(OUTDIR)/m4-$$(M4_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_MPDECIMAL),$$(OUTDIR)/mpdecimal-$$(MPDECIMAL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_NCURSES),$$(OUTDIR)/ncurses-$$(NCURSES_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_3_0),$$(OUTDIR)/openssl-3.0-$$(OPENSSL_3.0_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(NULL) + +ALL_PYTHON_DEPENDS_$(1) = \ + $$(PYTHON_DEP_DEPENDS) \ + $$(HERE)/build-cpython.sh \ + $$(PYTHON_DEPENDS_$(1)) \ + $$(NULL) + +$$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar: $$(PYTHON_HOST_DEPENDS) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1)-host + +$$(OUTDIR)/cpython-$$(CPYTHON_$(1)_VERSION)-$$(PACKAGE_SUFFIX).tar: $$(ALL_PYTHON_DEPENDS_$(1)) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1) +endef + +$(foreach local_version,$(ALL_PYTHON_VERSIONS),$(eval $(call python_version_template,$(local_version)))) diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index d8c61abc..49113a7e 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -143,6 +143,12 @@ def write_triples_makefiles( for triple, settings in targets.items(): for host_platform in settings["host_platforms"]: + # IMPORTANT: if we ever vary the content of these Makefiles by + # Python versions, the variable names will need add the Python + # version and the Makefile references updated to point to specific + # versions. If we don't do that, multi-version builds will fail + # to work correctly. + makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple)) lines = [] From c5e974cea36fc87d77ce1c983ff731046612bd0a Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 13:24:46 -0800 Subject: [PATCH 5/6] unix: support building multiple Python versions in parallel `build-main.py` now passes in space delimited Python versions to build, which the Makefile can happily translate to appropriate make targets for us. We added support for `--python all` to allow building all supported Python versions. --- cpython-unix/Makefile | 8 ++--- cpython-unix/build-main.py | 61 +++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 779bc12e..a716f77e 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -24,11 +24,11 @@ ifndef PYBUILD_PYTHON_SOURCE $(error PYBUILD_PYTHON_SOURCE not defined) endif -ifndef PYBUILD_PYTHON_VERSION - $(error PYBUILD_PYTHON_VERSION not defined) +ifndef PYBUILD_PYTHON_VERSIONS + $(error PYBUILD_PYTHON_VERSIONS not defined) endif -PYTHON_MAJOR_VERSION := $(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(PYBUILD_PYTHON_VERSION)))) +BUILD_MAJOR_VERSIONS := $(foreach version,$(PYBUILD_PYTHON_VERSIONS),$(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(version))))) TARGET_TRIPLE := $(PYBUILD_TARGET_TRIPLE) HOST_PLATFORM := $(PYBUILD_HOST_PLATFORM) @@ -70,7 +70,7 @@ PYTHON_DEP_DEPENDS := \ $(TOOLCHAIN_DEPENDS) \ $(NULL) -default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar +default: $(foreach version,$(BUILD_MAJOR_VERSIONS),$(OUTDIR)/cpython-$(CPYTHON_$(version)_VERSION)-$(PACKAGE_SUFFIX).tar) ifndef PYBUILD_NO_DOCKER $(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index e837dd5d..ca60d4b8 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -63,6 +63,7 @@ def main(): parser.add_argument( "--python", choices={ + "all", "cpython-3.9", "cpython-3.10", "cpython-3.11", @@ -117,13 +118,23 @@ def main(): supported_pythons = {"cpython-%s" % p for p in settings["pythons_supported"]} - if args.python not in supported_pythons: + if args.python != "all" and args.python not in supported_pythons: print( "%s only supports following Pythons: %s" % (target_triple, ", ".join(supported_pythons)) ) return 1 + if args.python == "all": + if args.python_source: + print("--python-source is not compatible with --python all") + return 1 + + # This can be removed once 3.13 is the minimum version. + if "freethreaded" in args.options: + print("freedthreaded builds not compatible with --python all") + return 1 + python_source = ( (str(pathlib.Path(args.python_source).resolve())) if args.python_source @@ -145,41 +156,40 @@ def main(): if args.no_docker: env["PYBUILD_NO_DOCKER"] = "1" - if not args.python_source: - entry = DOWNLOADS[args.python] - env["PYBUILD_PYTHON_VERSION"] = cpython_version = entry["version"] + if args.python == "all": + cpython_versions = [ + DOWNLOADS["cpython-%s" % p]["version"] + for p in settings["pythons_supported"] + ] + + elif not args.python_source: + cpython_versions = [DOWNLOADS[args.python]["version"]] else: # TODO consider parsing version from source checkout. Or defining version # from CLI argument. if "PYBUILD_PYTHON_VERSION" not in env: print("PYBUILD_PYTHON_VERSION must be set when using `--python-source`") return 1 - cpython_version = env["PYBUILD_PYTHON_VERSION"] + cpython_versions = [env["PYBUILD_PYTHON_VERSION"]] + + env["PYBUILD_PYTHON_VERSIONS"] = " ".join(cpython_versions) - python_majmin = ".".join(cpython_version.split(".")[0:2]) + python_majmins = [".".join(v.split(".")[0:2]) for v in cpython_versions] if "PYBUILD_RELEASE_TAG" in os.environ: release_tag = os.environ["PYBUILD_RELEASE_TAG"] else: release_tag = release_tag_from_git() - # Guard against accidental misuse of the free-threaded flag with older versions - if "freethreaded" in args.options and python_majmin not in ("3.13",): + # Guard against accidental misuse of the free-threaded flag with older versions. + if "freethreaded" in args.options and any( + v not in ("3.13",) for v in python_majmins + ): print( - "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+ (got %s)" - % cpython_version + "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+" ) return 1 - archive_components = [ - "cpython-%s" % cpython_version, - target_triple, - args.options, - ] - - build_basename = "-".join(archive_components) + ".tar" - dist_basename = "-".join(archive_components + [release_tag]) - # We run make with static parallelism no greater than the machine's CPU count # because we can get some speedup from parallel operations. But we also don't # share a make job server with each build. So if we didn't limit the @@ -195,7 +205,18 @@ def main(): DIST.mkdir(exist_ok=True) if args.make_target == "default": - compress_python_archive(BUILD / build_basename, DIST, dist_basename) + # TODO this could be made parallel. + for version in cpython_versions: + archive_components = [ + "cpython-%s" % version, + target_triple, + args.options, + ] + + build_basename = "-".join(archive_components) + ".tar" + dist_basename = "-".join(archive_components + [release_tag]) + + compress_python_archive(BUILD / build_basename, DIST, dist_basename) if __name__ == "__main__": From d17e9dc28c51089dc081790789a953af3ac69069 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 13:40:44 -0800 Subject: [PATCH 6/6] ci: build multiple Python versions in parallel Our build system supports this now. So update CI to make use of the feature. On Linux, 3.13 freethreaded builds are still their own CI job. We may need to tweak `build-main.py` to allow intelligently selecting freethreaded compatible versions. We could potentially also allow building freethreaded in parallel with non-freethreaded. This can be done as a followup. --- .github/workflows/apple.yml | 38 +- .github/workflows/linux.yml | 740 +++++++----------------------------- 2 files changed, 137 insertions(+), 641 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index f20bc3c4..4fb3e19b 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -2,7 +2,7 @@ name: MacOS Python build on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -47,7 +47,7 @@ jobs: build: - target_triple: 'aarch64-apple-darwin' runner: macos-14 - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 @@ -58,10 +58,6 @@ jobs: py: 'cpython-3.9' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.10' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.10' @@ -71,10 +67,6 @@ jobs: py: 'cpython-3.10' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.11' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.11' @@ -84,10 +76,6 @@ jobs: py: 'cpython-3.11' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.12' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.12' @@ -97,10 +85,6 @@ jobs: py: 'cpython-3.12' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.13' @@ -128,7 +112,7 @@ jobs: # or LTO builds. - target_triple: 'x86_64-apple-darwin' runner: macos-13 - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 @@ -139,10 +123,6 @@ jobs: py: 'cpython-3.9' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.10' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.10' @@ -152,10 +132,6 @@ jobs: py: 'cpython-3.10' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.11' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.11' @@ -165,10 +141,6 @@ jobs: py: 'cpython-3.11' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.12' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.12' @@ -178,10 +150,6 @@ jobs: py: 'cpython-3.12' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.13' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c64628e8..1fc2d37d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,7 +2,7 @@ name: Linux Python build on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -120,688 +120,279 @@ jobs: fail-fast: false matrix: build: - # Cross-compiles can't do PGO. - - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' + # For non-PGO builds we use the `all` magic Python version to allow + # building multiple Python versions on the same job. This amortizes + # overheads of building shared dependencies. + # + # For PGO builds, keep separate CI jobs so there isn't interference + # from other jobs when profiling the builds. We could work around this + # in the build system by requiring serial execution of the Python + # build targets. But PGO also takes a while to run and a monolithic + # job may take too long to run if we build all Python versions serially. + # + # It doesn't make sense to combine different targets or options in the + # same job because dependencies will be different. Since they will be + # different, splitting into multiple jobs allows us to leverage more + # machines / parallelism to get builds completed faster. - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' + # Cross-compiles can't do PGO. - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' + py: 'all' options: 'debug' - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' + py: 'all' options: 'noopt' - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'noopt' - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'debug' - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'noopt' - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'noopt' - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'debug' - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'lto' - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'all' options: 'lto' - # musl doesn't support PGO. - - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.12' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.12' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.12' - options: 'debug' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.12' + + # GitHub Actions runners don't support x86-64-v4 so we can't PGO. + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' + options: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' + options: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' options: 'lto' - run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + # musl doesn't support PGO. + + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'lto' needs: @@ -879,15 +470,6 @@ jobs: matrix: build: - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'aarch64-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -898,15 +480,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'armv7-unknown-linux-gnueabi' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -917,15 +490,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'armv7-unknown-linux-gnueabihf' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -936,15 +500,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'mips-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -955,15 +510,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'mipsel-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -974,15 +520,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 's390x-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -993,15 +530,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'ppc64le-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug'