diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index f20bc3c4..4fb3e19b 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -2,7 +2,7 @@ name: MacOS Python build on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -47,7 +47,7 @@ jobs: build: - target_triple: 'aarch64-apple-darwin' runner: macos-14 - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 @@ -58,10 +58,6 @@ jobs: py: 'cpython-3.9' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.10' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.10' @@ -71,10 +67,6 @@ jobs: py: 'cpython-3.10' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.11' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.11' @@ -84,10 +76,6 @@ jobs: py: 'cpython-3.11' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.12' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.12' @@ -97,10 +85,6 @@ jobs: py: 'cpython-3.12' options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'debug' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.13' @@ -128,7 +112,7 @@ jobs: # or LTO builds. - target_triple: 'x86_64-apple-darwin' runner: macos-13 - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 @@ -139,10 +123,6 @@ jobs: py: 'cpython-3.9' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.10' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.10' @@ -152,10 +132,6 @@ jobs: py: 'cpython-3.10' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.11' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.11' @@ -165,10 +141,6 @@ jobs: py: 'cpython-3.11' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.12' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.12' @@ -178,10 +150,6 @@ jobs: py: 'cpython-3.12' options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'debug' - target_triple: 'x86_64-apple-darwin' runner: macos-13 py: 'cpython-3.13' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c64628e8..1fc2d37d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,7 +2,7 @@ name: Linux Python build on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -120,688 +120,279 @@ jobs: fail-fast: false matrix: build: - # Cross-compiles can't do PGO. - - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' + # For non-PGO builds we use the `all` magic Python version to allow + # building multiple Python versions on the same job. This amortizes + # overheads of building shared dependencies. + # + # For PGO builds, keep separate CI jobs so there isn't interference + # from other jobs when profiling the builds. We could work around this + # in the build system by requiring serial execution of the Python + # build targets. But PGO also takes a while to run and a monolithic + # job may take too long to run if we build all Python versions serially. + # + # It doesn't make sense to combine different targets or options in the + # same job because dependencies will be different. Since they will be + # different, splitting into multiple jobs allows us to leverage more + # machines / parallelism to get builds completed faster. - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' + # Cross-compiles can't do PGO. - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' + py: 'all' options: 'debug' - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' + py: 'all' options: 'noopt' - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'noopt' - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'debug' - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'noopt' - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'debug' - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' + py: 'all' options: 'noopt' - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'lto' # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' + py: 'all' options: 'debug' - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'lto' - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64_v2-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'pgo+lto' - run: true - - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo' - run: true - - target_triple: 'x86_64_v3-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'pgo+lto' - run: true - - # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.9' - options: 'lto' - - # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.10' - options: 'lto' - - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.11' - options: 'lto' - - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-gnu' - py: 'cpython-3.12' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'all' options: 'lto' - # musl doesn't support PGO. - - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64-unknown-linux-musl' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64-unknown-linux-musl' + - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.12' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' - py: 'cpython-3.12' - options: 'debug' - run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v2-unknown-linux-musl' + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.12' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.9' + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'all' options: 'debug' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.10' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.10' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.10' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.11' - options: 'debug' - run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.11' - options: 'noopt' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.11' - options: 'lto' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.12' - options: 'debug' + options: 'pgo' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.12' - options: 'noopt' + options: 'pgo+lto' run: true - - target_triple: 'x86_64_v3-unknown-linux-musl' - py: 'cpython-3.12' + + # GitHub Actions runners don't support x86-64-v4 so we can't PGO. + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' + options: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' + options: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'all' options: 'lto' - run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + # musl doesn't support PGO. + + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.9' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.10' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'debug' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'noopt' - - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.11' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'all' options: 'lto' + run: true - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'debug' - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'noopt' - target_triple: 'x86_64_v4-unknown-linux-musl' - py: 'cpython-3.12' + py: 'all' options: 'lto' needs: @@ -879,15 +470,6 @@ jobs: matrix: build: - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'aarch64-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'aarch64-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -898,15 +480,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabi' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'armv7-unknown-linux-gnueabi' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -917,15 +490,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'armv7-unknown-linux-gnueabihf' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'armv7-unknown-linux-gnueabihf' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -936,15 +500,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'mips-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'mips-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -955,15 +510,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'mipsel-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'mipsel-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -974,15 +520,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 's390x-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 's390x-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' @@ -993,15 +530,6 @@ jobs: py: 'cpython-3.13' options: 'freethreaded+lto' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'noopt' - - target_triple: 'ppc64le-unknown-linux-gnu' - py: 'cpython-3.13' - options: 'lto' - target_triple: 'ppc64le-unknown-linux-gnu' py: 'cpython-3.13' options: 'freethreaded+debug' diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 2827fc88..a716f77e 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -4,6 +4,9 @@ OUTDIR := $(ROOT)/build BUILD := $(HERE)/build.py NULL := +SPACE := $(subst ,, ) + +ALL_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 3.13 ifndef PYBUILD_TARGET_TRIPLE $(error PYBUILD_TARGET_TRIPLE not defined) @@ -21,13 +24,11 @@ ifndef PYBUILD_PYTHON_SOURCE $(error PYBUILD_PYTHON_SOURCE not defined) endif -ifndef PYBUILD_PYTHON_VERSION - $(error PYBUILD_PYTHON_VERSION not defined) +ifndef PYBUILD_PYTHON_VERSIONS + $(error PYBUILD_PYTHON_VERSIONS not defined) endif -ifndef PYBUILD_PYTHON_MAJOR_VERSION - $(error PYBUILD_PYTHON_MAJOR_VERSION not defined) -endif +BUILD_MAJOR_VERSIONS := $(foreach version,$(PYBUILD_PYTHON_VERSIONS),$(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(version))))) TARGET_TRIPLE := $(PYBUILD_TARGET_TRIPLE) HOST_PLATFORM := $(PYBUILD_HOST_PLATFORM) @@ -47,7 +48,7 @@ endif # Always write out settings files. $(shell $(RUN_BUILD) placeholder_archive makefiles) -include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYBUILD_PYTHON_MAJOR_VERSION) +include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE) include $(OUTDIR)/versions/VERSION.* # Always write out expanded Dockerfiles. @@ -69,7 +70,7 @@ PYTHON_DEP_DEPENDS := \ $(TOOLCHAIN_DEPENDS) \ $(NULL) -default: $(OUTDIR)/cpython-$(PYBUILD_PYTHON_VERSION)-$(PACKAGE_SUFFIX).tar +default: $(foreach version,$(BUILD_MAJOR_VERSIONS),$(OUTDIR)/cpython-$(CPYTHON_$(version)_VERSION)-$(PACKAGE_SUFFIX).tar) ifndef PYBUILD_NO_DOCKER $(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile @@ -255,65 +256,48 @@ PYTHON_HOST_DEPENDS := \ $(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar \ $(NULL) -$(OUTDIR)/cpython-3.9-$(CPYTHON_3.9_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9-host - -$(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10-host - -$(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host - -$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host - -$(OUTDIR)/cpython-3.13-$(CPYTHON_3.13_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13-host - -PYTHON_DEPENDS := \ - $(PYTHON_SUPPORT_FILES) \ - $(OUTDIR)/versions/VERSION.pip \ - $(OUTDIR)/versions/VERSION.setuptools \ - $(OUTDIR)/cpython-$(PYBUILD_PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ - $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_EXPAT),$(OUTDIR)/expat-$(EXPAT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBEDIT),$(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI_3_3),$(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI),$(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_m4),$(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_MPDECIMAL),$(OUTDIR)/mpdecimal-$(MPDECIMAL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_NCURSES),$(OUTDIR)/ncurses-$(NCURSES_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_1_1),$(OUTDIR)/openssl-1.1-$(OPENSSL_1.1_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_3_0),$(OUTDIR)/openssl-3.0-$(OPENSSL_3.0_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_PATCHELF),$(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_SQLITE),$(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TCL),$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TK),$(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TIX),$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_UUID),$(OUTDIR)/uuid-$(UUID_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_XZ),$(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_ZLIB),$(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(NULL) - -ALL_PYTHON_DEPENDS = \ - $(PYTHON_DEP_DEPENDS) \ - $(HERE)/build-cpython.sh \ - $(PYTHON_DEPENDS) \ - $(NULL) - -$(OUTDIR)/cpython-$(CPYTHON_3.9_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9 - -$(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10 - -$(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11 - -$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12 - -$(OUTDIR)/cpython-$(CPYTHON_3.13_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13 +# Each X.Y Python version has its own set of variables and targets. This independent +# definition allows multiple Python versions to be built using the same Makefile +# invocation. +define python_version_template +PYTHON_DEPENDS_$(1) := \ + $$(PYTHON_SUPPORT_FILES) \ + $$(OUTDIR)/versions/VERSION.pip \ + $$(OUTDIR)/versions/VERSION.setuptools \ + $$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar \ + $$(if$$(NEED_AUTOCONF),$$(OUTDIR)/autoconf-$$(AUTOCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BDB),$$(OUTDIR)/bdb-$$(BDB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BZIP2),$$(OUTDIR)/bzip2-$$(BZIP2_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_EXPAT),$$(OUTDIR)/expat-$$(EXPAT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBEDIT),$$(OUTDIR)/libedit-$$(LIBEDIT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI_3_3),$$(OUTDIR)/libffi-3.3-$$(LIBFFI_3.3_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI),$$(OUTDIR)/libffi-$$(LIBFFI_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_m4),$$(OUTDIR)/m4-$$(M4_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_MPDECIMAL),$$(OUTDIR)/mpdecimal-$$(MPDECIMAL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_NCURSES),$$(OUTDIR)/ncurses-$$(NCURSES_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_3_0),$$(OUTDIR)/openssl-3.0-$$(OPENSSL_3.0_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(NULL) + +ALL_PYTHON_DEPENDS_$(1) = \ + $$(PYTHON_DEP_DEPENDS) \ + $$(HERE)/build-cpython.sh \ + $$(PYTHON_DEPENDS_$(1)) \ + $$(NULL) + +$$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar: $$(PYTHON_HOST_DEPENDS) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1)-host + +$$(OUTDIR)/cpython-$$(CPYTHON_$(1)_VERSION)-$$(PACKAGE_SUFFIX).tar: $$(ALL_PYTHON_DEPENDS_$(1)) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1) +endef + +$(foreach local_version,$(ALL_PYTHON_VERSIONS),$(eval $(call python_version_template,$(local_version)))) diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 1e310b22..ca60d4b8 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -63,6 +63,7 @@ def main(): parser.add_argument( "--python", choices={ + "all", "cpython-3.9", "cpython-3.10", "cpython-3.11", @@ -117,13 +118,23 @@ def main(): supported_pythons = {"cpython-%s" % p for p in settings["pythons_supported"]} - if args.python not in supported_pythons: + if args.python != "all" and args.python not in supported_pythons: print( "%s only supports following Pythons: %s" % (target_triple, ", ".join(supported_pythons)) ) return 1 + if args.python == "all": + if args.python_source: + print("--python-source is not compatible with --python all") + return 1 + + # This can be removed once 3.13 is the minimum version. + if "freethreaded" in args.options: + print("freedthreaded builds not compatible with --python all") + return 1 + python_source = ( (str(pathlib.Path(args.python_source).resolve())) if args.python_source @@ -145,43 +156,40 @@ def main(): if args.no_docker: env["PYBUILD_NO_DOCKER"] = "1" - if not args.python_source: - entry = DOWNLOADS[args.python] - env["PYBUILD_PYTHON_VERSION"] = cpython_version = entry["version"] + if args.python == "all": + cpython_versions = [ + DOWNLOADS["cpython-%s" % p]["version"] + for p in settings["pythons_supported"] + ] + + elif not args.python_source: + cpython_versions = [DOWNLOADS[args.python]["version"]] else: # TODO consider parsing version from source checkout. Or defining version # from CLI argument. if "PYBUILD_PYTHON_VERSION" not in env: print("PYBUILD_PYTHON_VERSION must be set when using `--python-source`") return 1 - cpython_version = env["PYBUILD_PYTHON_VERSION"] + cpython_versions = [env["PYBUILD_PYTHON_VERSION"]] + + env["PYBUILD_PYTHON_VERSIONS"] = " ".join(cpython_versions) - env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2]) + python_majmins = [".".join(v.split(".")[0:2]) for v in cpython_versions] if "PYBUILD_RELEASE_TAG" in os.environ: release_tag = os.environ["PYBUILD_RELEASE_TAG"] else: release_tag = release_tag_from_git() - # Guard against accidental misuse of the free-threaded flag with older versions - if "freethreaded" in args.options and env["PYBUILD_PYTHON_MAJOR_VERSION"] not in ( - "3.13" + # Guard against accidental misuse of the free-threaded flag with older versions. + if "freethreaded" in args.options and any( + v not in ("3.13",) for v in python_majmins ): print( - "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+ (got %s)" - % env["PYBUILD_PYTHON_MAJOR_VERSION"] + "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+" ) return 1 - archive_components = [ - "cpython-%s" % cpython_version, - target_triple, - args.options, - ] - - build_basename = "-".join(archive_components) + ".tar" - dist_basename = "-".join(archive_components + [release_tag]) - # We run make with static parallelism no greater than the machine's CPU count # because we can get some speedup from parallel operations. But we also don't # share a make job server with each build. So if we didn't limit the @@ -197,7 +205,18 @@ def main(): DIST.mkdir(exist_ok=True) if args.make_target == "default": - compress_python_archive(BUILD / build_basename, DIST, dist_basename) + # TODO this could be made parallel. + for version in cpython_versions: + archive_components = [ + "cpython-%s" % version, + target_triple, + args.options, + ] + + build_basename = "-".join(archive_components) + ".tar" + dist_basename = "-".join(archive_components + [release_tag]) + + compress_python_archive(BUILD / build_basename, DIST, dist_basename) if __name__ == "__main__": diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 1c1ebef0..49113a7e 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -12,7 +12,9 @@ import os import pathlib import platform +import random import stat +import string import subprocess import sys import tarfile @@ -141,35 +143,37 @@ def write_triples_makefiles( for triple, settings in targets.items(): for host_platform in settings["host_platforms"]: - for python in settings["pythons_supported"]: - makefile_path = dest_dir / ( - "Makefile.%s.%s.%s" % (host_platform, triple, python) - ) - - lines = [] - for need in settings.get("needs", []): - lines.append( - "NEED_%s := 1\n" - % need.upper().replace("-", "_").replace(".", "_") - ) - - image_suffix = settings.get("docker_image_suffix", "") + # IMPORTANT: if we ever vary the content of these Makefiles by + # Python versions, the variable names will need add the Python + # version and the Makefile references updated to point to specific + # versions. If we don't do that, multi-version builds will fail + # to work correctly. - lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) - lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) + makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple)) - entry = clang_toolchain(host_platform, triple) + lines = [] + for need in settings.get("needs", []): lines.append( - "CLANG_FILENAME := %s-%s-%s.tar\n" - % (entry, DOWNLOADS[entry]["version"], host_platform) + "NEED_%s := 1\n" % need.upper().replace("-", "_").replace(".", "_") ) - lines.append( - "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" - % (support_search_dir / "extension-modules.yml") - ) + image_suffix = settings.get("docker_image_suffix", "") - write_if_different(makefile_path, "".join(lines).encode("ascii")) + lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) + + entry = clang_toolchain(host_platform, triple) + lines.append( + "CLANG_FILENAME := %s-%s-%s.tar\n" + % (entry, DOWNLOADS[entry]["version"], host_platform) + ) + + lines.append( + "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" + % (support_search_dir / "extension-modules.yml") + ) + + write_if_different(makefile_path, "".join(lines).encode("ascii")) def write_package_versions(dest_path: pathlib.Path): @@ -269,7 +273,15 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): path.unlink() - tmp = path.with_name("%s.tmp" % path.name) + # Need to write to random path to avoid race conditions. If there is a + # race, worst case we'll download the same file N>1 times. Meh. + tmp = path.with_name( + "%s.tmp%s" + % ( + path.name, + "".join(random.choices(string.ascii_uppercase + string.digits, k=8)), + ) + ) for attempt in range(5): try: