diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7445e80b..e7281574 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -240,6 +240,13 @@ jobs: chmod +x build/pythonbuild if [ "${{ matrix.run }}" == "true" ]; then + if [ "${{ matrix.libc }}" == "musl" ]; then + sudo apt install musl-dev + + # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH` + # as used in the musl builds. + unset LD_LIBRARY_PATH + fi EXTRA_ARGS="--run" fi @@ -324,6 +331,13 @@ jobs: chmod +x build/pythonbuild if [ "${{ matrix.run }}" == "true" ]; then + if [ "${{ matrix.libc }}" == "musl" ]; then + sudo apt install musl-dev + + # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH` + # as used in the musl builds. + unset LD_LIBRARY_PATH + fi EXTRA_ARGS="--run" fi diff --git a/ci-targets.yaml b/ci-targets.yaml index ec08bd9a..cd892ac9 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -257,6 +257,9 @@ linux: - "3.12" - "3.13" build_options: + - debug+static + - noopt+static + - lto+static - debug - noopt - lto @@ -273,6 +276,9 @@ linux: - "3.12" - "3.13" build_options: + - debug+static + - noopt+static + - lto+static - debug - noopt - lto @@ -289,6 +295,9 @@ linux: - "3.12" - "3.13" build_options: + - debug+static + - noopt+static + - lto+static - debug - noopt - lto @@ -305,6 +314,9 @@ linux: - "3.12" - "3.13" build_options: + - debug+static + - noopt+static + - lto+static - debug - noopt - lto diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index ea1b1c0a..522252ee 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -381,12 +381,8 @@ CONFIGURE_FLAGS=" --without-ensurepip ${EXTRA_CONFIGURE_FLAGS}" -if [ "${CC}" = "musl-clang" ]; then - CFLAGS="${CFLAGS} -static" - CPPFLAGS="${CPPFLAGS} -static" - LDFLAGS="${LDFLAGS} -static" - PYBUILD_SHARED=0 +if [ "${CC}" = "musl-clang" ]; then # In order to build the _blake2 extension module with SSE3+ instructions, we need # musl-clang to find headers that provide access to the intrinsics, as they are not # provided by musl. These are part of the include files that are part of clang. @@ -400,6 +396,13 @@ if [ "${CC}" = "musl-clang" ]; then fi cp "$h" /tools/host/include/ done +fi + +if [ -n "${CPYTHON_STATIC}" ]; then + CFLAGS="${CFLAGS} -static" + CPPFLAGS="${CPPFLAGS} -static" + LDFLAGS="${LDFLAGS} -static" + PYBUILD_SHARED=0 else CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared" PYBUILD_SHARED=1 @@ -640,21 +643,41 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0 LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} - # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before - # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED - # contains a slash, the explicit path is used. - patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} - - # libpython3.so isn't present in debug builds. - if [ -z "${CPYTHON_DEBUG}" ]; then - patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/lib/libpython3.so - fi - - if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + if [ "${CC}" == "musl-clang" ]; then + # musl does not support $ORIGIN in DT_NEEDED, so we use RPATH instead. This could be + # problematic, i.e., we could load the shared library from the wrong location if + # `LD_LIBRARY_PATH` is set, but there's not a clear alternative at this time. The + # long term solution is probably to statically link to libpython instead. + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} + + # libpython3.so isn't present in debug builds. + if [ -z "${CPYTHON_DEBUG}" ]; then + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/lib/libpython3.so + fi + + if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + fi + else + # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before + # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED contains a + # slash, the explicit path is used. patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} + + # libpython3.so isn't present in debug builds. + if [ -z "${CPYTHON_DEBUG}" ]; then + patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + ${ROOT}/out/python/install/lib/libpython3.so + fi + + if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + fi fi fi fi diff --git a/cpython-unix/build-libX11.sh b/cpython-unix/build-libX11.sh index 6870514c..e8f65ad4 100755 --- a/cpython-unix/build-libX11.sh +++ b/cpython-unix/build-libX11.sh @@ -75,6 +75,30 @@ if [ -n "${CROSS_COMPILING}" ]; then s390x-unknown-linux-gnu) EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" ;; + x86_64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + aarch64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + i686-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + mips-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + mipsel-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + ppc64le-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + riscv64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + s390x-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; *) echo "cross-compiling but malloc(0) override not set; failures possible" ;; diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index dd4d2d7c..6bf91b49 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -45,6 +45,7 @@ def main(): print("Unsupported build platform: %s" % sys.platform) return 1 + # Note these arguments must be synced with `build.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -54,10 +55,14 @@ def main(): help="Target host triple to build for", ) - optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"} + # Construct possible options, we use a set here for canonical ordering + options = set() + options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) + options.update({f"freethreaded+{option}" for option in options}) + options.update({f"{option}+static" for option in options}) parser.add_argument( "--options", - choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}), + choices=options, default="noopt", help="Build options to apply when compiling Python", ) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 1bf075ce..e31691e7 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -set -e +set -ex cd /build @@ -18,7 +18,43 @@ pushd musl-${MUSL_VERSION} # added reallocarray(), which gets used by at least OpenSSL. # Here, we disable this single function so as to not introduce # symbol dependencies on clients using an older musl version. -patch -p1 < +-#include +- +-void *reallocarray(void *ptr, size_t m, size_t n) +-{ +- if (n && m > -1 / n) { +- errno = ENOMEM; +- return 0; +- } +- +- return realloc(ptr, m * n); +-} +EOF +else + # There is a different patch for newer musl versions, used in static distributions + patch -p1 <>( allowed_libraries.extend(extra.iter().map(|x| x.to_string())); } - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}d.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}t.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}td.so.1.0", - python_major_minor - )); + if json.libpython_link_mode == "shared" { + if target_triple.contains("-musl") { + // On musl, we link to `libpython` and rely on `RUN PATH` + allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}d.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}t.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}td.so.1.0", python_major_minor)); + } else { + // On glibc, we can use `$ORIGIN` for relative, reloctable linking + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}d.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}t.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}td.so.1.0", + python_major_minor + )); + } + } + + if !json.build_options.contains("static") && target_triple.contains("-musl") { + // Allow linking musl `libc` + allowed_libraries.push("libc.so".to_string()); + } // Allow the _crypt extension module - and only it - to link against libcrypt, // which is no longer universally present in Linux distros. @@ -1719,8 +1735,7 @@ fn validate_distribution( }; let is_debug = dist_filename.contains("-debug-"); - - let is_static = triple.contains("unknown-linux-musl"); + let is_static = dist_filename.contains("+static"); let mut tf = crate::open_distribution_archive(dist_path)?; @@ -2074,12 +2089,17 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { std::fs::write(&test_file, PYTHON_VERIFICATIONS.as_bytes())?; eprintln!(" running interpreter tests (output should follow)"); - let output = duct::cmd(python_exe, [test_file.display().to_string()]) + let output = duct::cmd(&python_exe, [test_file.display().to_string()]) .stdout_to_stderr() .unchecked() .env("TARGET_TRIPLE", &python_json.target_triple) .env("BUILD_OPTIONS", &python_json.build_options) - .run()?; + .run() + .context(format!( + "Failed to run `{} {}`", + python_exe.display(), + test_file.display() + ))?; if !output.status.success() { errors.push("errors running interpreter tests".to_string()); diff --git a/src/verify_distribution.py b/src/verify_distribution.py index 46ac48c6..2131d7aa 100644 --- a/src/verify_distribution.py +++ b/src/verify_distribution.py @@ -53,7 +53,8 @@ def test_ctypes(self): import ctypes # pythonapi will be None on statically linked binaries. - if os.environ["TARGET_TRIPLE"].endswith("-unknown-linux-musl"): + is_static = "static" in os.environ["BUILD_OPTIONS"] + if is_static: self.assertIsNone(ctypes.pythonapi) else: self.assertIsNotNone(ctypes.pythonapi)