Add dynamically linked musl distributions

.github/workflows/linux.yml

@@ -240,6 +240,13 @@ jobs:
           chmod +x build/pythonbuild
 
           if [ "${{ matrix.run }}" == "true" ]; then
+            if [ "${{ matrix.libc }}" == "musl" ]; then
+              sudo apt install musl-dev
+
+              # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH`
+              # as used in the musl builds.
+              unset LD_LIBRARY_PATH
+            fi
             EXTRA_ARGS="--run"
           fi
 
@@ -324,6 +331,13 @@ jobs:
           chmod +x build/pythonbuild
 
           if [ "${{ matrix.run }}" == "true" ]; then
+            if [ "${{ matrix.libc }}" == "musl" ]; then
+              sudo apt install musl-dev
+
+              # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH`
+              # as used in the musl builds.
+              unset LD_LIBRARY_PATH
+            fi
             EXTRA_ARGS="--run"
           fi
 

ci-targets.yaml

@@ -257,6 +257,9 @@ linux:
       - "3.12"
       - "3.13"
     build_options:
+      - debug+static
+      - noopt+static
+      - lto+static
       - debug
       - noopt
       - lto
@@ -273,6 +276,9 @@ linux:
       - "3.12"
       - "3.13"
     build_options:
+      - debug+static
+      - noopt+static
+      - lto+static
       - debug
       - noopt
       - lto
@@ -289,6 +295,9 @@ linux:
       - "3.12"
       - "3.13"
     build_options:
+      - debug+static
+      - noopt+static
+      - lto+static
       - debug
       - noopt
       - lto
@@ -305,6 +314,9 @@ linux:
       - "3.12"
       - "3.13"
     build_options:
+      - debug+static
+      - noopt+static
+      - lto+static
       - debug
       - noopt
       - lto

cpython-unix/build-cpython.sh

@@ -381,12 +381,8 @@ CONFIGURE_FLAGS="
     --without-ensurepip
     ${EXTRA_CONFIGURE_FLAGS}"
 
-if [ "${CC}" = "musl-clang" ]; then
-    CFLAGS="${CFLAGS} -static"
-    CPPFLAGS="${CPPFLAGS} -static"
-    LDFLAGS="${LDFLAGS} -static"
-    PYBUILD_SHARED=0
 
+if [ "${CC}" = "musl-clang" ]; then
     # In order to build the _blake2 extension module with SSE3+ instructions, we need
     # musl-clang to find headers that provide access to the intrinsics, as they are not
     # provided by musl. These are part of the include files that are part of clang.
@@ -400,6 +396,13 @@ if [ "${CC}" = "musl-clang" ]; then
         fi
         cp "$h" /tools/host/include/
     done
+fi
+
+if [ -n "${CPYTHON_STATIC}" ]; then
+    CFLAGS="${CFLAGS} -static"
+    CPPFLAGS="${CPPFLAGS} -static"
+    LDFLAGS="${LDFLAGS} -static"
+    PYBUILD_SHARED=0 
 else
     CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared"
     PYBUILD_SHARED=1
@@ -640,21 +643,41 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
         LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0
         LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
 
-        # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before
-        # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED
-        # contains a slash, the explicit path is used.
-        patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
-            ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}
-
-        # libpython3.so isn't present in debug builds.
-        if [ -z "${CPYTHON_DEBUG}" ]; then
-            patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
-                ${ROOT}/out/python/install/lib/libpython3.so
-        fi
-
-        if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+        if [ "${CC}" == "musl-clang" ]; then
+            # musl does not support $ORIGIN in DT_NEEDED, so we use RPATH instead. This could be
+            # problematic, i.e., we could load the shared library from the wrong location if
+            # `LD_LIBRARY_PATH` is set, but there's not a clear alternative at this time. The
+            # long term solution is probably to statically link to libpython instead.
+            patchelf --set-rpath "\$ORIGIN/../lib" \
+                ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}
+
+            # libpython3.so isn't present in debug builds.
+            if [ -z "${CPYTHON_DEBUG}" ]; then
+                patchelf --set-rpath "\$ORIGIN/../lib" \
+                    ${ROOT}/out/python/install/lib/libpython3.so
+            fi
+
+            if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+                patchelf --set-rpath "\$ORIGIN/../lib" \
+                    ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}
+            fi
+        else
+            # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before
+            # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED contains a
+            # slash, the explicit path is used.
             patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
-                ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}
+                ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}
+
+            # libpython3.so isn't present in debug builds.
+            if [ -z "${CPYTHON_DEBUG}" ]; then
+                patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
+                    ${ROOT}/out/python/install/lib/libpython3.so
+            fi
+
+            if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+                patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
+                    ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}
+            fi
         fi
     fi
 fi

cpython-unix/build-libX11.sh

@@ -75,6 +75,30 @@ if [ -n "${CROSS_COMPILING}" ]; then
     s390x-unknown-linux-gnu)
       EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
       ;;
+    x86_64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    aarch64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    i686-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    mips-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    mipsel-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    ppc64le-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    riscv64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    s390x-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
     *)
       echo "cross-compiling but malloc(0) override not set; failures possible"
       ;;

cpython-unix/build-main.py

@@ -45,6 +45,7 @@ def main():
         print("Unsupported build platform: %s" % sys.platform)
         return 1
 
+    # Note these arguments must be synced with `build.py`
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
@@ -54,10 +55,14 @@ def main():
         help="Target host triple to build for",
     )
 
-    optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"}
+    # Construct possible options, we use a set here for canonical ordering
+    options = set()
+    options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"})
+    options.update({f"freethreaded+{option}" for option in options})
+    options.update({f"{option}+static" for option in options})
     parser.add_argument(
         "--options",
-        choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}),
+        choices=options,
         default="noopt",
         help="Build options to apply when compiling Python",
     )

cpython-unix/build-musl.sh

@@ -3,7 +3,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-set -e
+set -ex
 
 cd /build
 
@@ -18,7 +18,43 @@ pushd musl-${MUSL_VERSION}
 # added reallocarray(), which gets used by at least OpenSSL.
 # Here, we disable this single function so as to not introduce
 # symbol dependencies on clients using an older musl version.
-patch -p1 <<EOF
+if [ "${MUSL_VERSION}" = "1.2.2" ]; then
+    patch -p1 <<EOF
+diff --git a/include/stdlib.h b/include/stdlib.h
+index b54a051f..194c2033 100644
+--- a/include/stdlib.h
++++ b/include/stdlib.h
+@@ -145,7 +145,6 @@ int getloadavg(double *, int);
+ int clearenv(void);
+ #define WCOREDUMP(s) ((s) & 0x80)
+ #define WIFCONTINUED(s) ((s) == 0xffff)
+-void *reallocarray (void *, size_t, size_t);
+ #endif
+
+ #ifdef _GNU_SOURCE
+diff --git a/src/malloc/reallocarray.c b/src/malloc/reallocarray.c
+deleted file mode 100644
+index 4a6ebe46..00000000
+--- a/src/malloc/reallocarray.c
++++ /dev/null
+@@ -1,13 +0,0 @@
+-#define _BSD_SOURCE
+-#include <errno.h>
+-#include <stdlib.h>
+-
+-void *reallocarray(void *ptr, size_t m, size_t n)
+-{
+-	if (n && m > -1 / n) {
+-		errno = ENOMEM;
+-		return 0;
+-	}
+-
+-	return realloc(ptr, m * n);
+-}
+EOF
+else
+    # There is a different patch for newer musl versions, used in static distributions
+    patch -p1 <<EOF
 diff --git a/include/stdlib.h b/include/stdlib.h
 index b507ca3..8259e27 100644
 --- a/include/stdlib.h
@@ -51,10 +87,20 @@ index 4a6ebe4..0000000
 -	return realloc(ptr, m * n);
 -}
 EOF
+fi
+
+SHARED=
+if [ -n "${STATIC}" ]; then
+    SHARED="--disable-shared"
+else
+    SHARED="--enable-shared"
+    CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC"
+fi
+
 
-./configure \
+CFLAGS="${CFLAGS}" CPPFLAGS="${CPPFLAGS}" ./configure \
     --prefix=/tools/host \
-    --disable-shared
+    "${SHARED}"
 
 make -j `nproc`
 make -j `nproc` install DESTDIR=/build/out

cpython-unix/build-tcl.sh

@@ -13,6 +13,15 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib
 tar -xf tcl${TCL_VERSION}-src.tar.gz
 pushd tcl${TCL_VERSION}
 
+
+if [ -n "${STATIC}" ]; then
+	if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then
+		# tcl misbehaves when performing static musl builds
+		# `checking whether musl-clang accepts -g...` fails with a duplicate definition error
+		TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
+	fi
+fi
+
 patch -p1 << 'EOF'
 diff --git a/unix/Makefile.in b/unix/Makefile.in
 --- a/unix/Makefile.in

cpython-unix/build-xextproto.sh

@@ -15,10 +15,21 @@ export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig
 tar -xf xextproto-${XEXTPROTO_VERSION}.tar.gz
 pushd xextproto-${XEXTPROTO_VERSION}
 
+EXTRA_CONFIGURE_FLAGS=
+if [ -n "${CROSS_COMPILING}" ]; then
+    if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then
+    # xextproto does not support configuration of musl targets so we pretend the target matches the
+    # build triple and enable cross-compilation manually
+    TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
+    EXTRA_CONFIGURE_FLAGS="cross_compiling=yes"
+    fi
+fi
+
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
-    --prefix=/tools/deps
+    --prefix=/tools/deps \
+    ${EXTRA_CONFIGURE_FLAGS}
 
 make -j `nproc`
 make -j `nproc` install DESTDIR=${ROOT}/out

cpython-unix/build-xproto.sh

@@ -15,10 +15,21 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig
 tar -xf xproto-${XPROTO_VERSION}.tar.gz
 pushd xproto-${XPROTO_VERSION}
 
+EXTRA_CONFIGURE_FLAGS=
+if [ -n "${CROSS_COMPILING}" ]; then
+    if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then
+    # xproto does not support configuration of musl targets so we pretend the target matches the
+    # build triple and enable cross-compilation manually
+    TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
+    EXTRA_CONFIGURE_FLAGS="cross_compiling=yes"
+    fi
+fi
+
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
-    --prefix=/tools/deps
+    --prefix=/tools/deps \
+    ${EXTRA_CONFIGURE_FLAGS}
 
 make -j ${NUM_CPUS}
 make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out