From b36a062662c920fba01083de06da5274913d0b9e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 3 Oct 2024 14:30:50 +0100 Subject: [PATCH 01/19] Rename build-common.sh to android-env.sh, and update it from the cpython copy --- .../com/chaquo/python/internal/Common.java | 2 +- product/gradle-plugin/README.md | 2 +- target/android-env.sh | 93 ++++++++++++++++ target/build-common.sh | 104 ------------------ 4 files changed, 95 insertions(+), 106 deletions(-) create mode 100644 target/android-env.sh delete mode 100644 target/build-common.sh diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java index f88e83cabd..582334d4ee 100644 --- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java +++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java @@ -10,7 +10,7 @@ public class Common { // Minimum Android Gradle plugin version public static final String MIN_AGP_VERSION = "7.0.0"; - // This should match api_level in target/build-common.sh. + // This should match api_level in target/android-env.sh. public static final int MIN_SDK_VERSION = 24; public static final int COMPILE_SDK_VERSION = 34; diff --git a/product/gradle-plugin/README.md b/product/gradle-plugin/README.md index 906cb97135..af2996305f 100644 --- a/product/gradle-plugin/README.md +++ b/product/gradle-plugin/README.md @@ -122,7 +122,7 @@ After stable release: * Increment Chaquopy major version if not already done. * Update `MIN_SDK_VERSION` in Common.java. -* Update `api_level` in target/build-common.sh. +* Update `api_level` in target/android-env.sh. * Update default API level in server/pypi/build-wheel.py. * Search repository for other things that should be updated, including workarounds which are now unnecessary: diff --git a/target/android-env.sh b/target/android-env.sh new file mode 100644 index 0000000000..31700e31f0 --- /dev/null +++ b/target/android-env.sh @@ -0,0 +1,93 @@ +# This script must be sourced with the following variables already set: +: ${ANDROID_HOME:?} # Path to Android SDK +: ${HOST:?} # GNU target triplet + +# You may also override the following: +: ${api_level:=24} # Minimum Android API level the build will run on +: ${PREFIX:-} # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=27.1.12297006 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e $ndk ]; then + log "Installing NDK - this may take several minutes" + yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" +fi + +if [ $HOST = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet=$HOST +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ $HOST = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix=$(realpath $PREFIX) + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# Use the same variable name as conda-build +if [ $(uname) = "Darwin" ]; then + export CPU_COUNT=$(sysctl -n hw.ncpu) +else + export CPU_COUNT=$(nproc) +fi diff --git a/target/build-common.sh b/target/build-common.sh deleted file mode 100644 index 85266bdcc8..0000000000 --- a/target/build-common.sh +++ /dev/null @@ -1,104 +0,0 @@ -# This script must be sourced with the following variables already set: -# * ANDROID_HOME: path to Android SDK -# * prefix: path with `include` and `lib` subdirectories to add to CFLAGS and LDFLAGS. -# -# You may also override the following: -: ${abi:=$(basename $prefix)} -: ${api_level:=24} # Should match MIN_SDK_VERSION in Common.java. - -# Print all messages on stderr so they're visible when running within build-wheel. -log() { - echo "$1" >&2 -} - -fail() { - log "$1" - exit 1 -} - -# When moving to a new version of the NDK, carefully review the following: -# -# * The release notes (https://developer.android.com/ndk/downloads/revision_history) -# -# * https://android.googlesource.com/platform/ndk/+/ndk-release-rXX/docs/BuildSystemMaintainers.md, -# where XX is the NDK version. Do a diff against the version you're upgrading from. -# -# * According to https://github.com/kivy/python-for-android/pull/2615, the mzakharo -# build of gfortran is not compatible with NDK version 23, which is the version in -# which they removed the GNU binutils. -ndk_version=22.1.7171670 # See ndkDir in product/runtime/build.gradle. - -ndk=${ANDROID_HOME:?}/ndk/$ndk_version -if ! [ -e $ndk ]; then - log "Installing NDK: this may take several minutes" - yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" -fi - -case $abi in - armeabi-v7a) - host_triplet=arm-linux-androideabi - clang_triplet=armv7a-linux-androideabi - ;; - arm64-v8a) - host_triplet=aarch64-linux-android - ;; - x86) - host_triplet=i686-linux-android - ;; - x86_64) - host_triplet=x86_64-linux-android - ;; - *) - fail "Unknown ABI: '$abi'" - ;; -esac - -# These variables are based on BuildSystemMaintainers.md above, and -# $ndk/build/cmake/android.toolchain.cmake. -toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) -export AR="$toolchain/bin/llvm-ar" -export AS="$toolchain/bin/$host_triplet-as" -export CC="$toolchain/bin/${clang_triplet:-$host_triplet}$api_level-clang" -export CXX="${CC}++" -export LD="$toolchain/bin/ld" -export NM="$toolchain/bin/llvm-nm" -export RANLIB="$toolchain/bin/llvm-ranlib" -export READELF="$toolchain/bin/llvm-readelf" -export STRIP="$toolchain/bin/llvm-strip" - -# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. -for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do - if ! [ -e "$path" ]; then - fail "$path does not exist" - fi -done - -# Use -idirafter so that package-specified -I directories take priority. For example, -# grpcio provides its own BoringSSL headers which must be used rather than our OpenSSL. -export CFLAGS="-idirafter ${prefix:?}/include" -export LDFLAGS="-L${prefix:?}/lib \ --Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libgcc_real.a -Wl,--exclude-libs,libunwind.a \ --Wl,--build-id=sha1 -Wl,--no-rosegment" - -# Many packages get away with omitting this on standard Linux, but Android is stricter. -LDFLAGS+=" -lm" - -case $abi in - armeabi-v7a) - CFLAGS+=" -march=armv7-a -mthumb" - ;; - x86) - # -mstackrealign is unnecessary because it's included in the clang launcher script - # which is pointed to by $CC. - ;; -esac - -export PKG_CONFIG="pkg-config --define-prefix" -export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig" - -# conda-build variable name -if [ $(uname) = "Darwin" ]; then - export CPU_COUNT=$(sysctl -n hw.ncpu) -else - export CPU_COUNT=$(nproc) -fi From 45be801df5108f56288addbc0c9a887a61f1205f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 3 Oct 2024 15:11:21 +0100 Subject: [PATCH 02/19] Support 16 KB pages in runtime, and use NDK version from android-env.sh --- product/runtime/build.gradle | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/product/runtime/build.gradle b/product/runtime/build.gradle index 5e5b0fcff0..10f5a8b7ff 100644 --- a/product/runtime/build.gradle +++ b/product/runtime/build.gradle @@ -271,18 +271,27 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) { args ("-DCHAQUOPY_INCLUDE_JAVA=$javaHome/include;" + "$javaHome/include/$javaIncludeSubdir") } else { - // This must be one of the NDK versions which are pre-installed on the - // GitHub Actions runner. Ideally it would also match the version in - // target/build-common.sh, but the latter is more difficult to change. - def ndkDir = sdkPath("ndk/26.3.11579264") + String ndkDir = null + def androidEnvFile = file("../../target/android-env.sh").absoluteFile + for (line in androidEnvFile.readLines()) { + def match = line =~ /ndk_version=(\S+)/ + if (match) { + ndkDir = sdkPath("ndk/${match.group(1)}") + break + } + } + if (ndkDir == null) { + throw new GradleException("Failed to find NDK version in $androidEnvFile") + } def prefixDir = "$projectDir/../../target/prefix/$abi" args "-DCMAKE_TOOLCHAIN_FILE=$ndkDir/build/cmake/android.toolchain.cmake", - "-DANDROID_ABI=$abi", "-DANDROID_STL=system", + "-DANDROID_ABI=$abi", "-DANDROID_NATIVE_API_LEVEL=$Common.MIN_SDK_VERSION", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", "-DCHAQUOPY_PYTHON_VERSIONS=${pyVersions.join(';')}", "-DCHAQUOPY_INCLUDE_PYTHON=$prefixDir/include", - "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib" + "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib", } args "-DCHAQUOPY_PY_SUFFIX=$pyLibSuffix", projectDir From eab5973fff18e2af3820934b98f5382c6f1bd302 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 3 Oct 2024 20:03:28 +0100 Subject: [PATCH 03/19] Update Python build script to use android-env.sh, and download libraries from cpython-android-source-deps --- .../com/chaquo/python/internal/Common.java | 10 +++--- server/pypi/packages/cryptography/meta.yaml | 3 +- target/README.md | 36 ++++++------------- target/abi-to-host.sh | 17 +++++++++ target/build-all.sh | 30 ---------------- target/for-each-abi.sh | 10 ------ target/package-target.sh | 6 ++-- target/python/build.sh | 33 +++++++++++++---- 8 files changed, 65 insertions(+), 80 deletions(-) create mode 100644 target/abi-to-host.sh delete mode 100755 target/build-all.sh delete mode 100755 target/for-each-abi.sh diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java index 582334d4ee..9629eab3c0 100644 --- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java +++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java @@ -18,11 +18,11 @@ public class Common { public static final Map PYTHON_VERSIONS = new LinkedHashMap<>(); static { // Version, build number - PYTHON_VERSIONS.put("3.8.18", "0"); - PYTHON_VERSIONS.put("3.9.18", "0"); - PYTHON_VERSIONS.put("3.10.13", "0"); - PYTHON_VERSIONS.put("3.11.6", "0"); - PYTHON_VERSIONS.put("3.12.1", "0"); + PYTHON_VERSIONS.put("3.8.20", "0"); + PYTHON_VERSIONS.put("3.9.20", "0"); + PYTHON_VERSIONS.put("3.10.15", "0"); + PYTHON_VERSIONS.put("3.11.10", "0"); + PYTHON_VERSIONS.put("3.12.7", "0"); } public static List PYTHON_VERSIONS_SHORT = new ArrayList<>(); diff --git a/server/pypi/packages/cryptography/meta.yaml b/server/pypi/packages/cryptography/meta.yaml index 70aed87a2d..23167805c7 100644 --- a/server/pypi/packages/cryptography/meta.yaml +++ b/server/pypi/packages/cryptography/meta.yaml @@ -21,7 +21,8 @@ requirements: # loads on startup. # # Instead, we link against OpenSSL 1.1 statically, as follows: - # * Run the OpenSSL 1.1 build command from target/build-all.sh. + # * Download an OpenSSL 1.1 build from + # https://github.com/beeware/cpython-android-source-deps/releases. # * For each combination of Python version and ABI, run build-with-static-openssl.sh # in this directory. # diff --git a/target/README.md b/target/README.md index a4cde39c89..15d1e5f580 100644 --- a/target/README.md +++ b/target/README.md @@ -3,34 +3,23 @@ This directory contains scripts to build Python for Android. -## Supporting libraries - -Before building Python, the correct versions of the supporting libraries must already be -present in the `prefix` subdirectory: - -* Bzip2, libffi and xz use static libraries, so you must build them yourself, using the - commands from build-all.sh. -* SQLite and OpenSSL use dynamic libraries, so you may either build them yourself in the - same way, or get pre-built copies using the download-target.sh and unpackage-target.sh - scripts, as shown in ci.yml. - - -## Python +## Building and testing Update Common.java with the version you want to build, and the build number you want to give it. -Run build-and-package.sh, as shown in build-all.sh. This will create a release in the -`maven` directory in the root of this repository. If the packaging phase fails, e.g. -because the version already exists, then rather than doing the whole build again, you -can re-run package-target.sh directly. +Run `python/build-and-package.sh VERSION`, where `VERSION` is the Python major.minor +version (e.g. `3.13`). This will create a release in the `maven` directory in the root +of this repository. If the packaging phase fails, e.g. because the version already +exists, then rather than doing the whole build again, you can re-run package-target.sh +directly. If this is a new major.minor version, do the "Adding a Python version" checklist below. -Run the PythonVersion integration tests. +Run the integration tests. -Use the demo app to run the Python and Java unit tests on the full set of pre-release -devices (see release/README.md). +Temporarily change the Python version of the demo app, and run the Python and Java unit +tests on the full set of pre-release devices (see release/README.md). To publish the build, follow the "Public release" instructions in release/README.md. Once a version has been published on Maven Central, it cannot be changed, so any fixes @@ -39,16 +28,13 @@ must be released under a different build number. ## Adding a Python version -* First add buildPython support for the same version (see gradle-plugin/README.md). -* Add the version to Common.java. -* Add the version to build-all.sh. +* Add buildPython support for the same version (see gradle-plugin/README.md). * In test_gradle_plugin.py, update the `PYTHON_VERSIONS` assertion. * Update the `MAGIC` lists in test_gradle_plugin.py and pyc.py. * Update android.rst and versions.rst. * Build any packages used by the demo app. -* Temporarily change the Python version of the demo app, and run all tests. -When building the other packages: +When building wheels for other packages: * For each package, in dependency order: * Update to the current stable version, unless it's been updated recently, or updating diff --git a/target/abi-to-host.sh b/target/abi-to-host.sh new file mode 100644 index 0000000000..6d727316fc --- /dev/null +++ b/target/abi-to-host.sh @@ -0,0 +1,17 @@ +case ${abi:?} in + armeabi-v7a) + HOST=arm-linux-androideabi + ;; + arm64-v8a) + HOST=aarch64-linux-android + ;; + x86) + HOST=i686-linux-android + ;; + x86_64) + HOST=x86_64-linux-android + ;; + *) + fail "Unknown ABI: '$abi'" + ;; +esac diff --git a/target/build-all.sh b/target/build-all.sh deleted file mode 100755 index ad3850400d..0000000000 --- a/target/build-all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -eu - -cd $(dirname $(realpath $0)) - -echo "This script needs to be updated to use https://github.com/beeware/cpython-android-source-deps" -exit 1 - -# Build libraries shared by all Python versions. -./for-each-abi.sh bzip2/build.sh 1.0.8 -./for-each-abi.sh libffi/build.sh 3.4.4 -./for-each-abi.sh sqlite/build.sh 2022 3390200 -./for-each-abi.sh xz/build.sh 5.4.5 - -# Build all supported versions of Python, and generate `target` artifacts for Maven. -# -# For a given Python version, we can't change the OpenSSL major version after we've made -# the first release, because that would break binary compatibility with our existing -# builds of the `cryptography` package. Also, multiple OpenSSL versions can't coexist -# within the same include directory, because they use the same header file names. So we -# build each OpenSSL version immediately before all the Python versions that use it. - -./for-each-abi.sh openssl/build.sh 1.1.1s -python/build-and-package.sh 3.8 - -./for-each-abi.sh openssl/build.sh 3.0.5 -python/build-and-package.sh 3.9 -python/build-and-package.sh 3.10 -python/build-and-package.sh 3.11 -python/build-and-package.sh 3.12 diff --git a/target/for-each-abi.sh b/target/for-each-abi.sh deleted file mode 100755 index a05ce184ff..0000000000 --- a/target/for-each-abi.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -eu - -target_dir=$(dirname $(realpath $0)) -script=$(realpath ${1:?}) - -shift -for prefix in $target_dir/prefix/*; do - $script $prefix "$@" -done diff --git a/target/package-target.sh b/target/package-target.sh index c883687d5a..e6c2c2678e 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -71,9 +71,11 @@ mkdir "$tmp_dir" cd "$tmp_dir" for prefix in $prefixes; do - unset abi api_level - . "$this_dir/build-common.sh" + abi=$(basename $prefix) echo "$abi" + . "$this_dir/abi-to-host.sh" + . "$this_dir/android-env.sh" # For STRIP + mkdir "$abi" cd "$abi" diff --git a/target/python/build.sh b/target/python/build.sh index 3fc2ac1275..f6c4901db9 100755 --- a/target/python/build.sh +++ b/target/python/build.sh @@ -1,15 +1,17 @@ #!/bin/bash -set -eu +set -eu -o pipefail recipe_dir=$(dirname $(realpath $0)) -prefix=$(realpath ${1:?}) +PREFIX=$(realpath ${1:?}) version=${2:?} read version_major version_minor < <(echo $version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') version_short=$version_major.$version_minor version_int=$(($version_major * 100 + $version_minor)) +abi=$(basename $PREFIX) cd $recipe_dir -. ../build-common.sh +. ../abi-to-host.sh +. ../android-env.sh version_dir=$recipe_dir/build/$version mkdir -p $version_dir @@ -41,9 +43,26 @@ for name in $patches; do patch -p1 -i $recipe_dir/patches/$name.patch done +# For a given Python version, we can't change the OpenSSL major version after we've +# made the first release, because that would break binary compatibility with our +# existing builds of the `cryptography` package. +libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-0 xz-5.4.6-1" +if [ $version_int -le 38 ]; then + libs+=" openssl-1.1.1w-0" +else + libs+=" openssl-3.0.15-1" +fi + +url_prefix="https://github.com/beeware/cpython-android-source-deps/releases/download" +for name_ver in $libs; do + url="$url_prefix/$name_ver/$name_ver-$HOST.tar.gz" + echo "$url" + curl -Lf "$url" | tar -x -C $PREFIX +done + # Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable. CFLAGS+=" -I$toolchain/sysroot/usr/include" -LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$host_triplet/$api_level" +LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$HOST/$api_level" # The configure script omits -fPIC on Android, because it was unnecessary on older versions of # the NDK (https://bugs.python.org/issue26851). But it's definitely necessary on the current @@ -60,8 +79,8 @@ ac_cv_file__dev_ptc=no EOF export CONFIG_SITE=$(pwd)/config.site -configure_args="--host=$host_triplet --build=$(./config.guess) \ ---enable-shared --without-ensurepip --with-openssl=$prefix" +configure_args="--host=$HOST --build=$(./config.guess) \ +--enable-shared --without-ensurepip --with-openssl=$PREFIX" # This prevents the "getaddrinfo bug" test, which can't be run when cross-compiling. configure_args+=" --enable-ipv6" @@ -73,4 +92,4 @@ fi ./configure $configure_args make -j $CPU_COUNT -make install prefix=$prefix +make install prefix=$PREFIX From e65974a3cf3910e1d914f80891037d9f13bc00af Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 4 Oct 2024 01:24:36 +0100 Subject: [PATCH 04/19] Update patches for Python 3.11 and 3.12 --- target/python/build.sh | 32 +++++-- target/python/patches/bldlibrary.patch | 83 +++++++++++-------- .../patches/python_for_build_deps.patch | 16 ++++ 3 files changed, 86 insertions(+), 45 deletions(-) create mode 100644 target/python/patches/python_for_build_deps.patch diff --git a/target/python/build.sh b/target/python/build.sh index f6c4901db9..fdaffba38c 100755 --- a/target/python/build.sh +++ b/target/python/build.sh @@ -13,6 +13,7 @@ cd $recipe_dir . ../abi-to-host.sh . ../android-env.sh +# Download and unpack Python source code. version_dir=$recipe_dir/build/$version mkdir -p $version_dir cd $version_dir @@ -26,26 +27,31 @@ cd $build_dir tar -xf $version_dir/$src_filename cd $(basename $src_filename .tgz) +# Apply patches. patches="soname" if [ $version_int -le 311 ]; then patches+=" sysroot_paths" fi -if [ $version_int -ge 311 ]; then - # Although this patch applies cleanly to 3.12, it no longer has the intended effect, - # because the stdlib extension modules are now built using autoconf rather than - # distutils. Replace it with the fix we upstreamed to 3.13. - patches+=" python_for_build_deps_REMOVED" +if [ $version_int -eq 311 ]; then + patches+=" python_for_build_deps" fi if [ $version_int -ge 312 ]; then patches+=" bldlibrary grp" fi for name in $patches; do - patch -p1 -i $recipe_dir/patches/$name.patch + patch_file="$recipe_dir/patches/$name.patch" + echo "$patch_file" + patch -p1 -i "$patch_file" done -# For a given Python version, we can't change the OpenSSL major version after we've -# made the first release, because that would break binary compatibility with our -# existing builds of the `cryptography` package. +# Remove any existing installation in the prefix. +rm -rf $PREFIX/{include,lib}/python$version_short +rm -rf $PREFIX/lib/libpython$version_short* + +# Download and unpack libraries needed to compile Python. For a given Python version, we +# can't change the OpenSSL major version after we've made the first release, because +# that would break binary compatibility with our existing builds of the `cryptography` +# package. libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-0 xz-5.4.6-1" if [ $version_int -le 38 ]; then libs+=" openssl-1.1.1w-0" @@ -85,6 +91,14 @@ configure_args="--host=$HOST --build=$(./config.guess) \ # This prevents the "getaddrinfo bug" test, which can't be run when cross-compiling. configure_args+=" --enable-ipv6" +# Some of the patches involve missing Makefile dependencies, which allowed extension +# modules to be built before libpython3.x.so in parallel builds. In case this happens +# again, make sure there's no libpython3.x.a, otherwise the modules may end up silently +# linking with that instead. +if [ $version_int -ge 310 ]; then + configure_args+=" --without-static-libpython" +fi + if [ $version_int -ge 311 ]; then configure_args+=" --with-build-python=yes" fi diff --git a/target/python/patches/bldlibrary.patch b/target/python/patches/bldlibrary.patch index 4dda79fe1c..72b0bac89b 100644 --- a/target/python/patches/bldlibrary.patch +++ b/target/python/patches/bldlibrary.patch @@ -1,44 +1,55 @@ ---- Python-3.12.0-original/configure 2023-11-22 09:33:49 -+++ Python-3.12.0/configure 2023-11-22 10:13:05 -@@ -7476,6 +7476,7 @@ - case $ac_sys_system in - CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' -+ BLDLIBRARY='-L. -lpython$(LDVERSION)' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; - SunOS*) -@@ -24374,7 +24375,7 @@ - # On Android and Cygwin the shared libraries must be linked with libpython. +diff --git a/configure b/configure +index 1c75810d9e8..d883a00d548 100755 +--- a/configure ++++ b/configure +@@ -841,6 +841,7 @@ PY_ENABLE_SHARED + PLATLIBDIR + BINLIBDEST + LIBPYTHON ++MODULE_DEPS_SHARED + EXT_SUFFIX + ALT_SOABI + SOABI +@@ -24402,12 +24403,17 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 + printf "%s\n" "$LDVERSION" >&6; } +-# On Android and Cygwin the shared libraries must be linked with libpython. ++# Configure the flags and dependencies used when compiling shared modules. ++# Do not rename LIBPYTHON - it's accessed via sysconfig by package build ++# systems (e.g. Meson) to decide whether to link extension modules against ++# libpython. ++MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' ++LIBPYTHON='' + ++# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -+ LIBPYTHON="$BLDLIBRARY" - else - LIBPYTHON='' +-else +- LIBPYTHON='' ++ MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" ++ LIBPYTHON="\$(BLDLIBRARY)" fi ---- Python-3.12.0-original/Modules/makesetup 2023-10-02 12:48:14 -+++ Python-3.12.0/Modules/makesetup 2023-11-22 10:11:40 -@@ -86,18 +86,6 @@ - # Newline for sed i and a commands - NL='\ - ' -- --# Setup to link with extra libraries when making shared extensions. --# Currently, only Cygwin needs this baggage. --case `uname -s` in --CYGWIN*) if test $libdir = . -- then -- ExtraLibDir=. -- else -- ExtraLibDir='$(LIBPL)' -- fi -- ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; --esac - # Main loop - for i in ${*-Setup} -@@ -286,7 +274,7 @@ + +diff --git a/Makefile.pre.in b/Makefile.pre.in +index 0e64ccc5c21..c4217424508 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -2797,7 +2797,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h + + # force rebuild when header file or module build flavor (static/shared) is changed + MODULE_DEPS_STATIC=Modules/config.c +-MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) ++MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ + + MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h + MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h +diff --git a/Modules/makesetup b/Modules/makesetup +index f000c9cd673..3231044230e 100755 +--- a/Modules/makesetup ++++ b/Modules/makesetup +@@ -286,7 +286,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" diff --git a/target/python/patches/python_for_build_deps.patch b/target/python/patches/python_for_build_deps.patch new file mode 100644 index 0000000000..820dd58dba --- /dev/null +++ b/target/python/patches/python_for_build_deps.patch @@ -0,0 +1,16 @@ +--- a/Makefile.pre.in 2022-10-24 17:35:39.000000000 +0000 ++++ b/Makefile.pre.in 2022-11-01 18:20:18.472102145 +0000 +@@ -292,7 +292,12 @@ + PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ + # Single-platform builds depend on $(BUILDPYTHON). Cross builds use an + # external "build Python" and have an empty PYTHON_FOR_BUILD_DEPS. +-PYTHON_FOR_BUILD_DEPS=@PYTHON_FOR_BUILD_DEPS@ ++# ++# Chaquopy: Was PYTHON_FOR_BUILD_DEPS from the configure script, which is empty when ++# cross-compiling (https://github.com/python/cpython/pull/93977). But this means that in ++# parallel builds, the sharedmods target can start running before libpython is available ++# (https://github.com/beeware/briefcase-android-gradle-template/pull/55). ++PYTHON_FOR_BUILD_DEPS=$(LDLIBRARY) + + # Single-platform builds use Programs/_freeze_module.c for bootstrapping and + # ./_bootstrap_python Programs/_freeze_module.py for remaining modules From 7f6caa797b7a21eac3eba37a97665d9499ce48df Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 4 Oct 2024 01:32:43 +0100 Subject: [PATCH 05/19] Remove dangerous trailing comma --- product/runtime/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/runtime/build.gradle b/product/runtime/build.gradle index 10f5a8b7ff..48f80ac853 100644 --- a/product/runtime/build.gradle +++ b/product/runtime/build.gradle @@ -291,7 +291,7 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) { "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", "-DCHAQUOPY_PYTHON_VERSIONS=${pyVersions.join(';')}", "-DCHAQUOPY_INCLUDE_PYTHON=$prefixDir/include", - "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib", + "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib" } args "-DCHAQUOPY_PY_SUFFIX=$pyLibSuffix", projectDir From 5cb20f888baee2197a1921d13067d1afe973a307 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 5 Oct 2024 01:59:47 +0100 Subject: [PATCH 06/19] Add support for Python 3.13 (target) --- .../com/chaquo/python/internal/Common.java | 1 + target/README.md | 18 +- target/abi-to-host.sh | 3 +- target/list-versions.py | 29 +-- target/package-target.sh | 44 +++-- target/python/build.sh | 170 +++++++++++------- target/python/patches/3.13_pending.patch | 52 ++++++ 7 files changed, 218 insertions(+), 99 deletions(-) create mode 100644 target/python/patches/3.13_pending.patch diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java index 9629eab3c0..71d13714c8 100644 --- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java +++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java @@ -23,6 +23,7 @@ public class Common { PYTHON_VERSIONS.put("3.10.15", "0"); PYTHON_VERSIONS.put("3.11.10", "0"); PYTHON_VERSIONS.put("3.12.7", "0"); + PYTHON_VERSIONS.put("3.13.0rc3", "0"); } public static List PYTHON_VERSIONS_SHORT = new ArrayList<>(); diff --git a/target/README.md b/target/README.md index 15d1e5f580..bc6bdcb4d5 100644 --- a/target/README.md +++ b/target/README.md @@ -1,6 +1,7 @@ # Chaquopy target -This directory contains scripts to build Python for Android. +This directory contains scripts to build Python for Android. They can be run on Linux or +macOS. ## Building and testing @@ -8,15 +9,17 @@ This directory contains scripts to build Python for Android. Update Common.java with the version you want to build, and the build number you want to give it. -Run `python/build-and-package.sh VERSION`, where `VERSION` is the Python major.minor -version (e.g. `3.13`). This will create a release in the `maven` directory in the root -of this repository. If the packaging phase fails, e.g. because the version already -exists, then rather than doing the whole build again, you can re-run package-target.sh -directly. +Make sure the build machine has `pythonX.Y` on the PATH, where `X.Y` is the Python +major.minor version you want to build (e.g. `3.13`). + +Run `python/build-and-package.sh X.Y`. This will create a release in the `maven` +directory in the root of this repository. If the packaging phase fails, e.g. because the +version already exists, then rather than doing the whole build again, you can re-run +package-target.sh directly. If this is a new major.minor version, do the "Adding a Python version" checklist below. -Run the integration tests. +Run the integration tests, starting with PythonVersion. Temporarily change the Python version of the demo app, and run the Python and Java unit tests on the full set of pre-release devices (see release/README.md). @@ -31,6 +34,7 @@ must be released under a different build number. * Add buildPython support for the same version (see gradle-plugin/README.md). * In test_gradle_plugin.py, update the `PYTHON_VERSIONS` assertion. * Update the `MAGIC` lists in test_gradle_plugin.py and pyc.py. +* Add a directory under integration/data/PythonVersion. * Update android.rst and versions.rst. * Build any packages used by the demo app. diff --git a/target/abi-to-host.sh b/target/abi-to-host.sh index 6d727316fc..6e717aac28 100644 --- a/target/abi-to-host.sh +++ b/target/abi-to-host.sh @@ -12,6 +12,7 @@ case ${abi:?} in HOST=x86_64-linux-android ;; *) - fail "Unknown ABI: '$abi'" + echo "Unknown ABI: '$abi'" + exit 1 ;; esac diff --git a/target/list-versions.py b/target/list-versions.py index 21a8eb5733..04667d65f8 100755 --- a/target/list-versions.py +++ b/target/list-versions.py @@ -11,6 +11,7 @@ mode_group.add_argument("--micro", action="store_true") mode_group.add_argument("--build", action="store_true") args = parser.parse_args() +versions_seen = set() product_dir = abspath(f"{dirname(__file__)}/../product") lines = [] @@ -22,16 +23,24 @@ if match: lines.append(match[1]) break - else: - match = re.search(r'PYTHON_VERSIONS.put\("(\d+)\.(\d+)\.(\d+)", "(\d+)"\)', line) - if match: - major, minor, micro, build = match.groups() - version = f"{major}.{minor}" - if args.micro or args.build: - version += f".{micro}" - if args.build: - version += f"-{build}" - lines.append(version) + + elif "PYTHON_VERSIONS.put" in line: + if not (match := re.search( + r'PYTHON_VERSIONS.put\("(\d+)\.(\d+)\.(\w+)", "(\d+)"\)', line + )): + raise ValueError(f"couldn't parse {line!r}") + major, minor, micro, build = match.groups() + + version = f"{major}.{minor}" + if version in versions_seen: + raise ValueError(f"more than one entry for Python {version}") + versions_seen.add(version) + + if args.micro or args.build: + version += f".{micro}" + if args.build: + version += f"-{build}" + lines.append(version) assert lines print("\n".join(lines)) diff --git a/target/package-target.sh b/target/package-target.sh index e6c2c2678e..2dbb954f96 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -25,9 +25,14 @@ mkdir "$target_dir" # Fail if it already exists: we don't want to overwrite thi target_dir=$(realpath $target_dir) full_ver=$(basename $target_dir) -short_ver=$(echo $full_ver | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') -target_prefix="$target_dir/target-$full_ver" +version=$(echo $full_ver | sed 's/-.*//') +read version_major version_minor version_micro < <( + echo $version | sed -E 's/^([0-9]+)\.([0-9]+)\.([0-9]+).*/\1 \2 \3/' +) +version_short=$version_major.$version_minor +version_int=$(($version_major * 100 + $version_minor)) +target_prefix="$target_dir/target-$full_ver" cat > "$target_prefix.pom" < config.site <<-EOF + # Things that can't be autodetected when cross-compiling. + ac_cv_aligned_required=no # Default of "yes" changes hash function to FNV, which breaks Numba. + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no + EOF + export CONFIG_SITE=$(pwd)/config.site + + configure_args="--host=$HOST --build=$(./config.guess) \ + --enable-shared --without-ensurepip --with-openssl=$PREFIX" + + # This prevents the "getaddrinfo bug" test, which can't be run when cross-compiling. + configure_args+=" --enable-ipv6" + + # Some of the patches involve missing Makefile dependencies, which allowed extension + # modules to be built before libpython3.x.so in parallel builds. In case this happens + # again, make sure there's no libpython3.x.a, otherwise the modules may end up silently + # linking with that instead. + if [ $version_int -ge 310 ]; then + configure_args+=" --without-static-libpython" + fi + + if [ $version_int -ge 311 ]; then + configure_args+=" --with-build-python=yes" + fi + + ./configure $configure_args + + make -j $CPU_COUNT + make install prefix=$PREFIX + +# Python 3.13 and later comes with an official Android build script. else - libs+=" openssl-3.0.15-1" -fi - -url_prefix="https://github.com/beeware/cpython-android-source-deps/releases/download" -for name_ver in $libs; do - url="$url_prefix/$name_ver/$name_ver-$HOST.tar.gz" - echo "$url" - curl -Lf "$url" | tar -x -C $PREFIX -done - -# Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable. -CFLAGS+=" -I$toolchain/sysroot/usr/include" -LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$HOST/$api_level" - -# The configure script omits -fPIC on Android, because it was unnecessary on older versions of -# the NDK (https://bugs.python.org/issue26851). But it's definitely necessary on the current -# version, otherwise we get linker errors like "Parser/myreadline.o: relocation R_386_GOTOFF -# against preemptible symbol PyOS_InputHook cannot be used when making a shared object". -export CCSHARED="-fPIC" - -# Override some tests. -cat > config.site < Date: Sat, 5 Oct 2024 17:59:11 +0100 Subject: [PATCH 07/19] Add support for Python 3.13 (gradle-plugin) --- .../src/main/kotlin/PythonTasks.kt | 5 +- .../src/main/python/chaquopy/pyc.py | 5 +- .../data/PythonVersion/3.10/app/build.gradle | 2 + .../data/PythonVersion/3.11/app/build.gradle | 2 + .../data/PythonVersion/3.12/app/build.gradle | 2 + .../data/PythonVersion/3.13/app/build.gradle | 25 ++++++++ .../data/PythonVersion/3.8/app/build.gradle | 2 + .../data/PythonVersion/3.9/app/build.gradle | 2 + .../test/integration/test_gradle_plugin.py | 42 +++++++++---- product/runtime/requirements-build.txt | 2 +- .../runtime/src/main/python/chaquopy_java.pyx | 63 +++++++++---------- 11 files changed, 102 insertions(+), 50 deletions(-) create mode 100644 product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index 119cb8667c..107df99335 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -224,6 +224,9 @@ internal class TaskBuilder( val customIndexUrl = listOf("--index-url", "-i").any { it in python.pip.options } + val versionFull = pythonVersionInfo(python).key + val versionFullNoPre = + """\d+\.\d+\.\d+""".toRegex().find(versionFull)!!.value execBuildPython { args("-m", "chaquopy.pip_install") @@ -237,7 +240,7 @@ internal class TaskBuilder( args("--extra-index-url", "https://chaquo.com/pypi-13.1") } args("--implementation", Common.PYTHON_IMPLEMENTATION) - args("--python-version", pythonVersionInfo(python).key) + args("--python-version", versionFullNoPre) args("--abi", (Common.PYTHON_IMPLEMENTATION + python.version!!.replace(".", ""))) args("--no-compile") diff --git a/product/gradle-plugin/src/main/python/chaquopy/pyc.py b/product/gradle-plugin/src/main/python/chaquopy/pyc.py index 8f91c9dcab..6faf141fd5 100644 --- a/product/gradle-plugin/src/main/python/chaquopy/pyc.py +++ b/product/gradle-plugin/src/main/python/chaquopy/pyc.py @@ -15,14 +15,15 @@ import warnings -# See the list in importlib/_bootstrap_external.py. +# See the CPython source code in Include/internal/pycore_magic_number.h or +# Lib/importlib/_bootstrap_external.py. MAGIC = { - "3.7": 3394, "3.8": 3413, "3.9": 3425, "3.10": 3439, "3.11": 3495, "3.12": 3531, + "3.13": 3571, } diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle index 71bf30fb59..bb178e3735 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle @@ -15,6 +15,8 @@ android { versionName "0.0.1" python { version "3.10" + pip { install "six" } + pyc { pip true } } ndk { abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle index f2031892b1..aea11d6abb 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle @@ -15,6 +15,8 @@ android { versionName "0.0.1" python { version "3.11" + pip { install "six" } + pyc { pip true } } ndk { abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle index aeac9bd522..8b0fd7573f 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle @@ -15,6 +15,8 @@ android { versionName "0.0.1" python { version "3.12" + pip { install "six" } + pyc { pip true } } ndk { abiFilters "arm64-v8a", "x86_64" diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle new file mode 100644 index 0000000000..5e2ae7744c --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle @@ -0,0 +1,25 @@ +plugins { + id 'com.android.application' + id 'com.chaquo.python' +} + +android { + namespace "com.chaquo.python.test" + compileSdk 31 + + defaultConfig { + applicationId "com.chaquo.python.test" + minSdk 24 + targetSdk 31 + versionCode 1 + versionName "0.0.1" + python { + version "3.13" + pip { install "six" } + pyc { pip true } + } + ndk { + abiFilters "arm64-v8a", "x86_64" + } + } +} diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle index 14a0db0fee..6867e8ee02 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle @@ -15,6 +15,8 @@ android { versionName "0.0.1" python { version "3.8" + pip { install "six" } + pyc { pip true } } ndk { abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle index 26145a1982..8abfd0835f 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle @@ -15,6 +15,8 @@ android { versionName "0.0.1" python { version "3.9" + pip { install "six" } + pyc { pip true } } ndk { abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index 2e330a221d..89f68b3b09 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -55,7 +55,7 @@ def list_versions(mode): for full_version in list_versions("micro").splitlines(): version = full_version.rpartition(".")[0] PYTHON_VERSIONS[version] = full_version -assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12"] +assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] DEFAULT_PYTHON_VERSION_FULL = PYTHON_VERSIONS[DEFAULT_PYTHON_VERSION] NON_DEFAULT_PYTHON_VERSION = "3.10" @@ -480,7 +480,9 @@ def check_version(self, run, version): abis = ["arm64-v8a", "x86_64"] if version in ["3.8", "3.9", "3.10", "3.11"]: abis += ["armeabi-v7a", "x86"] - run.rerun(f"PythonVersion/{version}", python_version=version, abis=abis) + run.rerun( + f"PythonVersion/{version}", python_version=version, abis=abis, + requirements=["six.py"]) if version == DEFAULT_PYTHON_VERSION: self.assertNotInLong(self.WARNING.format(".*"), run.stdout, re=True) @@ -1863,11 +1865,13 @@ def check_assets(self, apk_dir, kwargs): if "stdlib" in pyc: self.check_pyc(stdlib_zip, "argparse.pyc", kwargs) - # Data files packaged with stdlib: see target/package_target.sh. - for grammar_stem in ["Grammar", "PatternGrammar"]: - self.test.assertIn("lib2to3/{}{}.final.0.pickle".format( - grammar_stem, PYTHON_VERSIONS[python_version]), - stdlib_files) + # Data files packaged with lib2to3: see target/package_target.sh. + # This module was removed in Python 3.13. + if python_version_info < (3, 13): + for grammar_stem in ["Grammar", "PatternGrammar"]: + self.test.assertIn("lib2to3/{}{}.final.0.pickle".format( + grammar_stem, PYTHON_VERSIONS[python_version]), + stdlib_files) stdlib_native_expected = { # This is the list from the minimum supported Python version. @@ -1892,6 +1896,12 @@ def check_assets(self, apk_dir, kwargs): if python_version_info >= (3, 12): stdlib_native_expected -= {"_sha256.so", "_typing.so"} stdlib_native_expected |= {"_xxinterpchannels.so", "xxsubtype.so"} + if python_version_info >= (3, 13): + stdlib_native_expected -= { + "audioop.so", "_xxinterpchannels.so", "_multiprocessing.so", + "_xxsubinterpreters.so", "ossaudiodev.so"} + stdlib_native_expected |= { + "_interpreters.so", "_interpchannels.so", "_interpqueues.so"} for abi in abis: stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy")) @@ -1919,14 +1929,15 @@ def check_assets(self, apk_dir, kwargs): build_json["assets"]) def check_pyc(self, zip_file, pyc_filename, kwargs): - # See the list in importlib/_bootstrap_external.py. + # See the CPython source code at Include/internal/pycore_magic_number.h or + # Lib/importlib/_bootstrap_external.py. MAGIC = { - "3.7": 3394, "3.8": 3413, "3.9": 3425, "3.10": 3439, "3.11": 3495, "3.12": 3531, + "3.13": 3571, } with zip_file.open(pyc_filename) as pyc_file: self.test.assertEqual( @@ -1939,10 +1950,17 @@ def check_lib(self, lib_dir, kwargs): self.test.assertCountEqual(abis, os.listdir(lib_dir)) for abi in abis: abi_dir = join(lib_dir, abi) + suffix = ( + "chaquopy" if python_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] + else "python") self.test.assertCountEqual( - ["libchaquopy_java.so", "libcrypto_chaquopy.so", - f"libpython{kwargs['python_version']}.so", "libssl_chaquopy.so", - "libsqlite3_chaquopy.so"], + [ + "libchaquopy_java.so", + f"libcrypto_{suffix}.so", + f"libpython{python_version}.so", + f"libssl_{suffix}.so", + f"libsqlite3_{suffix}.so", + ], os.listdir(abi_dir)) self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi) diff --git a/product/runtime/requirements-build.txt b/product/runtime/requirements-build.txt index a521eeabc1..7b5a120452 100644 --- a/product/runtime/requirements-build.txt +++ b/product/runtime/requirements-build.txt @@ -1 +1 @@ -Cython==0.29.36 +Cython==3.0.11 diff --git a/product/runtime/src/main/python/chaquopy_java.pyx b/product/runtime/src/main/python/chaquopy_java.pyx index d683a664cf..d112d44004 100644 --- a/product/runtime/src/main/python/chaquopy_java.pyx +++ b/product/runtime/src/main/python/chaquopy_java.pyx @@ -30,7 +30,7 @@ cdef extern from "chaquopy_java_extra.h": void PyInit_chaquopy_java() except * -cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved): +cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved) noexcept: return JNI_VERSION_1_6 @@ -38,7 +38,7 @@ cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved): # This runs before Py_Initialize, so it must compile to pure C. cdef public void Java_com_chaquo_python_Python_startNative \ - (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path): + (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path) noexcept: if getenv("CHAQUOPY_PROCESS_TYPE") == NULL: # See jvm.pxi startNativeJava(env, j_platform, j_python_path) else: @@ -54,7 +54,7 @@ cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path throw_simple_exception(env, "GetStringUTFChars failed in startNativeJava") return try: - if not set_path(env, python_path): + if not set_env(env, "PYTHONPATH", python_path): return if not set_env(env, "CHAQUOPY_PROCESS_TYPE", "java"): # See chaquopy.pyx return @@ -86,7 +86,7 @@ cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path # module intialization, so the GIL now exists and we have it. We must release the GIL # before we return to Java so that the methods below can be called from any thread. # (http://bugs.python.org/issue1720250) - PyEval_SaveThread(); + PyEval_SaveThread() # WARNING: This function (specifically PyInit_chaquopy_java) will crash if called @@ -101,11 +101,6 @@ cdef bint init_module(JNIEnv *env) with gil: return False -# This runs before Py_Initialize, so it must compile to pure C. -cdef public bint set_path(JNIEnv *env, const char *python_path): - return set_env(env, "PYTHONPATH", python_path) - - # The POSIX setenv function is not available on MSYS2. # This runs before Py_Initialize, so it must compile to pure C. cdef bint set_env(JNIEnv *env, const char *name, const char *value): @@ -124,7 +119,7 @@ cdef bint set_env(JNIEnv *env, const char *name, const char *value): cdef public jlong Java_com_chaquo_python_Python_getModuleNative \ - (JNIEnv *env, jobject this, jobject j_name) with gil: + (JNIEnv *env, jobject this, jobject j_name) noexcept with gil: try: return p2j_pyobject(env, import_module(j2p_string(env, LocalRef.create(env, j_name)))) except BaseException: @@ -136,7 +131,7 @@ cdef public jlong Java_com_chaquo_python_Python_getModuleNative \ # === com.chaquo.python.PyObject ============================================== cdef public void Java_com_chaquo_python_PyObject_closeNative \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: Py_DECREF(j2p_pyobject(env, this)) # Matches with INCREF in p2j_pyobject. return @@ -146,7 +141,7 @@ cdef public void Java_com_chaquo_python_PyObject_closeNative \ cdef public jlong Java_com_chaquo_python_PyObject_fromJavaNative \ - (JNIEnv *env, jobject klass, jobject o) with gil: + (JNIEnv *env, jobject klass, jobject o) noexcept with gil: try: return p2j_pyobject(env, j2p(env, LocalRef.create(env, o))) except BaseException: @@ -156,7 +151,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_fromJavaNative \ cdef public jobject Java_com_chaquo_python_PyObject_toJava \ - (JNIEnv *env, jobject this, jobject to_klass) with gil: + (JNIEnv *env, jobject this, jobject to_klass) noexcept with gil: try: self = j2p_pyobject(env, this) if not to_klass: @@ -177,7 +172,7 @@ cdef public jobject Java_com_chaquo_python_PyObject_toJava \ # can't simply call a common function, because the main return statement has to be inside a # SavedException try block in case of numeric overflow. cdef public jboolean Java_com_chaquo_python_PyObject_toBoolean \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -190,7 +185,7 @@ cdef public jboolean Java_com_chaquo_python_PyObject_toBoolean \ return 0 cdef public jbyte Java_com_chaquo_python_PyObject_toByte \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -203,7 +198,7 @@ cdef public jbyte Java_com_chaquo_python_PyObject_toByte \ return 0 cdef public jchar Java_com_chaquo_python_PyObject_toChar \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -219,7 +214,7 @@ cdef public jchar Java_com_chaquo_python_PyObject_toChar \ return 0 cdef public jshort Java_com_chaquo_python_PyObject_toShort \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -232,7 +227,7 @@ cdef public jshort Java_com_chaquo_python_PyObject_toShort \ return 0 cdef public jint Java_com_chaquo_python_PyObject_toInt \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -245,7 +240,7 @@ cdef public jint Java_com_chaquo_python_PyObject_toInt \ return 0 cdef public jlong Java_com_chaquo_python_PyObject_toLong \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -258,7 +253,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_toLong \ return 0 cdef public jfloat Java_com_chaquo_python_PyObject_toFloat \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -274,7 +269,7 @@ cdef public jfloat Java_com_chaquo_python_PyObject_toFloat \ return 0 cdef public jdouble Java_com_chaquo_python_PyObject_toDouble \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) try: @@ -287,7 +282,7 @@ cdef public jdouble Java_com_chaquo_python_PyObject_toDouble \ return 0 cdef public jlong Java_com_chaquo_python_PyObject_id \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: return id(j2p_pyobject(env, this)) except BaseException: @@ -297,7 +292,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_id \ cdef public jlong Java_com_chaquo_python_PyObject_typeNative \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: return p2j_pyobject(env, type(j2p_pyobject(env, this))) except BaseException: @@ -307,7 +302,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_typeNative \ cdef public jlong Java_com_chaquo_python_PyObject_callThrowsNative \ - (JNIEnv *env, jobject this, jobject jargs) with gil: + (JNIEnv *env, jobject this, jobject jargs) noexcept with gil: try: return call(env, j2p_pyobject(env, this), jargs) except BaseException: @@ -319,7 +314,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_callThrowsNative \ # It's worth making this a native method in order to avoid the temporary PyObject which would # be created by `get(name).call(...)`. cdef public jlong Java_com_chaquo_python_PyObject_callAttrThrowsNative \ - (JNIEnv *env, jobject this, jobject j_key, jobject jargs) with gil: + (JNIEnv *env, jobject this, jobject j_key, jobject jargs) noexcept with gil: try: attr = getattr(j2p_pyobject(env, this), j2p_string(env, LocalRef.create(env, j_key))) @@ -357,7 +352,7 @@ cdef jlong call(JNIEnv *j_env, obj, jobject jargs) except? 0: # === com.chaquo.python.PyObject (Map) ======================================== cdef public jboolean Java_com_chaquo_python_PyObject_containsKeyNative \ - (JNIEnv *env, jobject this, jobject j_key) with gil: + (JNIEnv *env, jobject this, jobject j_key) noexcept with gil: try: self = j2p_pyobject(env, this) key = j2p_string(env, LocalRef.create(env, j_key)) @@ -369,7 +364,7 @@ cdef public jboolean Java_com_chaquo_python_PyObject_containsKeyNative \ cdef public jlong Java_com_chaquo_python_PyObject_getNative \ - (JNIEnv *env, jobject this, jobject j_key) with gil: + (JNIEnv *env, jobject this, jobject j_key) noexcept with gil: try: self = j2p_pyobject(env, this) key = j2p_string(env, LocalRef.create(env, j_key)) @@ -385,7 +380,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_getNative \ cdef public jlong Java_com_chaquo_python_PyObject_putNative \ - (JNIEnv *env, jobject this, jobject j_key, jobject j_value) with gil: + (JNIEnv *env, jobject this, jobject j_key, jobject j_value) noexcept with gil: try: self = j2p_pyobject(env, this) key = j2p_string(env, LocalRef.create(env, j_key)) @@ -402,7 +397,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_putNative \ cdef public jlong Java_com_chaquo_python_PyObject_removeNative \ - (JNIEnv *env, jobject this, jobject j_key) with gil: + (JNIEnv *env, jobject this, jobject j_key) noexcept with gil: try: self = j2p_pyobject(env, this) key = j2p_string(env, LocalRef.create(env, j_key)) @@ -419,7 +414,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_removeNative \ cdef public jobject Java_com_chaquo_python_PyObject_dir \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: keys = java.jclass("java.util.ArrayList")() for key in dir(j2p_pyobject(env, this)): @@ -433,7 +428,7 @@ cdef public jobject Java_com_chaquo_python_PyObject_dir \ # === com.chaquo.python.PyObject (Object) ===================================== cdef public jboolean Java_com_chaquo_python_PyObject_equals \ - (JNIEnv *env, jobject this, jobject that) with gil: + (JNIEnv *env, jobject this, jobject that) noexcept with gil: try: return j2p_pyobject(env, this) == j2p(env, LocalRef.create(env, that)) except BaseException: @@ -443,11 +438,11 @@ cdef public jboolean Java_com_chaquo_python_PyObject_equals \ cdef public jobject Java_com_chaquo_python_PyObject_toString \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: return to_string(env, this, str) cdef public jobject Java_com_chaquo_python_PyObject_repr \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: return to_string(env, this, repr) cdef jobject to_string(JNIEnv *env, jobject this, func): @@ -461,7 +456,7 @@ cdef jobject to_string(JNIEnv *env, jobject this, func): cdef public jint Java_com_chaquo_python_PyObject_hashCode \ - (JNIEnv *env, jobject this) with gil: + (JNIEnv *env, jobject this) noexcept with gil: try: self = j2p_pyobject(env, this) return ctypes.c_int32(hash(self)).value From 3af241e6c0ac40b47788f7f8ef76c920a8d5665e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 7 Oct 2024 17:42:26 +0100 Subject: [PATCH 08/19] Hide more import system frames in stack traces --- .../runtime/src/main/python/java/import.pxi | 30 ++++++++++++------- .../chaquopy/test/android/test_import.py | 1 - 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/product/runtime/src/main/python/java/import.pxi b/product/runtime/src/main/python/java/import.pxi index 9fac6ef3a8..81655b22c8 100644 --- a/product/runtime/src/main/python/java/import.pxi +++ b/product/runtime/src/main/python/java/import.pxi @@ -65,19 +65,27 @@ def import_override(name, globals={}, locals={}, fromlist=None, level=0): # only does this for the standard import system: see remove_import_frames in Python/import.c.) def clean_exception(e): tb = e.__traceback__ - while in_import_system(tb.tb_frame.f_code.co_filename): + + tb_clean = [] + while tb: + if not in_import_system(tb.tb_frame.f_code.co_filename): + tb_clean.append(tb) tb = tb.tb_next - if tb is None: - # A file with a SyntaxError is not actually in the traceback, because it was never - # compiled successfully. - if isinstance(e, SyntaxError) and not in_import_system(e.filename): - break - else: - # The exception originated within the import system, so return it untouched to - # assist debugging. - return e - return e.with_traceback(tb) + # A file with a SyntaxError is not actually in the traceback, because it was never + # compiled successfully. + if tb_clean or isinstance(e, SyntaxError): + # Construct a traceback from the non-import-system frames only. + for i, tb in enumerate(tb_clean): + tb.tb_next = ( + None if i + 1 == len(tb_clean) + else tb_clean[i + 1] + ) + return e.with_traceback(tb_clean[0] if tb_clean else None) + else: + # All frames came from the import system, so return the exception untouched to + # assist debugging. + return e def in_import_system(filename): filename = filename.replace("\\", "/") # For .pyc files compiled on Windows. diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_import.py b/product/runtime/src/test/python/chaquopy/test/android/test_import.py index 9b104a63a3..d920aa1454 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_import.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_import.py @@ -426,7 +426,6 @@ def test_exception(self): fr'line 1, in \n' fr' from . import other_error # noqa: F401\n' + col_marker + - import_frame + fr' File "{asset_path(APP_ZIP)}/package1/other_error.py", ' fr'line 1, in \n' fr' int\("hello"\)\n' From 97559bd674ad2f0ab9b56da15dad904103edd575 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 8 Oct 2024 19:12:55 +0100 Subject: [PATCH 09/19] Python 3.13 runtime starting up as far as REPL --- .../src/main/kotlin/PythonTasks.kt | 4 ++++ .../src/test/integration/test_gradle_plugin.py | 7 ++++--- product/runtime/docs/sphinx/android.rst | 2 +- .../chaquo/python/android/AndroidPlatform.java | 13 +++++++++---- .../runtime/src/main/python/chaquopy_java.pyx | 13 ++++++++----- .../src/main/python/java/android/__init__.py | 18 +----------------- .../chaquopy/test/android/test_import.py | 5 +++-- .../chaquopy/test/android/test_stdlib.py | 11 ++--------- target/package-target.sh | 15 ++++++++++----- 9 files changed, 42 insertions(+), 46 deletions(-) diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index 107df99335..2f24de088e 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -405,6 +405,10 @@ internal class TaskBuilder( "_ctypes.so", // java.primitive and importer "_datetime.so", // calendar < importer (see test_datetime) "_lzma.so", // zipfile < importer + "_opcode.so", // opcode < dis < inspect < importer. The importer + // dependency could be removed, but inspect is also + // imported via dataclasses < pprint < elftools in + // Python <= 3.13. "_random.so", // random < tempfile < zipimport "_sha2.so", // random < tempfile < zipimport (Python >= 3.12) "_sha512.so", // random < tempfile < zipimport (Python <= 3.11) diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index 89f68b3b09..9a74954565 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -1839,8 +1839,9 @@ def check_assets(self, apk_dir, kwargs): stdlib_bootstrap_expected = { # This is the list from our minimum Python version. For why each of these # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt. - "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", - "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so", + "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_opcode.so", + "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so", + "mmap.so", "zlib.so", } if python_version_info >= (3, 12): stdlib_bootstrap_expected -= {"_sha512.so"} @@ -1879,7 +1880,7 @@ def check_assets(self, apk_dir, kwargs): "_codecs_hk.so", "_codecs_iso2022.so", "_codecs_jp.so", "_codecs_kr.so", "_codecs_tw.so", "_contextvars.so", "_csv.so", "_decimal.so", "_elementtree.so", "_hashlib.so", "_heapq.so", "_json.so", "_lsprof.so", "_md5.so", - "_multibytecodec.so", "_multiprocessing.so", "_opcode.so", "_pickle.so", + "_multibytecodec.so", "_multiprocessing.so", "_pickle.so", "_posixsubprocess.so", "_queue.so", "_sha1.so", "_sha256.so", "_sha3.so", "_socket.so", "_sqlite3.so", "_ssl.so", "_statistics.so", "_xxsubinterpreters.so", "_xxtestfuzz.so", "array.so", diff --git a/product/runtime/docs/sphinx/android.rst b/product/runtime/docs/sphinx/android.rst index 0d3dd758d1..81e494d140 100644 --- a/product/runtime/docs/sphinx/android.rst +++ b/product/runtime/docs/sphinx/android.rst @@ -236,7 +236,7 @@ You can set your app's Python version like this:: } In :doc:`this version of Chaquopy <../versions>`, the default Python version is 3.8. The -other available versions are 3.9, 3.10, 3.11 and 3.12, but these may have fewer +other available versions are 3.9, 3.10, 3.11, 3.12 and 3.13, but these may have fewer :ref:`packages ` available. .. _android-source: diff --git a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java index 835e3f0c6e..8af06a46cf 100644 --- a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java +++ b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java @@ -259,14 +259,19 @@ private String streamToString(InputStream in) throws IOException { } private void loadNativeLibs() throws JSONException { + String pythonVersion = buildJson.getString("python_version"); + String suffix = + Arrays.asList("3.8", "3.9", "3.10", "3.11", "3.12").contains(pythonVersion) + ? "chaquopy" : "python"; + // From API level 18 we no longer need to load libraries in dependency order. But // we should still keep pre-loading the OpenSSL and SQLite libraries, because we // can't guarantee that our lib directory will always be on the LD_LIBRARY_PATH // (#1198). - System.loadLibrary("crypto_chaquopy"); - System.loadLibrary("ssl_chaquopy"); - System.loadLibrary("sqlite3_chaquopy"); - System.loadLibrary("python" + buildJson.getString("python_version")); + System.loadLibrary("crypto_" + suffix); + System.loadLibrary("ssl_" + suffix); + System.loadLibrary("sqlite3_" + suffix); + System.loadLibrary("python" + pythonVersion); System.loadLibrary("chaquopy_java"); } diff --git a/product/runtime/src/main/python/chaquopy_java.pyx b/product/runtime/src/main/python/chaquopy_java.pyx index d112d44004..5c9939ddbc 100644 --- a/product/runtime/src/main/python/chaquopy_java.pyx +++ b/product/runtime/src/main/python/chaquopy_java.pyx @@ -37,8 +37,9 @@ cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved) noexcept: # === com.chaquo.python.Python ================================================ # This runs before Py_Initialize, so it must compile to pure C. -cdef public void Java_com_chaquo_python_Python_startNative \ - (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path) noexcept: +cdef public void Java_com_chaquo_python_Python_startNative( + JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path +) noexcept: if getenv("CHAQUOPY_PROCESS_TYPE") == NULL: # See jvm.pxi startNativeJava(env, j_platform, j_python_path) else: @@ -46,7 +47,9 @@ cdef public void Java_com_chaquo_python_Python_startNative \ # We're running in a Java process, so start the Python VM. -cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path): +cdef void startNativeJava( + JNIEnv *env, jobject j_platform, jobject j_python_path +) noexcept: cdef const char *python_path if j_python_path != NULL: python_path = env[0].GetStringUTFChars(env, j_python_path, NULL) @@ -91,7 +94,7 @@ cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path # WARNING: This function (specifically PyInit_chaquopy_java) will crash if called # more than once. -cdef bint init_module(JNIEnv *env) with gil: +cdef bint init_module(JNIEnv *env) noexcept with gil: try: # See CYTHON_PEP489_MULTI_PHASE_INIT in chaquopy_java_extra.h. PyInit_chaquopy_java() @@ -103,7 +106,7 @@ cdef bint init_module(JNIEnv *env) with gil: # The POSIX setenv function is not available on MSYS2. # This runs before Py_Initialize, so it must compile to pure C. -cdef bint set_env(JNIEnv *env, const char *name, const char *value): +cdef bint set_env(JNIEnv *env, const char *name, const char *value) noexcept: cdef int putenvArgLen = strlen(name) + 1 + strlen(value) + 1 cdef char *putenvArg = malloc(putenvArgLen) if snprintf(putenvArg, putenvArgLen, "%s=%s", name, value) != (putenvArgLen - 1): diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py index c6defeb698..6d00cb67ac 100644 --- a/product/runtime/src/main/python/java/android/__init__.py +++ b/product/runtime/src/main/python/java/android/__init__.py @@ -18,7 +18,7 @@ def initialize(context_local, build_json_object, app_path): # These are ordered roughly from low to high level. for name in [ - "warnings", "sys", "os", "tempfile", "socket", "ssl", "multiprocessing" + "warnings", "sys", "os", "tempfile", "ssl", "multiprocessing" ]: importer.add_import_trigger(name, globals()[f"initialize_{name}"]) @@ -91,22 +91,6 @@ def initialize_tempfile(): os.environ["TMPDIR"] = tmpdir -def initialize_socket(): - import socket - - # Some functions aren't available until API level 24, so Python omits them from the - # module. Instead, make them throw OSError as documented. - def unavailable(*args, **kwargs): - raise OSError("this function is not available in this build of Python") - - for name in ["if_nameindex", "if_nametoindex", "if_indextoname"]: - if hasattr(socket, name): - raise Exception( - f"socket.{name} now exists: check if its workaround can be removed" - ) - setattr(socket, name, unavailable) - - def initialize_ssl(): # OpenSSL may be able to find the system CA store on some devices, but for consistency # we disable this and use our own bundled file. diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_import.py b/product/runtime/src/test/python/chaquopy/test/android/test_import.py index d920aa1454..7351bb1abf 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_import.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_import.py @@ -63,8 +63,9 @@ def test_bootstrap(self): stdlib_bootstrap_expected = { # This is the list from our minimum Python version. For why each of these # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt. - "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", - "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so", + "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_opcode.so", + "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so", + "mmap.so", "zlib.so", } if sys.version_info >= (3, 12): stdlib_bootstrap_expected -= {"_sha512.so"} diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py index 155f72f30f..8de9cdc4cf 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py @@ -2,6 +2,7 @@ from os.path import dirname, exists, join, realpath import subprocess import sys +from unittest import skipIf from warnings import catch_warnings, filterwarnings from android.os import Build @@ -61,6 +62,7 @@ def test_json(self): self.assertTrue(encoder.c_make_encoder) self.assertTrue(scanner.c_make_scanner) + @skipIf(sys.version_info >= (3, 13), "lib2to3 was removed in Python 3.13") def test_lib2to3(self): with catch_warnings(): for category in [DeprecationWarning, PendingDeprecationWarning]: @@ -166,15 +168,6 @@ def test_signal(self): self.assertIsInstance(vs[0], enum.IntEnum) self.assertEqual("", repr(vs[0])) - def test_socket(self): - import socket - for name in ["if_nameindex", "if_nametoindex", "if_indextoname"]: - for args in [[], ["whatever"]]: - with self.assertRaisesRegex( - OSError, "this function is not available in this build of Python" - ): - getattr(socket, name)(*args) - def test_sqlite(self): import sqlite3 conn = sqlite3.connect(":memory:") diff --git a/target/package-target.sh b/target/package-target.sh index 2dbb954f96..8e782d9a63 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -75,14 +75,18 @@ rm -rf "$tmp_dir" mkdir "$tmp_dir" cd "$tmp_dir" +stdlib_dir="$tmp_dir/stdlib" +mkdir "$stdlib_dir" + for prefix in $prefixes; do abi=$(basename $prefix) echo "$abi" . "$this_dir/abi-to-host.sh" . "$this_dir/android-env.sh" # For STRIP - mkdir "$abi" - cd "$abi" + abi_dir="$tmp_dir/$abi" + mkdir "$abi_dir" + cd "$abi_dir" mkdir include cp -a "$prefix/include/"{python$version_short,openssl,sqlite*} include @@ -114,12 +118,13 @@ for prefix in $prefixes; do abi_zip="$target_prefix-$abi.zip" rm -f "$abi_zip" zip -q -r "$abi_zip" . - cd .. + + # Merge all copies of the stdlib, including ABI-specific files like sysconfigdata. + cp -a $prefix/lib/python$version_short/* "$stdlib_dir" done echo "stdlib" -cp -a $prefix/lib/python$version_short stdlib -cd stdlib +cd "$stdlib_dir" rm -r lib-dynload site-packages # Remove things which depend on missing native modules. From cdd37dbb520a609bbcffcf0daf0d9a36c4b74e6b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 8 Oct 2024 21:25:26 +0100 Subject: [PATCH 10/19] Backport stdout/stderr logcat redirection from CPython main branch --- .../utils/python/chaquopy/utils/console.py | 6 +- .../src/main/python/java/android/__init__.py | 12 +- .../src/main/python/java/android/stream.py | 201 ++++++++--- .../src/main/python/java/conversion.pxi | 6 +- .../chaquopy/test/android/test_stream.py | 312 ++++++++++++++---- .../python/chaquopy/test/test_conversion.py | 11 +- 6 files changed, 437 insertions(+), 111 deletions(-) diff --git a/demo/app/src/utils/python/chaquopy/utils/console.py b/demo/app/src/utils/python/chaquopy/utils/console.py index ec84908ebb..b6931bd519 100644 --- a/demo/app/src/utils/python/chaquopy/utils/console.py +++ b/demo/app/src/utils/python/chaquopy/utils/console.py @@ -77,7 +77,7 @@ def __repr__(self): def __getattribute__(self, name): # Forward all attributes that have useful implementations. if name in [ - "close", "closed", "flush", "writable", # IOBase + "close", "closed", "fileno", "flush", "writable", # IOBase "encoding", "errors", "newlines", "buffer", "detach", # TextIOBase "line_buffering", "write_through", "reconfigure", # TextIOWrapper ]: @@ -90,5 +90,9 @@ def write(self, s): # exception, the app crashes in the same way whether it's using # ConsoleOutputStream or not. result = self.stream.write(s) + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) self.method(s) return result diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py index 6d00cb67ac..9bf9161e45 100644 --- a/product/runtime/src/main/python/java/android/__init__.py +++ b/product/runtime/src/main/python/java/android/__init__.py @@ -4,7 +4,7 @@ import traceback from types import ModuleType import warnings -from . import stream, importer +from . import importer from org.json import JSONArray, JSONObject @@ -13,7 +13,15 @@ def initialize(context_local, build_json_object, app_path): global context context = context_local - stream.initialize() + # Redirect stdout and stderr to logcat - this was upstreamed in Python 3.13. + if sys.version_info < (3, 13): + from ctypes import CDLL, c_char_p, c_int + from . import stream + + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + stream.init_streams(android_log_write, stdout_prio=4, stderr_prio=5) + importer.initialize(context, convert_json_object(build_json_object), app_path) # These are ordered roughly from low to high level. diff --git a/product/runtime/src/main/python/java/android/stream.py b/product/runtime/src/main/python/java/android/stream.py index 283e21b328..372f0fb820 100644 --- a/product/runtime/src/main/python/java/android/stream.py +++ b/product/runtime/src/main/python/java/android/stream.py @@ -1,62 +1,111 @@ -from android.util import Log +# This file is based on Lib/_android_support.py on the CPython main branch. + import io import sys - - -# The maximum length of a log message in bytes, including the level marker and tag, is defined -# as LOGGER_ENTRY_MAX_PAYLOAD in platform/system/logging/liblog/include/log/log.h. As of API -# level 30, messages longer than this will be be truncated by logcat. This limit has already -# been reduced at least once in the history of Android (from 4076 to 4068 between API level 23 -# and 26), so leave some headroom. -# -# This should match the native stdio buffer size in android_platform.c. -MAX_LINE_LEN_BYTES = 4000 - -# UTF-8 uses a maximum of 4 bytes per character. However, if the actual number of bytes -# per character is smaller than that, then TextIOWrapper may join multiple consecutive -# writes before passing them to the binary stream. -MAX_LINE_LEN_CHARS = MAX_LINE_LEN_BYTES // 4 - - -def initialize(): - # Log levels are consistent with those used by Java. - sys.stdout = TextLogStream(Log.INFO, "python.stdout") - sys.stderr = TextLogStream(Log.WARN, "python.stderr") +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + + sys.stdout = TextLogStream( + stdout_prio, "python.stdout", sys.stdout.fileno(), + errors="backslashreplace") + sys.stderr = TextLogStream( + stderr_prio, "python.stderr", sys.stderr.fileno(), + errors="backslashreplace") class TextLogStream(io.TextIOWrapper): - def __init__(self, level, tag): - super().__init__(BinaryLogStream(self, level, tag), - encoding="UTF-8", errors="backslashreplace", - line_buffering=True) - self._CHUNK_SIZE = MAX_LINE_LEN_BYTES + def __init__(self, prio, tag, fileno=None, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 def __repr__(self): return f"" def write(self, s): if not isinstance(s, str): - # Same wording as TextIOWrapper.write. - raise TypeError(f"write() argument must be str, not {type(s).__name__}") - - # To avoid combining multiple lines into a single log message, we split the string - # into separate lines before sending it to the superclass. Note that - # "".splitlines() == [], so nothing will be logged in that case. - for line, line_keepends in zip(s.splitlines(), s.splitlines(keepends=True)): - # Simplify the later stages by translating all newlines into "\n". - if line != line_keepends: - line += "\n" - while line: - super().write(line[:MAX_LINE_LEN_CHARS]) - line = line[MAX_LINE_LEN_CHARS:] + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + return len(s) + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + class BinaryLogStream(io.RawIOBase): - def __init__(self, text_stream, level, tag): - self.text_stream = text_stream - self.level = level + def __init__(self, prio, tag, fileno=None): + self.prio = prio self.tag = tag + self._fileno = fileno def __repr__(self): return f"" @@ -65,11 +114,67 @@ def writable(self): return True def write(self, b): - # This form of `str` throws a TypeError on any non-bytes-like object, as opposed - # to the AttributeError we would probably get from trying to call `encode`. - s = str(b, self.text_stream.encoding, self.text_stream.errors) + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None # Writing an empty string to the stream should have no effect. - if s: - Log.println(self.level, self.tag, s) + if b: + logcat.write(self.prio, self.tag, b) return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag.encode("UTF-8"), message) diff --git a/product/runtime/src/main/python/java/conversion.pxi b/product/runtime/src/main/python/java/conversion.pxi index ff051a9d42..d2216711f8 100644 --- a/product/runtime/src/main/python/java/conversion.pxi +++ b/product/runtime/src/main/python/java/conversion.pxi @@ -303,7 +303,11 @@ cpdef check_range_char(value): cdef JNIRef p2j_string(JNIEnv *j_env, unicode s): - utf16 = s.encode(JCHAR_ENCODING) + # Python strings can contain invalid surrogates, but Java strings cannot. + # "backslashreplace" would retain some information about the invalid character, but + # we don't know what context this string will be used in, so changing its length + # could cause much worse confusion. + utf16 = s.encode(JCHAR_ENCODING, errors="replace") return LocalRef.adopt(j_env, j_env[0].NewString( j_env, utf16, len(utf16)//2)) # 2 bytes/char for UTF-16. diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py index 40bdf8b4eb..5c3a944fa1 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py @@ -1,27 +1,39 @@ -from contextlib import contextmanager +# This file is based on Lib/test/test_android.py on the CPython main branch. + import ctypes.util +import io import queue import re import subprocess import sys +import unittest +from array import array +from contextlib import ExitStack, contextmanager from threading import Thread from time import time +from unittest.mock import patch + +from java.android.stream import TextLogStream +from ..test_utils import API_LEVEL as api_level, FilterWarningsCase -from android.util import Log -from ..test_utils import API_LEVEL, FilterWarningsCase +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] +# Was `from test.support import LOOPBACK_TIMEOUT`, but Chaquopy doesn't include the +# stdlib `test` module. +LOOPBACK_TIMEOUT = 10.0 redirected_native = False class TestAndroidOutput(FilterWarningsCase): - maxDiff = None def setUp(self): self.logcat_process = subprocess.Popen( - ["logcat", "-v", "tag"], stdout=subprocess.PIPE, errors="backslashreplace" + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" ) self.logcat_queue = queue.Queue() @@ -29,11 +41,19 @@ def logcat_thread(): for line in self.logcat_process.stdout: self.logcat_queue.put(line.rstrip("\n")) self.logcat_process.stdout.close() - Thread(target=logcat_thread).start() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() - tag, start_marker = "python.test", f"{self.id()} {time()}" - Log.i(tag, start_marker) - self.assert_log("I", tag, start_marker, skip=True, timeout=5) + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) def assert_logs(self, level, tag, expected, **kwargs): for line in expected: @@ -57,7 +77,8 @@ def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): def tearDown(self): self.logcat_process.terminate() - self.logcat_process.wait(0.1) + self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) @contextmanager def unbuffered(self, stream): @@ -67,9 +88,32 @@ def unbuffered(self, stream): finally: stream.reconfigure(write_through=False) + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" + ), + ) + ) + return stack + def test_str(self): - for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name) original_stream = getattr(sys, f"__{stream_name}__") self.assertIsNot(stream, original_stream) @@ -79,15 +123,21 @@ def test_str(self): tag = f"python.{stream_name}" self.assertIn(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) self.assertEqual("UTF-8", stream.encoding) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. self.assertEqual("backslashreplace", stream.errors) - self.assertTrue(stream.line_buffering) - self.assertFalse(stream.write_through) - def write(s, lines=None): - self.assertEqual(len(s), stream.write(s)) + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) if lines is None: lines = [s] self.assert_logs(level, tag, lines) @@ -109,14 +159,20 @@ def write(s, lines=None): # Non-BMP emoji write("\U0001f600") + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + # Null characters are logged using "modified UTF-8". write("\u0000", [r"\xc0\x80"]) write("a\u0000", [r"a\xc0\x80"]) write("\u0000b", [r"\xc0\x80b"]) write("a\u0000b", [r"a\xc0\x80b"]) - # Multi-line messages. Avoid identical consecutive lines, as they may - # activate "chatty" filtering and break the tests. + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. write("\nx", [""]) write("\na\n", ["x", "a"]) write("\n", [""]) @@ -127,6 +183,13 @@ def write(s, lines=None): write("f\n\ng", ["exxf", ""]) write("\n", ["g"]) + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. with self.unbuffered(stream): write("\nx", ["", "x"]) write("\na\n", ["", "a"]) @@ -143,10 +206,34 @@ def write(s, lines=None): write("hello\r\nworld\r\n", ["hello", "world"]) write("\r\n", [""]) + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. for obj in [b"", b"hello", None, 42]: with self.subTest(obj=obj): - with self.assertRaisesRegex(TypeError, fr"write\(\) argument must be " - fr"str, not {type(obj).__name__}"): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): stream.write(obj) # Manual flushing is supported. @@ -158,38 +245,43 @@ def write(s, lines=None): stream.flush() self.assert_log(level, tag, "helloworld") - # Long lines are split into blocks of 1000 *characters*, but TextIOWrapper - # should then join them back together as much as possible without - # exceeding 4000 UTF-8 *bytes*. + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in java/android/stream.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). # # ASCII (1 byte per character) - write(("foobar" * 700) + "\n", - [("foobar" * 666) + "foob", # 4000 bytes - "ar" + ("foobar" * 33)]) # 200 bytes + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out # "Full-width" digits 0-9 (3 bytes per character) s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" - write((s * 150) + "\n", - [s * 100, # 3000 bytes - s * 50]) # 1500 bytes + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out s = "0123456789" - write(s * 200, []) - write(s * 150, []) - write(s * 51, [s * 350]) # 3500 bytes - write("\n", [s * 51]) # 510 bytes + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out def test_bytes(self): - for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name).buffer tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) - - def write(b, lines=None): - self.assertEqual(len(b), stream.write(b)) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) if lines is None: lines = [b.decode()] self.assert_logs(level, tag, lines) @@ -210,7 +302,7 @@ def write(b, lines=None): # Non-BMP emoji write(b"\xf0\x9f\x98\x80") - # Null characters are logged using "modified UTF-8". + # Null bytes are logged using "modified UTF-8". write(b"\x00", [r"\xc0\x80"]) write(b"a\x00", [r"a\xc0\x80"]) write(b"\x00b", [r"\xc0\x80b"]) @@ -222,15 +314,17 @@ def write(b, lines=None): write(b"\xffb", [r"\xffb"]) write(b"a\xffb", [r"a\xffb"]) - # Log entries containing newlines are shown differently by `logcat -v - # tag`, `logcat -v long`, and Android Studio. We currently use `logcat -v - # tag`, which shows each line as if it was a separate log entry, but - # strips a single trailing newline. + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. # - # On newer versions of Android, all three of the above tools (or maybe - # Logcat itself) will also strip any number of leading newlines. - write(b"\nx", ["", "x"] if API_LEVEL < 30 else ["x"]) - write(b"\na\n", ["", "a"] if API_LEVEL < 30 else ["a"]) + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) write(b"\n", [""]) write(b"b\n", ["b"]) write(b"c\n\n", ["c", ""]) @@ -244,14 +338,41 @@ def write(b, lines=None): write(b"hello\r\nworld\r\n", ["hello", "world"]) write(b"\r\n", [""]) + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. for obj in ["", "hello", None, 42]: with self.subTest(obj=obj): - if isinstance(obj, str): - message = r"decoding str is not supported" - else: - message = (fr"decoding to str: need a bytes-like object, " - fr"{type(obj).__name__} found") - with self.assertRaisesRegex(TypeError, message): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): stream.write(obj) def test_native(self): @@ -303,6 +424,83 @@ def write(b, lines=None): write(b"Hello\nworld\n", ["Hello", "world"]) +class TestAndroidRateLimit(unittest.TestCase): + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + + # See java/android/stream.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with patch("java.android.stream.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), \ + patch("java.android.stream.BUCKET_SIZE", BUCKET_KB * 1024), \ + patch("java.android.stream.sleep", mock_sleep), \ + patch("java.android.stream.time", mock_time): + # Make sure the token bucket is full. + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = mock_time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (mock_time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + class TestAndroidInput(FilterWarningsCase): def test_str(self): diff --git a/product/runtime/src/test/python/chaquopy/test/test_conversion.py b/product/runtime/src/test/python/chaquopy/test/test_conversion.py index cf0a26e556..1e8dc1687b 100644 --- a/product/runtime/src/test/python/chaquopy/test/test_conversion.py +++ b/product/runtime/src/test/python/chaquopy/test/test_conversion.py @@ -107,7 +107,7 @@ def verify_float(self, obj, name, exponent_bits, wrapper=None, allow_bool=False, self.verify_value(obj, name, float("nan"), # NaN is unequal to everything including itself. wrapper=wrapper, - verify=lambda expected, actual: self.assertTrue(isnan(actual))) + verify=lambda _, actual: self.assertTrue(isnan(actual))) # Wrapper type and bounds checks are tested in test_signatures. self.verify_value(obj, name, True, context=self.conv_error_unless(allow_bool)) @@ -166,16 +166,23 @@ def verify_string(self, obj, name): "\U00012345"]: # Non-BMP character self.verify_value(obj, name, val) + # Invalid surrogates should be replaced by a question mark. + self.verify_value( + obj, name, "a\ud800b", + verify=lambda _, actual: self.assertEqual("a?b", actual) + ) + # Byte strings cannot be implicitly converted to Java Strings. However, if the target # type is Object, we will fall back on the default conversion of a Python iterable to # Object[]. - context = verify = None if name == "Object": + context = None def verify(expected, actual): self.assertEqual(expected, actual) self.assertIsInstance(actual, jarray("Ljava/lang/Object;")) else: context = self.conv_error + verify = None for val in [b"", b"h", b"hello"]: self.verify_value(obj, name, val, context=context, verify=verify) From ea7d1a53287784419cbb4c1f88b6efceb4acba23 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 9 Oct 2024 16:28:58 +0000 Subject: [PATCH 11/19] Update build-wheel to use android-env and handle Python RC version numbers --- product/gradle-plugin/README.md | 4 ++- server/pypi/build-wheel.py | 60 ++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/product/gradle-plugin/README.md b/product/gradle-plugin/README.md index af2996305f..4777f489fb 100644 --- a/product/gradle-plugin/README.md +++ b/product/gradle-plugin/README.md @@ -123,7 +123,9 @@ After stable release: * Increment Chaquopy major version if not already done. * Update `MIN_SDK_VERSION` in Common.java. * Update `api_level` in target/android-env.sh. -* Update default API level in server/pypi/build-wheel.py. +* In server/pypi/build-wheel.py: + * Update default API level. + * Update `STANDARD_LIBS` with any libraries added in the new level. * Search repository for other things that should be updated, including workarounds which are now unnecessary: * Useful regex: `api.?level|android.?ver|android \d|min.?sdk|SDK_INT` diff --git a/server/pypi/build-wheel.py b/server/pypi/build-wheel.py index fbb6c6ce50..95e38b9ad8 100755 --- a/server/pypi/build-wheel.py +++ b/server/pypi/build-wheel.py @@ -38,10 +38,11 @@ # Libraries are grouped by minimum API level and listed under their SONAMEs. STANDARD_LIBS = [ # Android native APIs (https://developer.android.com/ndk/guides/stable_apis) - (16, ["libandroid.so", "libc.so", "libdl.so", "libEGL.so", "libGLESv1_CM.so", "libGLESv2.so", - "libjnigraphics.so", "liblog.so", "libm.so", "libOpenMAXAL.so", "libOpenSLES.so", - "libz.so"]), + (16, ["libandroid.so", "libc.so", "libdl.so", "libEGL.so", "libGLESv1_CM.so", + "libGLESv2.so", "libjnigraphics.so", "liblog.so", "libm.so", + "libOpenMAXAL.so", "libOpenSLES.so", "libz.so"]), (21, ["libmediandk.so"]), + (24, ["libcamera2ndk.so", "libvulkan.so"]), # Chaquopy-provided libraries (0, ["libcrypto_chaquopy.so", "libsqlite3_chaquopy.so", "libssl_chaquopy.so"]), @@ -233,10 +234,25 @@ def find_target(self): raise ERROR target_dir = abspath(f"{PYPI_DIR}/../../maven/com/chaquo/python/target") - versions = [ver for ver in os.listdir(target_dir) if ver.startswith(self.python)] + versions = [ + ver for ver in os.listdir(target_dir) + if ver.startswith(f"{self.python}.")] if not versions: raise CommandError(f"Can't find Python {self.python} in {target_dir}") - max_ver = max(versions, key=lambda ver: [int(x) for x in re.split(r"[.-]", ver)]) + + def version_key(ver): + # Use the same "releaselevel" notation as sys.version_info so it sorts in + # the correct order. + groups = list( + re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:([a-z]+)(\d+))?-(\d+)", ver).groups()) + groups[3] = { + "a": "alpha", "b": "beta", "rc": "candidate", None: "final" + }[groups[3]] + if groups[4] is None: + groups[4] = 0 + return groups + + max_ver = max(versions, key=version_key) target_version_dir = f"{target_dir}/{max_ver}" zips = glob(f"{target_version_dir}/target-*-{self.abi}.zip") @@ -459,9 +475,11 @@ def create_host_env(self): # doesn't support it, and the links wouldn't survive on Windows anyway. So our library # wheels include external shared libraries only under their SONAMEs, and we need to # create links from the other names so the compiler can find them. - SONAME_PATTERNS = [(r"^(lib.*)\.so\..*$", r"\1.so"), - (r"^(lib.*?)\d+\.so$", r"\1.so"), # e.g. libpng - (r"^(lib.*)_chaquopy\.so$", r"\1.so")] # e.g. libjpeg + SONAME_PATTERNS = [ + (r"^(lib.*)\.so\..*$", r"\1.so"), + (r"^(lib.*?)[\d.]+\.so$", r"\1.so"), # e.g. libpng + (r"^(lib.*)_(chaquopy|python)\.so$", r"\1.so"), # e.g. libssl, libjpeg + ] reqs_lib_dir = f"{self.host_env}/chaquopy/lib" for filename in os.listdir(reqs_lib_dir): for pattern, repl in SONAME_PATTERNS: @@ -502,9 +520,15 @@ def build_with_pep517(self): raise CommandError(e) def get_common_env_vars(self, env): + # HOST is set by conda-forge's compiler activation scripts, e.g. + # https://github.com/conda-forge/clang-compiler-activation-feedstock/blob/main/recipe/activate-clang.sh + tool_prefix = ABIS[self.abi].tool_prefix build_common_output = run( - f"abi={self.abi}; api_level={self.api_level}; prefix={self.host_env}/chaquopy; " - f". {PYPI_DIR}/../../target/build-common.sh; export", + f"export HOST={tool_prefix}; " + f"api_level={self.api_level}; " + f"PREFIX={self.host_env}/chaquopy; " + f". {PYPI_DIR}/../../target/android-env.sh; " + f"export", shell=True, executable="bash", text=True, stdout=subprocess.PIPE ).stdout for line in build_common_output.splitlines(): @@ -517,14 +541,9 @@ def get_common_env_vars(self, env): if os.environ.get(key) != value: env[key] = value if not env: - raise CommandError("Found no variables in build-common.sh output:\n" + raise CommandError("Found no variables in android-env.sh output:\n" + build_common_output) - # This flag often catches errors in .so files which would otherwise be delayed - # until runtime. (Some of the more complex build.sh scripts need to remove this, or - # use it more selectively.) - env["LDFLAGS"] += " -Wl,--no-undefined" - # Set all other variables used by distutils to prevent the host Python values (if # any) from taking effect. env["CPPFLAGS"] = "" @@ -533,7 +552,6 @@ def get_common_env_vars(self, env): compiler_vars = ["CC", "CXX", "LD"] if "fortran" in self.non_python_build_reqs: - tool_prefix = ABIS[self.abi].tool_prefix toolchain = self.abi if self.abi in ["x86", "x86_64"] else tool_prefix gfortran = f"{PYPI_DIR}/fortran/{toolchain}-4.9/bin/{tool_prefix}-gfortran" if not exists(gfortran): @@ -602,13 +620,9 @@ def env_vars(self): # TODO: make everything use HOST instead, and remove this. "CHAQUOPY_ABI": self.abi, - # Set by conda-forge's compiler activation scripts, e.g. - # https://github.com/conda-forge/clang-compiler-activation-feedstock/blob/main/recipe/activate-clang.sh - "HOST": ABIS[self.abi].tool_prefix, - # conda-build variable names defined at # https://docs.conda.io/projects/conda-build/en/latest/user-guide/environment-variables.html - # CPU_COUNT is now in build-common.sh, so the target scripts can use it. + # CPU_COUNT is now in android-env.sh, so the target scripts can use it. "PKG_BUILDNUM": self.build_num, "PKG_NAME": self.package, "PKG_VERSION": self.version, @@ -627,7 +641,7 @@ def env_vars(self): if self.verbose: log("Environment set as follows:\n" + "\n".join(f"export {key}={shlex.quote(value)}" - for key, value in env.items())) + for key, value in sorted(env.items()))) original_env = {key: os.environ.get(key) for key in env} os.environ.update(env) From 070d838ae363a129c652a300b0c01a348e34d14a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 10 Oct 2024 12:12:35 +0100 Subject: [PATCH 12/19] Set sys.stdout.errors to backslashreplace on Python 3.13 --- .../runtime/src/main/python/java/android/__init__.py | 5 +++++ .../runtime/src/main/python/java/android/stream.py | 11 +++++++---- .../test/python/chaquopy/test/android/test_stream.py | 5 +---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py index 9bf9161e45..d6a42a1a18 100644 --- a/product/runtime/src/main/python/java/android/__init__.py +++ b/product/runtime/src/main/python/java/android/__init__.py @@ -21,6 +21,11 @@ def initialize(context_local, build_json_object, app_path): android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") android_log_write.argtypes = (c_int, c_char_p, c_char_p) stream.init_streams(android_log_write, stdout_prio=4, stderr_prio=5) + elif sys.stdout.errors == "backslashreplace": + # This fix should be upstreamed in Python 3.13.1. + raise Exception("see if sys.stdout.errors workaround can be removed") + else: + sys.stdout.reconfigure(errors="backslashreplace") importer.initialize(context, convert_json_object(build_json_object), app_path) diff --git a/product/runtime/src/main/python/java/android/stream.py b/product/runtime/src/main/python/java/android/stream.py index 372f0fb820..40df0d93fd 100644 --- a/product/runtime/src/main/python/java/android/stream.py +++ b/product/runtime/src/main/python/java/android/stream.py @@ -33,16 +33,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): logcat = Logcat(android_log_write) sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno(), - errors="backslashreplace") + stdout_prio, "python.stdout", sys.stdout.fileno()) sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno(), - errors="backslashreplace") + stderr_prio, "python.stderr", sys.stderr.fileno()) class TextLogStream(io.TextIOWrapper): def __init__(self, prio, tag, fileno=None, **kwargs): + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) self._lock = RLock() self._pending_bytes = [] diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py index 5c3a944fa1..968e1e610d 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py @@ -127,13 +127,10 @@ def test_str(self): self.assertIs(stream.readable(), False) self.assertEqual(stream.fileno(), fileno) self.assertEqual("UTF-8", stream.encoding) + self.assertEqual("backslashreplace", stream.errors) self.assertIs(stream.line_buffering, True) self.assertIs(stream.write_through, False) - # stderr is backslashreplace by default; stdout is configured - # that way by libregrtest.main. - self.assertEqual("backslashreplace", stream.errors) - def write(s, lines=None, *, write_len=None): if write_len is None: write_len = len(s) From 81474c260279f83b9b0158907948feb55c510dec Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 10 Oct 2024 14:22:11 +0100 Subject: [PATCH 13/19] Python 3.13 passing all tests except those involving OpenSSL --- .../src/main/kotlin/PythonTasks.kt | 25 +++++++++++++------ .../test/integration/test_gradle_plugin.py | 16 +++++++----- .../src/main/python/java/android/__init__.py | 11 ++++---- .../src/main/python/java/android/importer.py | 9 ++++--- .../chaquopy/test/android/test_import.py | 16 +++++++----- .../chaquopy/test/android/test_stdlib.py | 19 ++++++++++---- .../chaquopy/test/android/test_stream.py | 14 +++++++---- server/pypi/pkgtest/app/build.gradle | 8 +++--- 8 files changed, 76 insertions(+), 42 deletions(-) diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index 2f24de088e..e09a5bbb42 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -400,18 +400,13 @@ internal class TaskBuilder( // // If this list changes, search for references to this variable name to // find the tests that need to be updated. - val BOOTSTRAP_NATIVE_STDLIB = listOf( + val BOOTSTRAP_NATIVE_STDLIB = mutableListOf( "_bz2.so", // zipfile < importer "_ctypes.so", // java.primitive and importer "_datetime.so", // calendar < importer (see test_datetime) "_lzma.so", // zipfile < importer - "_opcode.so", // opcode < dis < inspect < importer. The importer - // dependency could be removed, but inspect is also - // imported via dataclasses < pprint < elftools in - // Python <= 3.13. "_random.so", // random < tempfile < zipimport - "_sha2.so", // random < tempfile < zipimport (Python >= 3.12) - "_sha512.so", // random < tempfile < zipimport (Python <= 3.11) + "_sha512.so", // random < tempfile < zipimport "_struct.so", // zipfile < importer "binascii.so", // zipfile < importer "math.so", // datetime < calendar < importer @@ -419,6 +414,22 @@ internal class TaskBuilder( "zlib.so" // zipimport ) + val versionParts = python.version!!.split(".") + val versionInt = + (versionParts[0].toInt() * 100) + versionParts[1].toInt() + if (versionInt >= 312) { + BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha512.so")) + BOOTSTRAP_NATIVE_STDLIB.addAll(listOf( + "_sha2.so" // random < tempfile < zipimport + )) + } + if (versionInt >= 313) { + BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha2.so")) + BOOTSTRAP_NATIVE_STDLIB.addAll(listOf( + "_opcode.so" // opcode < dis < inspect < importer + )) + } + for (abi in abis) { project.copy { from(project.zipTree(resolveArtifact(targetNative, abi).file)) diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index 9a74954565..471b56c25d 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -1837,15 +1837,18 @@ def check_assets(self, apk_dir, kwargs): python_version_info = tuple(int(x) for x in python_version.split(".")) stdlib_bootstrap_expected = { - # This is the list from our minimum Python version. For why each of these - # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt. - "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_opcode.so", + # For why each of these modules is needed, see BOOTSTRAP_NATIVE_STDLIB in + # PythonTasks.kt. + "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so", } if python_version_info >= (3, 12): stdlib_bootstrap_expected -= {"_sha512.so"} stdlib_bootstrap_expected |= {"_sha2.so"} + if python_version_info >= (3, 13): + stdlib_bootstrap_expected -= {"_sha2.so"} + stdlib_bootstrap_expected |= {"_opcode.so"} bootstrap_native_dir = join(asset_dir, "bootstrap-native") self.test.assertCountEqual(abis, os.listdir(bootstrap_native_dir)) @@ -1880,7 +1883,7 @@ def check_assets(self, apk_dir, kwargs): "_codecs_hk.so", "_codecs_iso2022.so", "_codecs_jp.so", "_codecs_kr.so", "_codecs_tw.so", "_contextvars.so", "_csv.so", "_decimal.so", "_elementtree.so", "_hashlib.so", "_heapq.so", "_json.so", "_lsprof.so", "_md5.so", - "_multibytecodec.so", "_multiprocessing.so", "_pickle.so", + "_multibytecodec.so", "_multiprocessing.so", "_opcode.so", "_pickle.so", "_posixsubprocess.so", "_queue.so", "_sha1.so", "_sha256.so", "_sha3.so", "_socket.so", "_sqlite3.so", "_ssl.so", "_statistics.so", "_xxsubinterpreters.so", "_xxtestfuzz.so", "array.so", @@ -1900,9 +1903,10 @@ def check_assets(self, apk_dir, kwargs): if python_version_info >= (3, 13): stdlib_native_expected -= { "audioop.so", "_xxinterpchannels.so", "_multiprocessing.so", - "_xxsubinterpreters.so", "ossaudiodev.so"} + "_opcode.so", "_xxsubinterpreters.so", "ossaudiodev.so"} stdlib_native_expected |= { - "_interpreters.so", "_interpchannels.so", "_interpqueues.so"} + "_interpreters.so", "_interpchannels.so", "_interpqueues.so", + "_sha2.so"} for abi in abis: stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy")) diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py index d6a42a1a18..953d788930 100644 --- a/product/runtime/src/main/python/java/android/__init__.py +++ b/product/runtime/src/main/python/java/android/__init__.py @@ -59,10 +59,8 @@ def initialize_warnings(): def initialize_sys(): - # argv defaults to not existing, which may crash some programs. - sys.argv = [""] - - # executable defaults to the empty string, but this causes platform.platform() to crash. + # executable defaults to the empty string, but this causes platform.platform() to + # crash, and would probably confuse a lot of other code as well. try: sys.executable = os.readlink("/proc/{}/exe".format(os.getpid())) except Exception: @@ -119,7 +117,6 @@ def set_default_verify_paths(self): def initialize_multiprocessing(): - import _multiprocessing from multiprocessing import context, heap, pool import threading @@ -161,7 +158,9 @@ def method(self, *args, **kwargs): "workaround can be removed") class SemLock: - SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX + # multiprocessing.synchronize reads this attribute during import. + SEM_VALUE_MAX = 99 + def __init__(self, *args, **kwargs): raise OSError(error_message) diff --git a/product/runtime/src/main/python/java/android/importer.py b/product/runtime/src/main/python/java/android/importer.py index 6ff8171569..b2f1884fe5 100644 --- a/product/runtime/src/main/python/java/android/importer.py +++ b/product/runtime/src/main/python/java/android/importer.py @@ -407,11 +407,12 @@ def joinpath(self, *segments): def __truediv__(self, child): return self.joinpath(child) - def open(self, mode="r", buffering="ignored", **kwargs): + # `buffering` has no effect because the whole file is read immediately. + def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None): if "r" in mode: bio = io.BytesIO(self.finder.get_data(self.zip_path)) if mode == "r": - return io.TextIOWrapper(bio, **kwargs) + return io.TextIOWrapper(bio, encoding, errors, newline) elif sorted(mode) == ["b", "r"]: return bio raise ValueError(f"unsupported mode: {mode!r}") @@ -420,8 +421,8 @@ def read_bytes(self): with self.open('rb') as strm: return strm.read() - def read_text(self, encoding=None): - with self.open(encoding=encoding) as strm: + def read_text(self, encoding=None, errors=None, newline=None): + with self.open("r", -1, encoding, errors, newline) as strm: return strm.read() diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_import.py b/product/runtime/src/test/python/chaquopy/test/android/test_import.py index 7351bb1abf..fd1cca69dd 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_import.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_import.py @@ -61,15 +61,18 @@ def test_bootstrap(self): self.assertCountEqual([ABI], os.listdir(bn_dir)) stdlib_bootstrap_expected = { - # This is the list from our minimum Python version. For why each of these - # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt. - "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_opcode.so", + # For why each of these modules is needed, see BOOTSTRAP_NATIVE_STDLIB in + # PythonTasks.kt. + "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so", } if sys.version_info >= (3, 12): stdlib_bootstrap_expected -= {"_sha512.so"} stdlib_bootstrap_expected |= {"_sha2.so"} + if sys.version_info >= (3, 13): + stdlib_bootstrap_expected -= {"_sha2.so"} + stdlib_bootstrap_expected |= {"_opcode.so"} for subdir, entries in [ (ABI, list(stdlib_bootstrap_expected)), @@ -380,7 +383,7 @@ def check_module(self, mod_name, filename, cache_filename, *, is_package=False, # Verify that the traceback builder can get source code from the loader in all contexts. # (The "package1" test files are also used in TestImport.) def test_exception(self): - col_marker = r'( +\^+\n)?' # Column marker (Python >= 3.11) + col_marker = r'( +[~^]+\n)?' # Column marker (Python >= 3.11) test_frame = ( fr' File "{asset_path(APP_ZIP)}/chaquopy/test/android/test_import.py", ' fr'line \d+, in test_exception\n' @@ -429,8 +432,9 @@ def test_exception(self): col_marker + fr' File "{asset_path(APP_ZIP)}/package1/other_error.py", ' fr'line 1, in \n' - fr' int\("hello"\)\n' - fr"ValueError: invalid literal for int\(\) with base 10: 'hello'\n$") + fr' int\("hello"\)\n' + + col_marker + + r"ValueError: invalid literal for int\(\) with base 10: 'hello'\n$") else: self.fail() diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py index 8de9cdc4cf..faa006dba7 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py @@ -104,7 +104,14 @@ def square_slowly(x): for name in ["Barrier", "BoundedSemaphore", "Condition", "Event", "Lock", "RLock", "Semaphore"]: cls = getattr(synchronize, name) - with self.assertRaisesRegex(OSError, "This platform lacks a functioning sem_open"): + with self.assertRaisesRegex( + OSError, + ( + "This platform lacks a functioning sem_open" + if sys.version_info < (3, 13) + else "No module named '_multiprocessing'" + ) + ): if name == "Barrier": cls(1, ctx=ctx) else: @@ -137,9 +144,9 @@ def test_platform(self): python_bits = platform.architecture()[0] self.assertEqual(python_bits, "64bit" if ("64" in Build.CPU_ABI) else "32bit") - # Requires sys.executable to exist. - p = platform.platform() - self.assertRegex(p, r"^Linux") + self.assertRegex( + platform.platform(), + r"^Linux" if sys.version_info < (3, 13) else r"^Android") def test_select(self): import select @@ -211,7 +218,9 @@ def test_sys(self): for p in sys.path: self.assertTrue(exists(p), p) - self.assertRegex(sys.platform, r"^linux") + self.assertEqual( + sys.platform, + "linux" if sys.version_info < (3, 13) else "android") self.assertNotIn("dirty", sys.version) def test_sysconfig(self): diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py index 968e1e610d..c1c234174d 100644 --- a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py +++ b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py @@ -13,7 +13,11 @@ from time import time from unittest.mock import patch -from java.android.stream import TextLogStream +from importlib import import_module +stream_mod_name = ( + "java.android.stream" if sys.version_info < (3, 13) else "_android_support") +TextLogStream = import_module(stream_mod_name).TextLogStream + from ..test_utils import API_LEVEL as api_level, FilterWarningsCase @@ -458,10 +462,10 @@ def mock_sleep(duration): # ensure a quick and reliable test that doesn't flood the log too much. MAX_KB_PER_SECOND = 100 BUCKET_KB = 10 - with patch("java.android.stream.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), \ - patch("java.android.stream.BUCKET_SIZE", BUCKET_KB * 1024), \ - patch("java.android.stream.sleep", mock_sleep), \ - patch("java.android.stream.time", mock_time): + with patch(f"{stream_mod_name}.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), \ + patch(f"{stream_mod_name}.BUCKET_SIZE", BUCKET_KB * 1024), \ + patch(f"{stream_mod_name}.sleep", mock_sleep), \ + patch(f"{stream_mod_name}.time", mock_time): # Make sure the token bucket is full. stream.write("Initial message to reset _prev_write_time") mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) diff --git a/server/pypi/pkgtest/app/build.gradle b/server/pypi/pkgtest/app/build.gradle index 5e8f63a901..32c1b1ca8d 100644 --- a/server/pypi/pkgtest/app/build.gradle +++ b/server/pypi/pkgtest/app/build.gradle @@ -57,9 +57,7 @@ android { // To test packages, edit the following line to list their names, separated by // spaces. Each name must be a subdirectory of PACKAGES_DIR. def PACKAGES = "" - if (!PACKAGES.isEmpty()) { - addPackages(delegate, PACKAGES.trim().split(/\s+/).toList()) - } + addPackages(delegate, PACKAGES.trim().split(/\s+/).toList()) python { version "3.8" @@ -108,6 +106,10 @@ def addPackages(flavor, List packages) { mkdir(outputDir) String suiteSrc = "" for (req in packages) { + // Splitting an empty string returns a list containing one empty string. + if (req.isEmpty()) { + continue + } def pkgName = req.split("==")[0] def pkgDir = file("${ext.PACKAGES_DIR}/$pkgName") def testPaths = ["test.py", "test"] From bafe2d6b9031d82fcacbd183037674b6e88028d6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 10 Oct 2024 19:58:06 +0100 Subject: [PATCH 14/19] Instead of running patchelf on openssl and sqlite a second time, create stub _chaquopy libraries --- .../com/chaquo/python/internal/Common.java | 2 +- .../test/integration/test_gradle_plugin.py | 12 +++++----- .../python/android/AndroidPlatform.java | 19 ++++++++------- server/pypi/build-wheel.py | 2 ++ target/package-target.sh | 15 ++++++------ target/python/build-and-package.sh | 9 +++++-- target/python/build.sh | 24 +++---------------- target/python/patches/3.13_pending.patch | 17 ++----------- 8 files changed, 39 insertions(+), 61 deletions(-) diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java index 71d13714c8..0260252794 100644 --- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java +++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java @@ -23,7 +23,7 @@ public class Common { PYTHON_VERSIONS.put("3.10.15", "0"); PYTHON_VERSIONS.put("3.11.10", "0"); PYTHON_VERSIONS.put("3.12.7", "0"); - PYTHON_VERSIONS.put("3.13.0rc3", "0"); + PYTHON_VERSIONS.put("3.13.0", "0"); } public static List PYTHON_VERSIONS_SHORT = new ArrayList<>(); diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index 471b56c25d..0f7a6d27a0 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -1955,16 +1955,16 @@ def check_lib(self, lib_dir, kwargs): self.test.assertCountEqual(abis, os.listdir(lib_dir)) for abi in abis: abi_dir = join(lib_dir, abi) - suffix = ( - "chaquopy" if python_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] - else "python") self.test.assertCountEqual( [ "libchaquopy_java.so", - f"libcrypto_{suffix}.so", + "libcrypto_chaquopy.so", + "libcrypto_python.so", f"libpython{python_version}.so", - f"libssl_{suffix}.so", - f"libsqlite3_{suffix}.so", + "libssl_chaquopy.so", + "libssl_python.so", + "libsqlite3_chaquopy.so", + "libsqlite3_python.so", ], os.listdir(abi_dir)) self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi) diff --git a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java index 8af06a46cf..e3a90d654e 100644 --- a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java +++ b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java @@ -259,19 +259,20 @@ private String streamToString(InputStream in) throws IOException { } private void loadNativeLibs() throws JSONException { - String pythonVersion = buildJson.getString("python_version"); - String suffix = - Arrays.asList("3.8", "3.9", "3.10", "3.11", "3.12").contains(pythonVersion) - ? "chaquopy" : "python"; - // From API level 18 we no longer need to load libraries in dependency order. But // we should still keep pre-loading the OpenSSL and SQLite libraries, because we // can't guarantee that our lib directory will always be on the LD_LIBRARY_PATH // (#1198). - System.loadLibrary("crypto_" + suffix); - System.loadLibrary("ssl_" + suffix); - System.loadLibrary("sqlite3_" + suffix); - System.loadLibrary("python" + pythonVersion); + // + // The "python" suffix is the actual library; "chaquopy" is a stub for + // compatibility with existing wheels. See target/package-target.sh. + for (String suffix : Arrays.asList("chaquopy", "python")) { + System.loadLibrary("crypto_" + suffix); + System.loadLibrary("ssl_" + suffix); + System.loadLibrary("sqlite3_" + suffix); + } + + System.loadLibrary("python" + buildJson.getString("python_version")); System.loadLibrary("chaquopy_java"); } diff --git a/server/pypi/build-wheel.py b/server/pypi/build-wheel.py index 95e38b9ad8..12ad017e4e 100755 --- a/server/pypi/build-wheel.py +++ b/server/pypi/build-wheel.py @@ -734,6 +734,8 @@ def fix_wheel(self, in_filename): if fixed_path != original_path: run(f"mv {original_path} {fixed_path}") + # Strip before patching, otherwise the libraries may be corrupted: + # https://github.com/NixOS/patchelf/issues?q=is%3Aissue+strip+in%3Atitle run(f"chmod +w {fixed_path}") run(f"{os.environ['STRIP']} --strip-unneeded {fixed_path}") diff --git a/target/package-target.sh b/target/package-target.sh index 8e782d9a63..687a630696 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -82,7 +82,7 @@ for prefix in $prefixes; do abi=$(basename $prefix) echo "$abi" . "$this_dir/abi-to-host.sh" - . "$this_dir/android-env.sh" # For STRIP + . "$this_dir/android-env.sh" # For CC and STRIP abi_dir="$tmp_dir/$abi" mkdir "$abi_dir" @@ -95,13 +95,14 @@ for prefix in $prefixes; do mkdir -p "$jniLibs_dir" cp $prefix/lib/libpython$version_short.so "$jniLibs_dir" - if [ $version_int -le 312 ]; then - lib_suffix="chaquopy" - else - lib_suffix="python" - fi for name in crypto ssl sqlite3; do - cp "$prefix/lib/lib${name}_$lib_suffix.so" "$jniLibs_dir" + # Add _chaquopy suffixed libraries for compatibility with existing wheels. We + # need this even on Python 3.13, for non-Python wheels like chaquopy-curl. + chaquopy_name=lib${name}_chaquopy.so + "$CC" -shared "-L$prefix/lib" "-l${name}_python" \ + "-Wl,-soname=$chaquopy_name" \ + -o "$prefix/lib/$chaquopy_name" + cp "$prefix/lib/lib${name}_"{chaquopy,python}.so "$jniLibs_dir" done mkdir lib-dynload diff --git a/target/python/build-and-package.sh b/target/python/build-and-package.sh index f868fedd6c..1d5c73f621 100755 --- a/target/python/build-and-package.sh +++ b/target/python/build-and-package.sh @@ -6,8 +6,13 @@ version_short=${1:?} cd $recipe_dir/.. -version_micro=$(./list-versions.py --micro | grep "^$version_short\.") -version_build=$(./list-versions.py --build | grep "^$version_short\.") +version_micro=$(./list-versions.py --micro | grep "^$version_short\." || true) +version_build=$(./list-versions.py --build | grep "^$version_short\." || true) + +if [ -z "$version_micro" ] || [ -z "$version_build" ]; then + echo "Invalid version '$version_short'" >&2 + exit 1 +fi case $version_short in 3.8|3.9|3.10|3.11) diff --git a/target/python/build.sh b/target/python/build.sh index 2d1d25c3f9..2be590f171 100755 --- a/target/python/build.sh +++ b/target/python/build.sh @@ -63,11 +63,11 @@ rm -rf $PREFIX/lib/libpython$version_short* if [ $version_int -le 312 ]; then # Download and unpack libraries needed to compile Python. For a given Python # version, we must maintain binary compatibility with existing wheels. - libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-1 xz-5.4.6-1" + libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-2 xz-5.4.6-1" if [ $version_int -le 308 ]; then - libs+=" openssl-1.1.1w-1" + libs+=" openssl-1.1.1w-2" else - libs+=" openssl-3.0.15-2" + libs+=" openssl-3.0.15-3" fi url_prefix="https://github.com/beeware/cpython-android-source-deps/releases/download" @@ -77,24 +77,6 @@ if [ $version_int -le 312 ]; then curl -Lf "$url" | tar -x -C $PREFIX done - # Rename libraries for binary compatibility with existing wheels. - cd $PREFIX/lib - for name in crypto ssl sqlite3; do - old_name="lib${name}_python.so" - new_name="lib${name}_chaquopy.so" - if [ "$name" = "crypto" ]; then - crypto_old_name=$old_name - crypto_new_name=$new_name - fi - - mv "$old_name" "$new_name" - ln -s "$new_name" "$old_name" - patchelf --set-soname "$new_name" "$new_name" - if [ "$name" = "ssl" ]; then - patchelf --replace-needed "$crypto_old_name" "$crypto_new_name" "$new_name" - fi - done - # Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable. CFLAGS+=" -I$toolchain/sysroot/usr/include" LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$HOST/$api_level" diff --git a/target/python/patches/3.13_pending.patch b/target/python/patches/3.13_pending.patch index fc1ee51111..3fa14f2c03 100644 --- a/target/python/patches/3.13_pending.patch +++ b/target/python/patches/3.13_pending.patch @@ -22,19 +22,6 @@ index 93372e3fe1c..94712602a23 100644 # Unlike Linux, Android does not implicitly use a dlopened library to resolve # relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used -@@ -78,7 +78,11 @@ fi - - if [ -n "${PREFIX:-}" ]; then - abs_prefix=$(realpath $PREFIX) -- CFLAGS="$CFLAGS -I$abs_prefix/include" -+ -+ # Use -idirafter so that package-specified -I directories take priority. For -+ # example, grpcio provides its own BoringSSL headers which must be used rather than -+ # our OpenSSL. -+ CFLAGS="$CFLAGS -idirafter $abs_prefix/include" - LDFLAGS="$LDFLAGS -L$abs_prefix/lib" - - export PKG_CONFIG="pkg-config --define-prefix" diff --git a/Android/android.py b/Android/android.py index 8696d9eaeca..b3ee449ba43 100755 --- a/Android/android.py @@ -45,8 +32,8 @@ index 8696d9eaeca..b3ee449ba43 100755 deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.15-0", - "sqlite-3.45.1-0", "xz-5.4.6-0"]: -+ for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-1", -+ "sqlite-3.45.3-0", "xz-5.4.6-1"]: ++ for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-3", ++ "sqlite-3.45.3-2", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") run(["tar", "-xf", filename]) From 4ca1653837d5440a10488d7979d145ea4da13ad4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 12 Oct 2024 14:32:49 +0100 Subject: [PATCH 15/19] Avoid stripping OpenSSL and SQLite libraries after they've been patched --- target/package-target.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/target/package-target.sh b/target/package-target.sh index 687a630696..41e5505564 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -113,8 +113,12 @@ for prefix in $prefixes; do done rm $dynload_dir/*_test*.so + # The OpenSSL and SQLite libraries were already stripped before being patched to add + # the _python suffix. Stripping them after the patch may corrupt them: + # https://github.com/NixOS/patchelf/issues/507 chmod u+w $(find . -name *.so) - $STRIP $(find . -name *.so) + find . -name "*.so" -type f | grep -vE 'lib(crypto|ssl|sqlite3)_python' \ + | xargs "$STRIP" abi_zip="$target_prefix-$abi.zip" rm -f "$abi_zip" From 571e9af7f9fd3ac94d2515eb7b6d9d51268737ee Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 12 Oct 2024 22:46:33 +0100 Subject: [PATCH 16/19] Avoid using patchelf entirely --- product/runtime/src/main/python/java/android/__init__.py | 3 +++ target/package-target.sh | 6 +----- target/python/build.sh | 6 +++--- target/python/patches/3.13_pending.patch | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py index 953d788930..6e3caab0cb 100644 --- a/product/runtime/src/main/python/java/android/__init__.py +++ b/product/runtime/src/main/python/java/android/__init__.py @@ -109,6 +109,9 @@ def initialize_ssl(): # Unfortunately we can't do this with SSL_CERT_FILE, because OpenSSL ignores # environment variables when getauxval(AT_SECURE) is enabled, which is always the case # on Android (https://android.googlesource.com/platform/bionic/+/6bb01b6%5E%21/). + # + # TODO: to pass the CPython test suite, we have now patched our OpenSSL build to + # ignore AT_SECURE, so we can probably use the environment variable now. import ssl cacert = join(str(context.getFilesDir()), "chaquopy/cacert.pem") def set_default_verify_paths(self): diff --git a/target/package-target.sh b/target/package-target.sh index 41e5505564..687a630696 100755 --- a/target/package-target.sh +++ b/target/package-target.sh @@ -113,12 +113,8 @@ for prefix in $prefixes; do done rm $dynload_dir/*_test*.so - # The OpenSSL and SQLite libraries were already stripped before being patched to add - # the _python suffix. Stripping them after the patch may corrupt them: - # https://github.com/NixOS/patchelf/issues/507 chmod u+w $(find . -name *.so) - find . -name "*.so" -type f | grep -vE 'lib(crypto|ssl|sqlite3)_python' \ - | xargs "$STRIP" + $STRIP $(find . -name *.so) abi_zip="$target_prefix-$abi.zip" rm -f "$abi_zip" diff --git a/target/python/build.sh b/target/python/build.sh index 2be590f171..b10f031fb4 100755 --- a/target/python/build.sh +++ b/target/python/build.sh @@ -63,11 +63,11 @@ rm -rf $PREFIX/lib/libpython$version_short* if [ $version_int -le 312 ]; then # Download and unpack libraries needed to compile Python. For a given Python # version, we must maintain binary compatibility with existing wheels. - libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-2 xz-5.4.6-1" + libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-3 xz-5.4.6-1" if [ $version_int -le 308 ]; then - libs+=" openssl-1.1.1w-2" + libs+=" openssl-1.1.1w-3" else - libs+=" openssl-3.0.15-3" + libs+=" openssl-3.0.15-4" fi url_prefix="https://github.com/beeware/cpython-android-source-deps/releases/download" diff --git a/target/python/patches/3.13_pending.patch b/target/python/patches/3.13_pending.patch index 3fa14f2c03..83ab8e7198 100644 --- a/target/python/patches/3.13_pending.patch +++ b/target/python/patches/3.13_pending.patch @@ -32,8 +32,8 @@ index 8696d9eaeca..b3ee449ba43 100755 deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.15-0", - "sqlite-3.45.1-0", "xz-5.4.6-0"]: -+ for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-3", -+ "sqlite-3.45.3-2", "xz-5.4.6-1"]: ++ for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", ++ "sqlite-3.45.3-3", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") run(["tar", "-xf", filename]) From 9b87b74ca6008968b9edf4eb64954f98abf90eff Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 13 Oct 2024 00:30:31 +0100 Subject: [PATCH 17/19] Increase target API level to 35, and disable edge to edge --- demo/app/build.gradle.kts | 4 ++-- demo/app/src/utils/res/values-v35/utils.xml | 9 +++++++++ demo/app/src/utils/res/values/utils.xml | 3 ++- server/pypi/pkgtest/app/build.gradle | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 demo/app/src/utils/res/values-v35/utils.xml diff --git a/demo/app/build.gradle.kts b/demo/app/build.gradle.kts index 281846dc04..0e7e281d77 100644 --- a/demo/app/build.gradle.kts +++ b/demo/app/build.gradle.kts @@ -23,12 +23,12 @@ afterEvaluate { android { namespace = "com.chaquo.python.demo" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "com.chaquo.python.demo3" minSdk = 24 - targetSdk = 34 + targetSdk = 35 val plugins = buildscript.configurations.getByName("classpath") .resolvedConfiguration.resolvedArtifacts.map { diff --git a/demo/app/src/utils/res/values-v35/utils.xml b/demo/app/src/utils/res/values-v35/utils.xml new file mode 100644 index 0000000000..99471bc758 --- /dev/null +++ b/demo/app/src/utils/res/values-v35/utils.xml @@ -0,0 +1,9 @@ + + + + + +