diff --git a/demo/app/build.gradle.kts b/demo/app/build.gradle.kts
index 281846dc04..e0d1d648a6 100644
--- a/demo/app/build.gradle.kts
+++ b/demo/app/build.gradle.kts
@@ -23,12 +23,12 @@ afterEvaluate {
android {
namespace = "com.chaquo.python.demo"
- compileSdk = 34
+ compileSdk = 35
defaultConfig {
applicationId = "com.chaquo.python.demo3"
minSdk = 24
- targetSdk = 34
+ targetSdk = 35
val plugins = buildscript.configurations.getByName("classpath")
.resolvedConfiguration.resolvedArtifacts.map {
@@ -97,7 +97,7 @@ chaquopy {
defaultConfig {
// Android UI demo
pip {
- install("Pygments==2.2.0") // Also used in Java API demo
+ install("Pygments==2.13.0") // Also used in Java API demo
}
staticProxy("chaquopy.demo.ui_demo")
diff --git a/demo/app/src/utils/python/chaquopy/utils/console.py b/demo/app/src/utils/python/chaquopy/utils/console.py
index ec84908ebb..b6931bd519 100644
--- a/demo/app/src/utils/python/chaquopy/utils/console.py
+++ b/demo/app/src/utils/python/chaquopy/utils/console.py
@@ -77,7 +77,7 @@ def __repr__(self):
def __getattribute__(self, name):
# Forward all attributes that have useful implementations.
if name in [
- "close", "closed", "flush", "writable", # IOBase
+ "close", "closed", "fileno", "flush", "writable", # IOBase
"encoding", "errors", "newlines", "buffer", "detach", # TextIOBase
"line_buffering", "write_through", "reconfigure", # TextIOWrapper
]:
@@ -90,5 +90,9 @@ def write(self, s):
# exception, the app crashes in the same way whether it's using
# ConsoleOutputStream or not.
result = self.stream.write(s)
+
+ # In case `s` is a str subclass that writes itself to stdout or stderr
+ # when we call its methods, convert it to an actual str.
+ s = str.__str__(s)
self.method(s)
return result
diff --git a/demo/app/src/utils/res/values-v35/utils.xml b/demo/app/src/utils/res/values-v35/utils.xml
new file mode 100644
index 0000000000..99471bc758
--- /dev/null
+++ b/demo/app/src/utils/res/values-v35/utils.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/demo/app/src/utils/res/values/utils.xml b/demo/app/src/utils/res/values/utils.xml
index 94e92a9569..731f735fe4 100644
--- a/demo/app/src/utils/res/values/utils.xml
+++ b/demo/app/src/utils/res/values/utils.xml
@@ -2,13 +2,14 @@
-
+
#3F51B5
#303F9F
diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java
index f88e83cabd..0260252794 100644
--- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java
+++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java
@@ -10,7 +10,7 @@ public class Common {
// Minimum Android Gradle plugin version
public static final String MIN_AGP_VERSION = "7.0.0";
- // This should match api_level in target/build-common.sh.
+ // This should match api_level in target/android-env.sh.
public static final int MIN_SDK_VERSION = 24;
public static final int COMPILE_SDK_VERSION = 34;
@@ -18,11 +18,12 @@ public class Common {
public static final Map PYTHON_VERSIONS = new LinkedHashMap<>();
static {
// Version, build number
- PYTHON_VERSIONS.put("3.8.18", "0");
- PYTHON_VERSIONS.put("3.9.18", "0");
- PYTHON_VERSIONS.put("3.10.13", "0");
- PYTHON_VERSIONS.put("3.11.6", "0");
- PYTHON_VERSIONS.put("3.12.1", "0");
+ PYTHON_VERSIONS.put("3.8.20", "0");
+ PYTHON_VERSIONS.put("3.9.20", "0");
+ PYTHON_VERSIONS.put("3.10.15", "0");
+ PYTHON_VERSIONS.put("3.11.10", "0");
+ PYTHON_VERSIONS.put("3.12.7", "0");
+ PYTHON_VERSIONS.put("3.13.0", "0");
}
public static List PYTHON_VERSIONS_SHORT = new ArrayList<>();
diff --git a/product/gradle-plugin/README.md b/product/gradle-plugin/README.md
index 906cb97135..4777f489fb 100644
--- a/product/gradle-plugin/README.md
+++ b/product/gradle-plugin/README.md
@@ -122,8 +122,10 @@ After stable release:
* Increment Chaquopy major version if not already done.
* Update `MIN_SDK_VERSION` in Common.java.
-* Update `api_level` in target/build-common.sh.
-* Update default API level in server/pypi/build-wheel.py.
+* Update `api_level` in target/android-env.sh.
+* In server/pypi/build-wheel.py:
+ * Update default API level.
+ * Update `STANDARD_LIBS` with any libraries added in the new level.
* Search repository for other things that should be updated, including workarounds which
are now unnecessary:
* Useful regex: `api.?level|android.?ver|android \d|min.?sdk|SDK_INT`
diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt
index 119cb8667c..e09a5bbb42 100644
--- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt
+++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt
@@ -224,6 +224,9 @@ internal class TaskBuilder(
val customIndexUrl = listOf("--index-url", "-i").any {
it in python.pip.options
}
+ val versionFull = pythonVersionInfo(python).key
+ val versionFullNoPre =
+ """\d+\.\d+\.\d+""".toRegex().find(versionFull)!!.value
execBuildPython {
args("-m", "chaquopy.pip_install")
@@ -237,7 +240,7 @@ internal class TaskBuilder(
args("--extra-index-url", "https://chaquo.com/pypi-13.1")
}
args("--implementation", Common.PYTHON_IMPLEMENTATION)
- args("--python-version", pythonVersionInfo(python).key)
+ args("--python-version", versionFullNoPre)
args("--abi", (Common.PYTHON_IMPLEMENTATION +
python.version!!.replace(".", "")))
args("--no-compile")
@@ -397,14 +400,13 @@ internal class TaskBuilder(
//
// If this list changes, search for references to this variable name to
// find the tests that need to be updated.
- val BOOTSTRAP_NATIVE_STDLIB = listOf(
+ val BOOTSTRAP_NATIVE_STDLIB = mutableListOf(
"_bz2.so", // zipfile < importer
"_ctypes.so", // java.primitive and importer
"_datetime.so", // calendar < importer (see test_datetime)
"_lzma.so", // zipfile < importer
"_random.so", // random < tempfile < zipimport
- "_sha2.so", // random < tempfile < zipimport (Python >= 3.12)
- "_sha512.so", // random < tempfile < zipimport (Python <= 3.11)
+ "_sha512.so", // random < tempfile < zipimport
"_struct.so", // zipfile < importer
"binascii.so", // zipfile < importer
"math.so", // datetime < calendar < importer
@@ -412,6 +414,22 @@ internal class TaskBuilder(
"zlib.so" // zipimport
)
+ val versionParts = python.version!!.split(".")
+ val versionInt =
+ (versionParts[0].toInt() * 100) + versionParts[1].toInt()
+ if (versionInt >= 312) {
+ BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha512.so"))
+ BOOTSTRAP_NATIVE_STDLIB.addAll(listOf(
+ "_sha2.so" // random < tempfile < zipimport
+ ))
+ }
+ if (versionInt >= 313) {
+ BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha2.so"))
+ BOOTSTRAP_NATIVE_STDLIB.addAll(listOf(
+ "_opcode.so" // opcode < dis < inspect < importer
+ ))
+ }
+
for (abi in abis) {
project.copy {
from(project.zipTree(resolveArtifact(targetNative, abi).file))
diff --git a/product/gradle-plugin/src/main/python/chaquopy/pyc.py b/product/gradle-plugin/src/main/python/chaquopy/pyc.py
index 8f91c9dcab..6faf141fd5 100644
--- a/product/gradle-plugin/src/main/python/chaquopy/pyc.py
+++ b/product/gradle-plugin/src/main/python/chaquopy/pyc.py
@@ -15,14 +15,15 @@
import warnings
-# See the list in importlib/_bootstrap_external.py.
+# See the CPython source code in Include/internal/pycore_magic_number.h or
+# Lib/importlib/_bootstrap_external.py.
MAGIC = {
- "3.7": 3394,
"3.8": 3413,
"3.9": 3425,
"3.10": 3439,
"3.11": 3495,
"3.12": 3531,
+ "3.13": 3571,
}
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle
index 71bf30fb59..bb178e3735 100644
--- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle
@@ -15,6 +15,8 @@ android {
versionName "0.0.1"
python {
version "3.10"
+ pip { install "six" }
+ pyc { pip true }
}
ndk {
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle
index f2031892b1..aea11d6abb 100644
--- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle
@@ -15,6 +15,8 @@ android {
versionName "0.0.1"
python {
version "3.11"
+ pip { install "six" }
+ pyc { pip true }
}
ndk {
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle
index aeac9bd522..8b0fd7573f 100644
--- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle
@@ -15,6 +15,8 @@ android {
versionName "0.0.1"
python {
version "3.12"
+ pip { install "six" }
+ pyc { pip true }
}
ndk {
abiFilters "arm64-v8a", "x86_64"
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle
new file mode 100644
index 0000000000..5e2ae7744c
--- /dev/null
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle
@@ -0,0 +1,25 @@
+plugins {
+ id 'com.android.application'
+ id 'com.chaquo.python'
+}
+
+android {
+ namespace "com.chaquo.python.test"
+ compileSdk 31
+
+ defaultConfig {
+ applicationId "com.chaquo.python.test"
+ minSdk 24
+ targetSdk 31
+ versionCode 1
+ versionName "0.0.1"
+ python {
+ version "3.13"
+ pip { install "six" }
+ pyc { pip true }
+ }
+ ndk {
+ abiFilters "arm64-v8a", "x86_64"
+ }
+ }
+}
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle
index 14a0db0fee..6867e8ee02 100644
--- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle
@@ -15,6 +15,8 @@ android {
versionName "0.0.1"
python {
version "3.8"
+ pip { install "six" }
+ pyc { pip true }
}
ndk {
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle
index 26145a1982..8abfd0835f 100644
--- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle
+++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle
@@ -15,6 +15,8 @@ android {
versionName "0.0.1"
python {
version "3.9"
+ pip { install "six" }
+ pyc { pip true }
}
ndk {
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py
index 2e330a221d..0f7a6d27a0 100644
--- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py
+++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py
@@ -55,7 +55,7 @@ def list_versions(mode):
for full_version in list_versions("micro").splitlines():
version = full_version.rpartition(".")[0]
PYTHON_VERSIONS[version] = full_version
-assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12"]
+assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
DEFAULT_PYTHON_VERSION_FULL = PYTHON_VERSIONS[DEFAULT_PYTHON_VERSION]
NON_DEFAULT_PYTHON_VERSION = "3.10"
@@ -480,7 +480,9 @@ def check_version(self, run, version):
abis = ["arm64-v8a", "x86_64"]
if version in ["3.8", "3.9", "3.10", "3.11"]:
abis += ["armeabi-v7a", "x86"]
- run.rerun(f"PythonVersion/{version}", python_version=version, abis=abis)
+ run.rerun(
+ f"PythonVersion/{version}", python_version=version, abis=abis,
+ requirements=["six.py"])
if version == DEFAULT_PYTHON_VERSION:
self.assertNotInLong(self.WARNING.format(".*"), run.stdout, re=True)
@@ -1835,14 +1837,18 @@ def check_assets(self, apk_dir, kwargs):
python_version_info = tuple(int(x) for x in python_version.split("."))
stdlib_bootstrap_expected = {
- # This is the list from our minimum Python version. For why each of these
- # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt.
- "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so",
- "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so",
+ # For why each of these modules is needed, see BOOTSTRAP_NATIVE_STDLIB in
+ # PythonTasks.kt.
+ "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so",
+ "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so",
+ "mmap.so", "zlib.so",
}
if python_version_info >= (3, 12):
stdlib_bootstrap_expected -= {"_sha512.so"}
stdlib_bootstrap_expected |= {"_sha2.so"}
+ if python_version_info >= (3, 13):
+ stdlib_bootstrap_expected -= {"_sha2.so"}
+ stdlib_bootstrap_expected |= {"_opcode.so"}
bootstrap_native_dir = join(asset_dir, "bootstrap-native")
self.test.assertCountEqual(abis, os.listdir(bootstrap_native_dir))
@@ -1863,11 +1869,13 @@ def check_assets(self, apk_dir, kwargs):
if "stdlib" in pyc:
self.check_pyc(stdlib_zip, "argparse.pyc", kwargs)
- # Data files packaged with stdlib: see target/package_target.sh.
- for grammar_stem in ["Grammar", "PatternGrammar"]:
- self.test.assertIn("lib2to3/{}{}.final.0.pickle".format(
- grammar_stem, PYTHON_VERSIONS[python_version]),
- stdlib_files)
+ # Data files packaged with lib2to3: see target/package_target.sh.
+ # This module was removed in Python 3.13.
+ if python_version_info < (3, 13):
+ for grammar_stem in ["Grammar", "PatternGrammar"]:
+ self.test.assertIn("lib2to3/{}{}.final.0.pickle".format(
+ grammar_stem, PYTHON_VERSIONS[python_version]),
+ stdlib_files)
stdlib_native_expected = {
# This is the list from the minimum supported Python version.
@@ -1892,6 +1900,13 @@ def check_assets(self, apk_dir, kwargs):
if python_version_info >= (3, 12):
stdlib_native_expected -= {"_sha256.so", "_typing.so"}
stdlib_native_expected |= {"_xxinterpchannels.so", "xxsubtype.so"}
+ if python_version_info >= (3, 13):
+ stdlib_native_expected -= {
+ "audioop.so", "_xxinterpchannels.so", "_multiprocessing.so",
+ "_opcode.so", "_xxsubinterpreters.so", "ossaudiodev.so"}
+ stdlib_native_expected |= {
+ "_interpreters.so", "_interpchannels.so", "_interpqueues.so",
+ "_sha2.so"}
for abi in abis:
stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy"))
@@ -1919,14 +1934,15 @@ def check_assets(self, apk_dir, kwargs):
build_json["assets"])
def check_pyc(self, zip_file, pyc_filename, kwargs):
- # See the list in importlib/_bootstrap_external.py.
+ # See the CPython source code at Include/internal/pycore_magic_number.h or
+ # Lib/importlib/_bootstrap_external.py.
MAGIC = {
- "3.7": 3394,
"3.8": 3413,
"3.9": 3425,
"3.10": 3439,
"3.11": 3495,
"3.12": 3531,
+ "3.13": 3571,
}
with zip_file.open(pyc_filename) as pyc_file:
self.test.assertEqual(
@@ -1940,9 +1956,16 @@ def check_lib(self, lib_dir, kwargs):
for abi in abis:
abi_dir = join(lib_dir, abi)
self.test.assertCountEqual(
- ["libchaquopy_java.so", "libcrypto_chaquopy.so",
- f"libpython{kwargs['python_version']}.so", "libssl_chaquopy.so",
- "libsqlite3_chaquopy.so"],
+ [
+ "libchaquopy_java.so",
+ "libcrypto_chaquopy.so",
+ "libcrypto_python.so",
+ f"libpython{python_version}.so",
+ "libssl_chaquopy.so",
+ "libssl_python.so",
+ "libsqlite3_chaquopy.so",
+ "libsqlite3_python.so",
+ ],
os.listdir(abi_dir))
self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi)
diff --git a/product/runtime/build.gradle b/product/runtime/build.gradle
index 5e5b0fcff0..48f80ac853 100644
--- a/product/runtime/build.gradle
+++ b/product/runtime/build.gradle
@@ -271,15 +271,24 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) {
args ("-DCHAQUOPY_INCLUDE_JAVA=$javaHome/include;" +
"$javaHome/include/$javaIncludeSubdir")
} else {
- // This must be one of the NDK versions which are pre-installed on the
- // GitHub Actions runner. Ideally it would also match the version in
- // target/build-common.sh, but the latter is more difficult to change.
- def ndkDir = sdkPath("ndk/26.3.11579264")
+ String ndkDir = null
+ def androidEnvFile = file("../../target/android-env.sh").absoluteFile
+ for (line in androidEnvFile.readLines()) {
+ def match = line =~ /ndk_version=(\S+)/
+ if (match) {
+ ndkDir = sdkPath("ndk/${match.group(1)}")
+ break
+ }
+ }
+ if (ndkDir == null) {
+ throw new GradleException("Failed to find NDK version in $androidEnvFile")
+ }
def prefixDir = "$projectDir/../../target/prefix/$abi"
args "-DCMAKE_TOOLCHAIN_FILE=$ndkDir/build/cmake/android.toolchain.cmake",
- "-DANDROID_ABI=$abi", "-DANDROID_STL=system",
+ "-DANDROID_ABI=$abi",
"-DANDROID_NATIVE_API_LEVEL=$Common.MIN_SDK_VERSION",
+ "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON",
"-DCHAQUOPY_PYTHON_VERSIONS=${pyVersions.join(';')}",
"-DCHAQUOPY_INCLUDE_PYTHON=$prefixDir/include",
"-DCHAQUOPY_LIB_DIRS=$prefixDir/lib"
diff --git a/product/runtime/docs/sphinx/android.rst b/product/runtime/docs/sphinx/android.rst
index 0d3dd758d1..81e494d140 100644
--- a/product/runtime/docs/sphinx/android.rst
+++ b/product/runtime/docs/sphinx/android.rst
@@ -236,7 +236,7 @@ You can set your app's Python version like this::
}
In :doc:`this version of Chaquopy <../versions>`, the default Python version is 3.8. The
-other available versions are 3.9, 3.10, 3.11 and 3.12, but these may have fewer
+other available versions are 3.9, 3.10, 3.11, 3.12 and 3.13, but these may have fewer
:ref:`packages ` available.
.. _android-source:
diff --git a/product/runtime/requirements-build.txt b/product/runtime/requirements-build.txt
index a521eeabc1..7b5a120452 100644
--- a/product/runtime/requirements-build.txt
+++ b/product/runtime/requirements-build.txt
@@ -1 +1 @@
-Cython==0.29.36
+Cython==3.0.11
diff --git a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java
index 835e3f0c6e..e3a90d654e 100644
--- a/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java
+++ b/product/runtime/src/main/java/com/chaquo/python/android/AndroidPlatform.java
@@ -263,9 +263,15 @@ private void loadNativeLibs() throws JSONException {
// we should still keep pre-loading the OpenSSL and SQLite libraries, because we
// can't guarantee that our lib directory will always be on the LD_LIBRARY_PATH
// (#1198).
- System.loadLibrary("crypto_chaquopy");
- System.loadLibrary("ssl_chaquopy");
- System.loadLibrary("sqlite3_chaquopy");
+ //
+ // The "python" suffix is the actual library; "chaquopy" is a stub for
+ // compatibility with existing wheels. See target/package-target.sh.
+ for (String suffix : Arrays.asList("chaquopy", "python")) {
+ System.loadLibrary("crypto_" + suffix);
+ System.loadLibrary("ssl_" + suffix);
+ System.loadLibrary("sqlite3_" + suffix);
+ }
+
System.loadLibrary("python" + buildJson.getString("python_version"));
System.loadLibrary("chaquopy_java");
}
diff --git a/product/runtime/src/main/python/chaquopy_java.pyx b/product/runtime/src/main/python/chaquopy_java.pyx
index d683a664cf..5c9939ddbc 100644
--- a/product/runtime/src/main/python/chaquopy_java.pyx
+++ b/product/runtime/src/main/python/chaquopy_java.pyx
@@ -30,15 +30,16 @@ cdef extern from "chaquopy_java_extra.h":
void PyInit_chaquopy_java() except *
-cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved):
+cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved) noexcept:
return JNI_VERSION_1_6
# === com.chaquo.python.Python ================================================
# This runs before Py_Initialize, so it must compile to pure C.
-cdef public void Java_com_chaquo_python_Python_startNative \
- (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path):
+cdef public void Java_com_chaquo_python_Python_startNative(
+ JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path
+) noexcept:
if getenv("CHAQUOPY_PROCESS_TYPE") == NULL: # See jvm.pxi
startNativeJava(env, j_platform, j_python_path)
else:
@@ -46,7 +47,9 @@ cdef public void Java_com_chaquo_python_Python_startNative \
# We're running in a Java process, so start the Python VM.
-cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path):
+cdef void startNativeJava(
+ JNIEnv *env, jobject j_platform, jobject j_python_path
+) noexcept:
cdef const char *python_path
if j_python_path != NULL:
python_path = env[0].GetStringUTFChars(env, j_python_path, NULL)
@@ -54,7 +57,7 @@ cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path
throw_simple_exception(env, "GetStringUTFChars failed in startNativeJava")
return
try:
- if not set_path(env, python_path):
+ if not set_env(env, "PYTHONPATH", python_path):
return
if not set_env(env, "CHAQUOPY_PROCESS_TYPE", "java"): # See chaquopy.pyx
return
@@ -86,12 +89,12 @@ cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path
# module intialization, so the GIL now exists and we have it. We must release the GIL
# before we return to Java so that the methods below can be called from any thread.
# (http://bugs.python.org/issue1720250)
- PyEval_SaveThread();
+ PyEval_SaveThread()
# WARNING: This function (specifically PyInit_chaquopy_java) will crash if called
# more than once.
-cdef bint init_module(JNIEnv *env) with gil:
+cdef bint init_module(JNIEnv *env) noexcept with gil:
try:
# See CYTHON_PEP489_MULTI_PHASE_INIT in chaquopy_java_extra.h.
PyInit_chaquopy_java()
@@ -101,14 +104,9 @@ cdef bint init_module(JNIEnv *env) with gil:
return False
-# This runs before Py_Initialize, so it must compile to pure C.
-cdef public bint set_path(JNIEnv *env, const char *python_path):
- return set_env(env, "PYTHONPATH", python_path)
-
-
# The POSIX setenv function is not available on MSYS2.
# This runs before Py_Initialize, so it must compile to pure C.
-cdef bint set_env(JNIEnv *env, const char *name, const char *value):
+cdef bint set_env(JNIEnv *env, const char *name, const char *value) noexcept:
cdef int putenvArgLen = strlen(name) + 1 + strlen(value) + 1
cdef char *putenvArg = malloc(putenvArgLen)
if snprintf(putenvArg, putenvArgLen, "%s=%s", name, value) != (putenvArgLen - 1):
@@ -124,7 +122,7 @@ cdef bint set_env(JNIEnv *env, const char *name, const char *value):
cdef public jlong Java_com_chaquo_python_Python_getModuleNative \
- (JNIEnv *env, jobject this, jobject j_name) with gil:
+ (JNIEnv *env, jobject this, jobject j_name) noexcept with gil:
try:
return p2j_pyobject(env, import_module(j2p_string(env, LocalRef.create(env, j_name))))
except BaseException:
@@ -136,7 +134,7 @@ cdef public jlong Java_com_chaquo_python_Python_getModuleNative \
# === com.chaquo.python.PyObject ==============================================
cdef public void Java_com_chaquo_python_PyObject_closeNative \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
Py_DECREF(j2p_pyobject(env, this)) # Matches with INCREF in p2j_pyobject.
return
@@ -146,7 +144,7 @@ cdef public void Java_com_chaquo_python_PyObject_closeNative \
cdef public jlong Java_com_chaquo_python_PyObject_fromJavaNative \
- (JNIEnv *env, jobject klass, jobject o) with gil:
+ (JNIEnv *env, jobject klass, jobject o) noexcept with gil:
try:
return p2j_pyobject(env, j2p(env, LocalRef.create(env, o)))
except BaseException:
@@ -156,7 +154,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_fromJavaNative \
cdef public jobject Java_com_chaquo_python_PyObject_toJava \
- (JNIEnv *env, jobject this, jobject to_klass) with gil:
+ (JNIEnv *env, jobject this, jobject to_klass) noexcept with gil:
try:
self = j2p_pyobject(env, this)
if not to_klass:
@@ -177,7 +175,7 @@ cdef public jobject Java_com_chaquo_python_PyObject_toJava \
# can't simply call a common function, because the main return statement has to be inside a
# SavedException try block in case of numeric overflow.
cdef public jboolean Java_com_chaquo_python_PyObject_toBoolean \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -190,7 +188,7 @@ cdef public jboolean Java_com_chaquo_python_PyObject_toBoolean \
return 0
cdef public jbyte Java_com_chaquo_python_PyObject_toByte \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -203,7 +201,7 @@ cdef public jbyte Java_com_chaquo_python_PyObject_toByte \
return 0
cdef public jchar Java_com_chaquo_python_PyObject_toChar \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -219,7 +217,7 @@ cdef public jchar Java_com_chaquo_python_PyObject_toChar \
return 0
cdef public jshort Java_com_chaquo_python_PyObject_toShort \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -232,7 +230,7 @@ cdef public jshort Java_com_chaquo_python_PyObject_toShort \
return 0
cdef public jint Java_com_chaquo_python_PyObject_toInt \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -245,7 +243,7 @@ cdef public jint Java_com_chaquo_python_PyObject_toInt \
return 0
cdef public jlong Java_com_chaquo_python_PyObject_toLong \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -258,7 +256,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_toLong \
return 0
cdef public jfloat Java_com_chaquo_python_PyObject_toFloat \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -274,7 +272,7 @@ cdef public jfloat Java_com_chaquo_python_PyObject_toFloat \
return 0
cdef public jdouble Java_com_chaquo_python_PyObject_toDouble \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
try:
@@ -287,7 +285,7 @@ cdef public jdouble Java_com_chaquo_python_PyObject_toDouble \
return 0
cdef public jlong Java_com_chaquo_python_PyObject_id \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
return id(j2p_pyobject(env, this))
except BaseException:
@@ -297,7 +295,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_id \
cdef public jlong Java_com_chaquo_python_PyObject_typeNative \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
return p2j_pyobject(env, type(j2p_pyobject(env, this)))
except BaseException:
@@ -307,7 +305,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_typeNative \
cdef public jlong Java_com_chaquo_python_PyObject_callThrowsNative \
- (JNIEnv *env, jobject this, jobject jargs) with gil:
+ (JNIEnv *env, jobject this, jobject jargs) noexcept with gil:
try:
return call(env, j2p_pyobject(env, this), jargs)
except BaseException:
@@ -319,7 +317,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_callThrowsNative \
# It's worth making this a native method in order to avoid the temporary PyObject which would
# be created by `get(name).call(...)`.
cdef public jlong Java_com_chaquo_python_PyObject_callAttrThrowsNative \
- (JNIEnv *env, jobject this, jobject j_key, jobject jargs) with gil:
+ (JNIEnv *env, jobject this, jobject j_key, jobject jargs) noexcept with gil:
try:
attr = getattr(j2p_pyobject(env, this),
j2p_string(env, LocalRef.create(env, j_key)))
@@ -357,7 +355,7 @@ cdef jlong call(JNIEnv *j_env, obj, jobject jargs) except? 0:
# === com.chaquo.python.PyObject (Map) ========================================
cdef public jboolean Java_com_chaquo_python_PyObject_containsKeyNative \
- (JNIEnv *env, jobject this, jobject j_key) with gil:
+ (JNIEnv *env, jobject this, jobject j_key) noexcept with gil:
try:
self = j2p_pyobject(env, this)
key = j2p_string(env, LocalRef.create(env, j_key))
@@ -369,7 +367,7 @@ cdef public jboolean Java_com_chaquo_python_PyObject_containsKeyNative \
cdef public jlong Java_com_chaquo_python_PyObject_getNative \
- (JNIEnv *env, jobject this, jobject j_key) with gil:
+ (JNIEnv *env, jobject this, jobject j_key) noexcept with gil:
try:
self = j2p_pyobject(env, this)
key = j2p_string(env, LocalRef.create(env, j_key))
@@ -385,7 +383,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_getNative \
cdef public jlong Java_com_chaquo_python_PyObject_putNative \
- (JNIEnv *env, jobject this, jobject j_key, jobject j_value) with gil:
+ (JNIEnv *env, jobject this, jobject j_key, jobject j_value) noexcept with gil:
try:
self = j2p_pyobject(env, this)
key = j2p_string(env, LocalRef.create(env, j_key))
@@ -402,7 +400,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_putNative \
cdef public jlong Java_com_chaquo_python_PyObject_removeNative \
- (JNIEnv *env, jobject this, jobject j_key) with gil:
+ (JNIEnv *env, jobject this, jobject j_key) noexcept with gil:
try:
self = j2p_pyobject(env, this)
key = j2p_string(env, LocalRef.create(env, j_key))
@@ -419,7 +417,7 @@ cdef public jlong Java_com_chaquo_python_PyObject_removeNative \
cdef public jobject Java_com_chaquo_python_PyObject_dir \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
keys = java.jclass("java.util.ArrayList")()
for key in dir(j2p_pyobject(env, this)):
@@ -433,7 +431,7 @@ cdef public jobject Java_com_chaquo_python_PyObject_dir \
# === com.chaquo.python.PyObject (Object) =====================================
cdef public jboolean Java_com_chaquo_python_PyObject_equals \
- (JNIEnv *env, jobject this, jobject that) with gil:
+ (JNIEnv *env, jobject this, jobject that) noexcept with gil:
try:
return j2p_pyobject(env, this) == j2p(env, LocalRef.create(env, that))
except BaseException:
@@ -443,11 +441,11 @@ cdef public jboolean Java_com_chaquo_python_PyObject_equals \
cdef public jobject Java_com_chaquo_python_PyObject_toString \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
return to_string(env, this, str)
cdef public jobject Java_com_chaquo_python_PyObject_repr \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
return to_string(env, this, repr)
cdef jobject to_string(JNIEnv *env, jobject this, func):
@@ -461,7 +459,7 @@ cdef jobject to_string(JNIEnv *env, jobject this, func):
cdef public jint Java_com_chaquo_python_PyObject_hashCode \
- (JNIEnv *env, jobject this) with gil:
+ (JNIEnv *env, jobject this) noexcept with gil:
try:
self = j2p_pyobject(env, this)
return ctypes.c_int32(hash(self)).value
diff --git a/product/runtime/src/main/python/java/android/__init__.py b/product/runtime/src/main/python/java/android/__init__.py
index c6defeb698..6e3caab0cb 100644
--- a/product/runtime/src/main/python/java/android/__init__.py
+++ b/product/runtime/src/main/python/java/android/__init__.py
@@ -4,7 +4,7 @@
import traceback
from types import ModuleType
import warnings
-from . import stream, importer
+from . import importer
from org.json import JSONArray, JSONObject
@@ -13,12 +13,25 @@ def initialize(context_local, build_json_object, app_path):
global context
context = context_local
- stream.initialize()
+ # Redirect stdout and stderr to logcat - this was upstreamed in Python 3.13.
+ if sys.version_info < (3, 13):
+ from ctypes import CDLL, c_char_p, c_int
+ from . import stream
+
+ android_log_write = getattr(CDLL("liblog.so"), "__android_log_write")
+ android_log_write.argtypes = (c_int, c_char_p, c_char_p)
+ stream.init_streams(android_log_write, stdout_prio=4, stderr_prio=5)
+ elif sys.stdout.errors == "backslashreplace":
+ # This fix should be upstreamed in Python 3.13.1.
+ raise Exception("see if sys.stdout.errors workaround can be removed")
+ else:
+ sys.stdout.reconfigure(errors="backslashreplace")
+
importer.initialize(context, convert_json_object(build_json_object), app_path)
# These are ordered roughly from low to high level.
for name in [
- "warnings", "sys", "os", "tempfile", "socket", "ssl", "multiprocessing"
+ "warnings", "sys", "os", "tempfile", "ssl", "multiprocessing"
]:
importer.add_import_trigger(name, globals()[f"initialize_{name}"])
@@ -46,10 +59,8 @@ def initialize_warnings():
def initialize_sys():
- # argv defaults to not existing, which may crash some programs.
- sys.argv = [""]
-
- # executable defaults to the empty string, but this causes platform.platform() to crash.
+ # executable defaults to the empty string, but this causes platform.platform() to
+ # crash, and would probably confuse a lot of other code as well.
try:
sys.executable = os.readlink("/proc/{}/exe".format(os.getpid()))
except Exception:
@@ -91,22 +102,6 @@ def initialize_tempfile():
os.environ["TMPDIR"] = tmpdir
-def initialize_socket():
- import socket
-
- # Some functions aren't available until API level 24, so Python omits them from the
- # module. Instead, make them throw OSError as documented.
- def unavailable(*args, **kwargs):
- raise OSError("this function is not available in this build of Python")
-
- for name in ["if_nameindex", "if_nametoindex", "if_indextoname"]:
- if hasattr(socket, name):
- raise Exception(
- f"socket.{name} now exists: check if its workaround can be removed"
- )
- setattr(socket, name, unavailable)
-
-
def initialize_ssl():
# OpenSSL may be able to find the system CA store on some devices, but for consistency
# we disable this and use our own bundled file.
@@ -114,6 +109,9 @@ def initialize_ssl():
# Unfortunately we can't do this with SSL_CERT_FILE, because OpenSSL ignores
# environment variables when getauxval(AT_SECURE) is enabled, which is always the case
# on Android (https://android.googlesource.com/platform/bionic/+/6bb01b6%5E%21/).
+ #
+ # TODO: to pass the CPython test suite, we have now patched our OpenSSL build to
+ # ignore AT_SECURE, so we can probably use the environment variable now.
import ssl
cacert = join(str(context.getFilesDir()), "chaquopy/cacert.pem")
def set_default_verify_paths(self):
@@ -122,7 +120,6 @@ def set_default_verify_paths(self):
def initialize_multiprocessing():
- import _multiprocessing
from multiprocessing import context, heap, pool
import threading
@@ -164,7 +161,9 @@ def method(self, *args, **kwargs):
"workaround can be removed")
class SemLock:
- SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
+ # multiprocessing.synchronize reads this attribute during import.
+ SEM_VALUE_MAX = 99
+
def __init__(self, *args, **kwargs):
raise OSError(error_message)
diff --git a/product/runtime/src/main/python/java/android/importer.py b/product/runtime/src/main/python/java/android/importer.py
index 6ff8171569..b2f1884fe5 100644
--- a/product/runtime/src/main/python/java/android/importer.py
+++ b/product/runtime/src/main/python/java/android/importer.py
@@ -407,11 +407,12 @@ def joinpath(self, *segments):
def __truediv__(self, child):
return self.joinpath(child)
- def open(self, mode="r", buffering="ignored", **kwargs):
+ # `buffering` has no effect because the whole file is read immediately.
+ def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
if "r" in mode:
bio = io.BytesIO(self.finder.get_data(self.zip_path))
if mode == "r":
- return io.TextIOWrapper(bio, **kwargs)
+ return io.TextIOWrapper(bio, encoding, errors, newline)
elif sorted(mode) == ["b", "r"]:
return bio
raise ValueError(f"unsupported mode: {mode!r}")
@@ -420,8 +421,8 @@ def read_bytes(self):
with self.open('rb') as strm:
return strm.read()
- def read_text(self, encoding=None):
- with self.open(encoding=encoding) as strm:
+ def read_text(self, encoding=None, errors=None, newline=None):
+ with self.open("r", -1, encoding, errors, newline) as strm:
return strm.read()
diff --git a/product/runtime/src/main/python/java/android/stream.py b/product/runtime/src/main/python/java/android/stream.py
index 283e21b328..40df0d93fd 100644
--- a/product/runtime/src/main/python/java/android/stream.py
+++ b/product/runtime/src/main/python/java/android/stream.py
@@ -1,62 +1,114 @@
-from android.util import Log
+# This file is based on Lib/_android_support.py on the CPython main branch.
+
import io
import sys
-
-
-# The maximum length of a log message in bytes, including the level marker and tag, is defined
-# as LOGGER_ENTRY_MAX_PAYLOAD in platform/system/logging/liblog/include/log/log.h. As of API
-# level 30, messages longer than this will be be truncated by logcat. This limit has already
-# been reduced at least once in the history of Android (from 4076 to 4068 between API level 23
-# and 26), so leave some headroom.
-#
-# This should match the native stdio buffer size in android_platform.c.
-MAX_LINE_LEN_BYTES = 4000
-
-# UTF-8 uses a maximum of 4 bytes per character. However, if the actual number of bytes
-# per character is smaller than that, then TextIOWrapper may join multiple consecutive
-# writes before passing them to the binary stream.
-MAX_LINE_LEN_CHARS = MAX_LINE_LEN_BYTES // 4
-
-
-def initialize():
- # Log levels are consistent with those used by Java.
- sys.stdout = TextLogStream(Log.INFO, "python.stdout")
- sys.stderr = TextLogStream(Log.WARN, "python.stderr")
+from threading import RLock
+from time import sleep, time
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then we may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+ if sys.executable:
+ return # Not embedded in an app.
+
+ global logcat
+ logcat = Logcat(android_log_write)
+
+ sys.stdout = TextLogStream(
+ stdout_prio, "python.stdout", sys.stdout.fileno())
+ sys.stderr = TextLogStream(
+ stderr_prio, "python.stderr", sys.stderr.fileno())
class TextLogStream(io.TextIOWrapper):
- def __init__(self, level, tag):
- super().__init__(BinaryLogStream(self, level, tag),
- encoding="UTF-8", errors="backslashreplace",
- line_buffering=True)
- self._CHUNK_SIZE = MAX_LINE_LEN_BYTES
+ def __init__(self, prio, tag, fileno=None, **kwargs):
+ # The default is surrogateescape for stdout and backslashreplace for
+ # stderr, but in the context of an Android log, readability is more
+ # important than reversibility.
+ kwargs.setdefault("encoding", "UTF-8")
+ kwargs.setdefault("errors", "backslashreplace")
+
+ super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
+ self._lock = RLock()
+ self._pending_bytes = []
+ self._pending_bytes_count = 0
def __repr__(self):
return f""
def write(self, s):
if not isinstance(s, str):
- # Same wording as TextIOWrapper.write.
- raise TypeError(f"write() argument must be str, not {type(s).__name__}")
-
- # To avoid combining multiple lines into a single log message, we split the string
- # into separate lines before sending it to the superclass. Note that
- # "".splitlines() == [], so nothing will be logged in that case.
- for line, line_keepends in zip(s.splitlines(), s.splitlines(keepends=True)):
- # Simplify the later stages by translating all newlines into "\n".
- if line != line_keepends:
- line += "\n"
- while line:
- super().write(line[:MAX_LINE_LEN_CHARS])
- line = line[MAX_LINE_LEN_CHARS:]
+ raise TypeError(
+ f"write() argument must be str, not {type(s).__name__}")
+
+ # In case `s` is a str subclass that writes itself to stdout or stderr
+ # when we call its methods, convert it to an actual str.
+ s = str.__str__(s)
+
+ # We want to emit one log message per line wherever possible, so split
+ # the string into lines first. Note that "".splitlines() == [], so
+ # nothing will be logged for an empty string.
+ with self._lock:
+ for line in s.splitlines(keepends=True):
+ while line:
+ chunk = line[:MAX_CHARS_PER_WRITE]
+ line = line[MAX_CHARS_PER_WRITE:]
+ self._write_chunk(chunk)
+
return len(s)
+ # The size and behavior of TextIOWrapper's buffer is not part of its public
+ # API, so we handle buffering ourselves to avoid truncation.
+ def _write_chunk(self, s):
+ b = s.encode(self.encoding, self.errors)
+ if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+ self.flush()
+
+ self._pending_bytes.append(b)
+ self._pending_bytes_count += len(b)
+ if (
+ self.write_through
+ or b.endswith(b"\n")
+ or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+ ):
+ self.flush()
+
+ def flush(self):
+ with self._lock:
+ self.buffer.write(b"".join(self._pending_bytes))
+ self._pending_bytes.clear()
+ self._pending_bytes_count = 0
+
+ # Since this is a line-based logging system, line buffering cannot be turned
+ # off, i.e. a newline always causes a flush.
+ @property
+ def line_buffering(self):
+ return True
+
class BinaryLogStream(io.RawIOBase):
- def __init__(self, text_stream, level, tag):
- self.text_stream = text_stream
- self.level = level
+ def __init__(self, prio, tag, fileno=None):
+ self.prio = prio
self.tag = tag
+ self._fileno = fileno
def __repr__(self):
return f""
@@ -65,11 +117,67 @@ def writable(self):
return True
def write(self, b):
- # This form of `str` throws a TypeError on any non-bytes-like object, as opposed
- # to the AttributeError we would probably get from trying to call `encode`.
- s = str(b, self.text_stream.encoding, self.text_stream.errors)
+ if type(b) is not bytes:
+ try:
+ b = bytes(memoryview(b))
+ except TypeError:
+ raise TypeError(
+ f"write() argument must be bytes-like, not {type(b).__name__}"
+ ) from None
# Writing an empty string to the stream should have no effect.
- if s:
- Log.println(self.level, self.tag, s)
+ if b:
+ logcat.write(self.prio, self.tag, b)
return len(b)
+
+ # This is needed by the test suite --timeout option, which uses faulthandler.
+ def fileno(self):
+ if self._fileno is None:
+ raise io.UnsupportedOperation("fileno")
+ return self._fileno
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+ def __init__(self, android_log_write):
+ self.android_log_write = android_log_write
+ self._lock = RLock()
+ self._bucket_level = 0
+ self._prev_write_time = time()
+
+ def write(self, prio, tag, message):
+ # Encode null bytes using "modified UTF-8" to avoid them truncating the
+ # message.
+ message = message.replace(b"\x00", b"\xc0\x80")
+
+ with self._lock:
+ now = time()
+ self._bucket_level += (
+ (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+
+ # If the bucket level is still below zero, the clock must have gone
+ # backwards, so reset it to zero and continue.
+ self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
+ self._prev_write_time = now
+
+ self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+ if self._bucket_level < 0:
+ sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+ self.android_log_write(prio, tag.encode("UTF-8"), message)
diff --git a/product/runtime/src/main/python/java/conversion.pxi b/product/runtime/src/main/python/java/conversion.pxi
index ff051a9d42..d2216711f8 100644
--- a/product/runtime/src/main/python/java/conversion.pxi
+++ b/product/runtime/src/main/python/java/conversion.pxi
@@ -303,7 +303,11 @@ cpdef check_range_char(value):
cdef JNIRef p2j_string(JNIEnv *j_env, unicode s):
- utf16 = s.encode(JCHAR_ENCODING)
+ # Python strings can contain invalid surrogates, but Java strings cannot.
+ # "backslashreplace" would retain some information about the invalid character, but
+ # we don't know what context this string will be used in, so changing its length
+ # could cause much worse confusion.
+ utf16 = s.encode(JCHAR_ENCODING, errors="replace")
return LocalRef.adopt(j_env, j_env[0].NewString(
j_env, utf16, len(utf16)//2)) # 2 bytes/char for UTF-16.
diff --git a/product/runtime/src/main/python/java/import.pxi b/product/runtime/src/main/python/java/import.pxi
index 9fac6ef3a8..81655b22c8 100644
--- a/product/runtime/src/main/python/java/import.pxi
+++ b/product/runtime/src/main/python/java/import.pxi
@@ -65,19 +65,27 @@ def import_override(name, globals={}, locals={}, fromlist=None, level=0):
# only does this for the standard import system: see remove_import_frames in Python/import.c.)
def clean_exception(e):
tb = e.__traceback__
- while in_import_system(tb.tb_frame.f_code.co_filename):
+
+ tb_clean = []
+ while tb:
+ if not in_import_system(tb.tb_frame.f_code.co_filename):
+ tb_clean.append(tb)
tb = tb.tb_next
- if tb is None:
- # A file with a SyntaxError is not actually in the traceback, because it was never
- # compiled successfully.
- if isinstance(e, SyntaxError) and not in_import_system(e.filename):
- break
- else:
- # The exception originated within the import system, so return it untouched to
- # assist debugging.
- return e
- return e.with_traceback(tb)
+ # A file with a SyntaxError is not actually in the traceback, because it was never
+ # compiled successfully.
+ if tb_clean or isinstance(e, SyntaxError):
+ # Construct a traceback from the non-import-system frames only.
+ for i, tb in enumerate(tb_clean):
+ tb.tb_next = (
+ None if i + 1 == len(tb_clean)
+ else tb_clean[i + 1]
+ )
+ return e.with_traceback(tb_clean[0] if tb_clean else None)
+ else:
+ # All frames came from the import system, so return the exception untouched to
+ # assist debugging.
+ return e
def in_import_system(filename):
filename = filename.replace("\\", "/") # For .pyc files compiled on Windows.
diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_import.py b/product/runtime/src/test/python/chaquopy/test/android/test_import.py
index 9b104a63a3..fd1cca69dd 100644
--- a/product/runtime/src/test/python/chaquopy/test/android/test_import.py
+++ b/product/runtime/src/test/python/chaquopy/test/android/test_import.py
@@ -61,14 +61,18 @@ def test_bootstrap(self):
self.assertCountEqual([ABI], os.listdir(bn_dir))
stdlib_bootstrap_expected = {
- # This is the list from our minimum Python version. For why each of these
- # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt.
- "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so",
- "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so",
+ # For why each of these modules is needed, see BOOTSTRAP_NATIVE_STDLIB in
+ # PythonTasks.kt.
+ "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so",
+ "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so",
+ "mmap.so", "zlib.so",
}
if sys.version_info >= (3, 12):
stdlib_bootstrap_expected -= {"_sha512.so"}
stdlib_bootstrap_expected |= {"_sha2.so"}
+ if sys.version_info >= (3, 13):
+ stdlib_bootstrap_expected -= {"_sha2.so"}
+ stdlib_bootstrap_expected |= {"_opcode.so"}
for subdir, entries in [
(ABI, list(stdlib_bootstrap_expected)),
@@ -379,7 +383,7 @@ def check_module(self, mod_name, filename, cache_filename, *, is_package=False,
# Verify that the traceback builder can get source code from the loader in all contexts.
# (The "package1" test files are also used in TestImport.)
def test_exception(self):
- col_marker = r'( +\^+\n)?' # Column marker (Python >= 3.11)
+ col_marker = r'( +[~^]+\n)?' # Column marker (Python >= 3.11)
test_frame = (
fr' File "{asset_path(APP_ZIP)}/chaquopy/test/android/test_import.py", '
fr'line \d+, in test_exception\n'
@@ -426,11 +430,11 @@ def test_exception(self):
fr'line 1, in \n'
fr' from . import other_error # noqa: F401\n' +
col_marker +
- import_frame +
fr' File "{asset_path(APP_ZIP)}/package1/other_error.py", '
fr'line 1, in \n'
- fr' int\("hello"\)\n'
- fr"ValueError: invalid literal for int\(\) with base 10: 'hello'\n$")
+ fr' int\("hello"\)\n' +
+ col_marker +
+ r"ValueError: invalid literal for int\(\) with base 10: 'hello'\n$")
else:
self.fail()
diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py
index 155f72f30f..faa006dba7 100644
--- a/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py
+++ b/product/runtime/src/test/python/chaquopy/test/android/test_stdlib.py
@@ -2,6 +2,7 @@
from os.path import dirname, exists, join, realpath
import subprocess
import sys
+from unittest import skipIf
from warnings import catch_warnings, filterwarnings
from android.os import Build
@@ -61,6 +62,7 @@ def test_json(self):
self.assertTrue(encoder.c_make_encoder)
self.assertTrue(scanner.c_make_scanner)
+ @skipIf(sys.version_info >= (3, 13), "lib2to3 was removed in Python 3.13")
def test_lib2to3(self):
with catch_warnings():
for category in [DeprecationWarning, PendingDeprecationWarning]:
@@ -102,7 +104,14 @@ def square_slowly(x):
for name in ["Barrier", "BoundedSemaphore", "Condition", "Event", "Lock", "RLock",
"Semaphore"]:
cls = getattr(synchronize, name)
- with self.assertRaisesRegex(OSError, "This platform lacks a functioning sem_open"):
+ with self.assertRaisesRegex(
+ OSError,
+ (
+ "This platform lacks a functioning sem_open"
+ if sys.version_info < (3, 13)
+ else "No module named '_multiprocessing'"
+ )
+ ):
if name == "Barrier":
cls(1, ctx=ctx)
else:
@@ -135,9 +144,9 @@ def test_platform(self):
python_bits = platform.architecture()[0]
self.assertEqual(python_bits, "64bit" if ("64" in Build.CPU_ABI) else "32bit")
- # Requires sys.executable to exist.
- p = platform.platform()
- self.assertRegex(p, r"^Linux")
+ self.assertRegex(
+ platform.platform(),
+ r"^Linux" if sys.version_info < (3, 13) else r"^Android")
def test_select(self):
import select
@@ -166,15 +175,6 @@ def test_signal(self):
self.assertIsInstance(vs[0], enum.IntEnum)
self.assertEqual("", repr(vs[0]))
- def test_socket(self):
- import socket
- for name in ["if_nameindex", "if_nametoindex", "if_indextoname"]:
- for args in [[], ["whatever"]]:
- with self.assertRaisesRegex(
- OSError, "this function is not available in this build of Python"
- ):
- getattr(socket, name)(*args)
-
def test_sqlite(self):
import sqlite3
conn = sqlite3.connect(":memory:")
@@ -218,7 +218,9 @@ def test_sys(self):
for p in sys.path:
self.assertTrue(exists(p), p)
- self.assertRegex(sys.platform, r"^linux")
+ self.assertEqual(
+ sys.platform,
+ "linux" if sys.version_info < (3, 13) else "android")
self.assertNotIn("dirty", sys.version)
def test_sysconfig(self):
diff --git a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py
index 40bdf8b4eb..c1c234174d 100644
--- a/product/runtime/src/test/python/chaquopy/test/android/test_stream.py
+++ b/product/runtime/src/test/python/chaquopy/test/android/test_stream.py
@@ -1,27 +1,43 @@
-from contextlib import contextmanager
+# This file is based on Lib/test/test_android.py on the CPython main branch.
+
import ctypes.util
+import io
import queue
import re
import subprocess
import sys
+import unittest
+from array import array
+from contextlib import ExitStack, contextmanager
from threading import Thread
from time import time
+from unittest.mock import patch
+
+from importlib import import_module
+stream_mod_name = (
+ "java.android.stream" if sys.version_info < (3, 13) else "_android_support")
+TextLogStream = import_module(stream_mod_name).TextLogStream
+
+from ..test_utils import API_LEVEL as api_level, FilterWarningsCase
-from android.util import Log
-from ..test_utils import API_LEVEL, FilterWarningsCase
+# (name, level, fileno)
+STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)]
+# Was `from test.support import LOOPBACK_TIMEOUT`, but Chaquopy doesn't include the
+# stdlib `test` module.
+LOOPBACK_TIMEOUT = 10.0
redirected_native = False
class TestAndroidOutput(FilterWarningsCase):
-
maxDiff = None
def setUp(self):
self.logcat_process = subprocess.Popen(
- ["logcat", "-v", "tag"], stdout=subprocess.PIPE, errors="backslashreplace"
+ ["logcat", "-v", "tag"], stdout=subprocess.PIPE,
+ errors="backslashreplace"
)
self.logcat_queue = queue.Queue()
@@ -29,11 +45,19 @@ def logcat_thread():
for line in self.logcat_process.stdout:
self.logcat_queue.put(line.rstrip("\n"))
self.logcat_process.stdout.close()
- Thread(target=logcat_thread).start()
+ self.logcat_thread = Thread(target=logcat_thread)
+ self.logcat_thread.start()
+
+ from ctypes import CDLL, c_char_p, c_int
+ android_log_write = getattr(CDLL("liblog.so"), "__android_log_write")
+ android_log_write.argtypes = (c_int, c_char_p, c_char_p)
+ ANDROID_LOG_INFO = 4
- tag, start_marker = "python.test", f"{self.id()} {time()}"
- Log.i(tag, start_marker)
- self.assert_log("I", tag, start_marker, skip=True, timeout=5)
+ # Separate tests using a marker line with a different tag.
+ tag, message = "python.test", f"{self.id()} {time()}"
+ android_log_write(
+ ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8"))
+ self.assert_log("I", tag, message, skip=True, timeout=5)
def assert_logs(self, level, tag, expected, **kwargs):
for line in expected:
@@ -57,7 +81,8 @@ def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5):
def tearDown(self):
self.logcat_process.terminate()
- self.logcat_process.wait(0.1)
+ self.logcat_process.wait(LOOPBACK_TIMEOUT)
+ self.logcat_thread.join(LOOPBACK_TIMEOUT)
@contextmanager
def unbuffered(self, stream):
@@ -67,9 +92,32 @@ def unbuffered(self, stream):
finally:
stream.reconfigure(write_through=False)
+ # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't
+ # test them directly. Detect this mode and use some temporary streams with
+ # the same properties.
+ def stream_context(self, stream_name, level):
+ # https://developer.android.com/ndk/reference/group/logging
+ prio = {"I": 4, "W": 5}[level]
+
+ stack = ExitStack()
+ stack.enter_context(self.subTest(stream_name))
+ stream = getattr(sys, stream_name)
+ native_stream = getattr(sys, f"__{stream_name}__")
+ if isinstance(stream, io.StringIO):
+ stack.enter_context(
+ patch(
+ f"sys.{stream_name}",
+ TextLogStream(
+ prio, f"python.{stream_name}", native_stream.fileno(),
+ errors="backslashreplace"
+ ),
+ )
+ )
+ return stack
+
def test_str(self):
- for stream_name, level in [("stdout", "I"), ("stderr", "W")]:
- with self.subTest(stream=stream_name):
+ for stream_name, level, fileno in STREAM_INFO:
+ with self.stream_context(stream_name, level):
stream = getattr(sys, stream_name)
original_stream = getattr(sys, f"__{stream_name}__")
self.assertIsNot(stream, original_stream)
@@ -79,15 +127,18 @@ def test_str(self):
tag = f"python.{stream_name}"
self.assertIn(f"", repr(stream))
- self.assertTrue(stream.writable())
- self.assertFalse(stream.readable())
+ self.assertIs(stream.writable(), True)
+ self.assertIs(stream.readable(), False)
+ self.assertEqual(stream.fileno(), fileno)
self.assertEqual("UTF-8", stream.encoding)
self.assertEqual("backslashreplace", stream.errors)
- self.assertTrue(stream.line_buffering)
- self.assertFalse(stream.write_through)
+ self.assertIs(stream.line_buffering, True)
+ self.assertIs(stream.write_through, False)
- def write(s, lines=None):
- self.assertEqual(len(s), stream.write(s))
+ def write(s, lines=None, *, write_len=None):
+ if write_len is None:
+ write_len = len(s)
+ self.assertEqual(write_len, stream.write(s))
if lines is None:
lines = [s]
self.assert_logs(level, tag, lines)
@@ -109,14 +160,20 @@ def write(s, lines=None):
# Non-BMP emoji
write("\U0001f600")
+ # Non-encodable surrogates
+ write("\ud800\udc00", [r"\ud800\udc00"])
+
+ # Code used by surrogateescape (which isn't enabled here)
+ write("\udc80", [r"\udc80"])
+
# Null characters are logged using "modified UTF-8".
write("\u0000", [r"\xc0\x80"])
write("a\u0000", [r"a\xc0\x80"])
write("\u0000b", [r"\xc0\x80b"])
write("a\u0000b", [r"a\xc0\x80b"])
- # Multi-line messages. Avoid identical consecutive lines, as they may
- # activate "chatty" filtering and break the tests.
+ # Multi-line messages. Avoid identical consecutive lines, as
+ # they may activate "chatty" filtering and break the tests.
write("\nx", [""])
write("\na\n", ["x", "a"])
write("\n", [""])
@@ -127,6 +184,13 @@ def write(s, lines=None):
write("f\n\ng", ["exxf", ""])
write("\n", ["g"])
+ # Since this is a line-based logging system, line buffering
+ # cannot be turned off, i.e. a newline always causes a flush.
+ stream.reconfigure(line_buffering=False)
+ self.assertIs(stream.line_buffering, True)
+
+ # However, buffering can be turned off completely if you want a
+ # flush after every write.
with self.unbuffered(stream):
write("\nx", ["", "x"])
write("\na\n", ["", "a"])
@@ -143,10 +207,34 @@ def write(s, lines=None):
write("hello\r\nworld\r\n", ["hello", "world"])
write("\r\n", [""])
+ # Non-standard line separators should be preserved.
+ write("before form feed\x0cafter form feed\n",
+ ["before form feed\x0cafter form feed"])
+ write("before line separator\u2028after line separator\n",
+ ["before line separator\u2028after line separator"])
+
+ # String subclasses are accepted, but they should be converted
+ # to a standard str without calling any of their methods.
+ class CustomStr(str):
+ def splitlines(self, *args, **kwargs):
+ raise AssertionError()
+
+ def __len__(self):
+ raise AssertionError()
+
+ def __str__(self):
+ raise AssertionError()
+
+ write(CustomStr("custom\n"), ["custom"], write_len=7)
+
+ # Non-string classes are not accepted.
for obj in [b"", b"hello", None, 42]:
with self.subTest(obj=obj):
- with self.assertRaisesRegex(TypeError, fr"write\(\) argument must be "
- fr"str, not {type(obj).__name__}"):
+ with self.assertRaisesRegex(
+ TypeError,
+ fr"write\(\) argument must be str, not "
+ fr"{type(obj).__name__}"
+ ):
stream.write(obj)
# Manual flushing is supported.
@@ -158,38 +246,43 @@ def write(s, lines=None):
stream.flush()
self.assert_log(level, tag, "helloworld")
- # Long lines are split into blocks of 1000 *characters*, but TextIOWrapper
- # should then join them back together as much as possible without
- # exceeding 4000 UTF-8 *bytes*.
+ # Long lines are split into blocks of 1000 characters
+ # (MAX_CHARS_PER_WRITE in java/android/stream.py), but
+ # TextIOWrapper should then join them back together as much as
+ # possible without exceeding 4000 UTF-8 bytes
+ # (MAX_BYTES_PER_WRITE).
#
# ASCII (1 byte per character)
- write(("foobar" * 700) + "\n",
- [("foobar" * 666) + "foob", # 4000 bytes
- "ar" + ("foobar" * 33)]) # 200 bytes
+ write(("foobar" * 700) + "\n", # 4200 bytes in
+ [("foobar" * 666) + "foob", # 4000 bytes out
+ "ar" + ("foobar" * 33)]) # 200 bytes out
# "Full-width" digits 0-9 (3 bytes per character)
s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19"
- write((s * 150) + "\n",
- [s * 100, # 3000 bytes
- s * 50]) # 1500 bytes
+ write((s * 150) + "\n", # 4500 bytes in
+ [s * 100, # 3000 bytes out
+ s * 50]) # 1500 bytes out
s = "0123456789"
- write(s * 200, [])
- write(s * 150, [])
- write(s * 51, [s * 350]) # 3500 bytes
- write("\n", [s * 51]) # 510 bytes
+ write(s * 200, []) # 2000 bytes in
+ write(s * 150, []) # 1500 bytes in
+ write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out
+ write("\n", [s * 51]) # 0 bytes in, 510 bytes out
def test_bytes(self):
- for stream_name, level in [("stdout", "I"), ("stderr", "W")]:
- with self.subTest(stream=stream_name):
+ for stream_name, level, fileno in STREAM_INFO:
+ with self.stream_context(stream_name, level):
stream = getattr(sys, stream_name).buffer
tag = f"python.{stream_name}"
self.assertEqual(f"", repr(stream))
- self.assertTrue(stream.writable())
- self.assertFalse(stream.readable())
-
- def write(b, lines=None):
- self.assertEqual(len(b), stream.write(b))
+ self.assertIs(stream.writable(), True)
+ self.assertIs(stream.readable(), False)
+ self.assertEqual(stream.fileno(), fileno)
+
+ def write(b, lines=None, *, write_len=None):
+ if write_len is None:
+ write_len = len(b)
+ self.assertEqual(write_len, stream.write(b))
if lines is None:
lines = [b.decode()]
self.assert_logs(level, tag, lines)
@@ -210,7 +303,7 @@ def write(b, lines=None):
# Non-BMP emoji
write(b"\xf0\x9f\x98\x80")
- # Null characters are logged using "modified UTF-8".
+ # Null bytes are logged using "modified UTF-8".
write(b"\x00", [r"\xc0\x80"])
write(b"a\x00", [r"a\xc0\x80"])
write(b"\x00b", [r"\xc0\x80b"])
@@ -222,15 +315,17 @@ def write(b, lines=None):
write(b"\xffb", [r"\xffb"])
write(b"a\xffb", [r"a\xffb"])
- # Log entries containing newlines are shown differently by `logcat -v
- # tag`, `logcat -v long`, and Android Studio. We currently use `logcat -v
- # tag`, which shows each line as if it was a separate log entry, but
- # strips a single trailing newline.
+ # Log entries containing newlines are shown differently by
+ # `logcat -v tag`, `logcat -v long`, and Android Studio. We
+ # currently use `logcat -v tag`, which shows each line as if it
+ # was a separate log entry, but strips a single trailing
+ # newline.
#
- # On newer versions of Android, all three of the above tools (or maybe
- # Logcat itself) will also strip any number of leading newlines.
- write(b"\nx", ["", "x"] if API_LEVEL < 30 else ["x"])
- write(b"\na\n", ["", "a"] if API_LEVEL < 30 else ["a"])
+ # On newer versions of Android, all three of the above tools (or
+ # maybe Logcat itself) will also strip any number of leading
+ # newlines.
+ write(b"\nx", ["", "x"] if api_level < 30 else ["x"])
+ write(b"\na\n", ["", "a"] if api_level < 30 else ["a"])
write(b"\n", [""])
write(b"b\n", ["b"])
write(b"c\n\n", ["c", ""])
@@ -244,14 +339,41 @@ def write(b, lines=None):
write(b"hello\r\nworld\r\n", ["hello", "world"])
write(b"\r\n", [""])
+ # Other bytes-like objects are accepted.
+ write(bytearray(b"bytearray"))
+
+ mv = memoryview(b"memoryview")
+ write(mv, ["memoryview"]) # Continuous
+ write(mv[::2], ["mmrve"]) # Discontinuous
+
+ write(
+ # Android only supports little-endian architectures, so the
+ # bytes representation is as follows:
+ array("H", [
+ 0, # 00 00
+ 1, # 01 00
+ 65534, # FE FF
+ 65535, # FF FF
+ ]),
+
+ # After encoding null bytes with modified UTF-8, the only
+ # valid UTF-8 sequence is \x01. All other bytes are handled
+ # by backslashreplace.
+ ["\\xc0\\x80\\xc0\\x80"
+ "\x01\\xc0\\x80"
+ "\\xfe\\xff"
+ "\\xff\\xff"],
+ write_len=8,
+ )
+
+ # Non-bytes-like classes are not accepted.
for obj in ["", "hello", None, 42]:
with self.subTest(obj=obj):
- if isinstance(obj, str):
- message = r"decoding str is not supported"
- else:
- message = (fr"decoding to str: need a bytes-like object, "
- fr"{type(obj).__name__} found")
- with self.assertRaisesRegex(TypeError, message):
+ with self.assertRaisesRegex(
+ TypeError,
+ fr"write\(\) argument must be bytes-like, not "
+ fr"{type(obj).__name__}"
+ ):
stream.write(obj)
def test_native(self):
@@ -303,6 +425,83 @@ def write(b, lines=None):
write(b"Hello\nworld\n", ["Hello", "world"])
+class TestAndroidRateLimit(unittest.TestCase):
+ def test_rate_limit(self):
+ # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+ PER_MESSAGE_OVERHEAD = 28
+
+ # https://developer.android.com/ndk/reference/group/logging
+ ANDROID_LOG_DEBUG = 3
+
+ # To avoid flooding the test script output, use a different tag rather
+ # than stdout or stderr.
+ tag = "python.rate_limit"
+ stream = TextLogStream(ANDROID_LOG_DEBUG, tag)
+
+ # Make a test message which consumes 1 KB of the logcat buffer.
+ message = "Line {:03d} "
+ message += "." * (
+ 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0))
+ ) + "\n"
+
+ # To avoid depending on the performance of the test device, we mock the
+ # passage of time.
+ mock_now = time()
+
+ def mock_time():
+ # Avoid division by zero by simulating a small delay.
+ mock_sleep(0.0001)
+ return mock_now
+
+ def mock_sleep(duration):
+ nonlocal mock_now
+ mock_now += duration
+
+ # See java/android/stream.py. The default values of these parameters work
+ # well across a wide range of devices, but we'll use smaller values to
+ # ensure a quick and reliable test that doesn't flood the log too much.
+ MAX_KB_PER_SECOND = 100
+ BUCKET_KB = 10
+ with patch(f"{stream_mod_name}.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), \
+ patch(f"{stream_mod_name}.BUCKET_SIZE", BUCKET_KB * 1024), \
+ patch(f"{stream_mod_name}.sleep", mock_sleep), \
+ patch(f"{stream_mod_name}.time", mock_time):
+ # Make sure the token bucket is full.
+ stream.write("Initial message to reset _prev_write_time")
+ mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND)
+ line_num = 0
+
+ # Write BUCKET_KB messages, and return the rate at which they were
+ # accepted in KB per second.
+ def write_bucketful():
+ nonlocal line_num
+ start = mock_time()
+ max_line_num = line_num + BUCKET_KB
+ while line_num < max_line_num:
+ stream.write(message.format(line_num))
+ line_num += 1
+ return BUCKET_KB / (mock_time() - start)
+
+ # The first bucketful should be written with minimal delay. The
+ # factor of 2 here is not arbitrary: it verifies that the system can
+ # write fast enough to empty the bucket within two bucketfuls, which
+ # the next part of the test depends on.
+ self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2)
+
+ # Write another bucketful to empty the token bucket completely.
+ write_bucketful()
+
+ # The next bucketful should be written at the rate limit.
+ self.assertAlmostEqual(
+ write_bucketful(), MAX_KB_PER_SECOND,
+ delta=MAX_KB_PER_SECOND * 0.1
+ )
+
+ # Once the token bucket refills, we should go back to full speed.
+ mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND)
+ self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2)
+
+
class TestAndroidInput(FilterWarningsCase):
def test_str(self):
diff --git a/product/runtime/src/test/python/chaquopy/test/test_conversion.py b/product/runtime/src/test/python/chaquopy/test/test_conversion.py
index cf0a26e556..1e8dc1687b 100644
--- a/product/runtime/src/test/python/chaquopy/test/test_conversion.py
+++ b/product/runtime/src/test/python/chaquopy/test/test_conversion.py
@@ -107,7 +107,7 @@ def verify_float(self, obj, name, exponent_bits, wrapper=None, allow_bool=False,
self.verify_value(obj, name, float("nan"), # NaN is unequal to everything including itself.
wrapper=wrapper,
- verify=lambda expected, actual: self.assertTrue(isnan(actual)))
+ verify=lambda _, actual: self.assertTrue(isnan(actual)))
# Wrapper type and bounds checks are tested in test_signatures.
self.verify_value(obj, name, True, context=self.conv_error_unless(allow_bool))
@@ -166,16 +166,23 @@ def verify_string(self, obj, name):
"\U00012345"]: # Non-BMP character
self.verify_value(obj, name, val)
+ # Invalid surrogates should be replaced by a question mark.
+ self.verify_value(
+ obj, name, "a\ud800b",
+ verify=lambda _, actual: self.assertEqual("a?b", actual)
+ )
+
# Byte strings cannot be implicitly converted to Java Strings. However, if the target
# type is Object, we will fall back on the default conversion of a Python iterable to
# Object[].
- context = verify = None
if name == "Object":
+ context = None
def verify(expected, actual):
self.assertEqual(expected, actual)
self.assertIsInstance(actual, jarray("Ljava/lang/Object;"))
else:
context = self.conv_error
+ verify = None
for val in [b"", b"h", b"hello"]:
self.verify_value(obj, name, val, context=context, verify=verify)
diff --git a/release/README.md b/release/README.md
index 2e22abe63d..7ad138ad60 100644
--- a/release/README.md
+++ b/release/README.md
@@ -25,15 +25,13 @@ representative device under the following conditions:
Download the `demo` artifact from GitHub Actions, and unpack the APK from it.
-Install the APK and run the Java and Python unit tests on the following devices, with at
-least one device being a clean install, and at least one being an upgrade from the
-previous public release with the tests already run.
+Install the APK and run the Java and Python unit tests on all ABIs, with at least one
+device being each of the following:
-* x86 emulator with minSdkVersion
-* x86_64 emulator with minSdkVersion
-* x86_64 emulator with targetSdkVersion
-* Any armeabi-v7a device
-* Any arm64-v8a device
+* minSdk
+* targetSdk
+* A clean install
+* An upgrade from the previous public release, with the tests already run
Test all the UI elements of the app on both minSdkVersion and targetSdkVersion.
@@ -76,12 +74,12 @@ test on a corresponding device:
* armeabi-v7a (use a 32-bit device)
* arm64-v8a
-Set `abiFilters` to `"x86", "x86_64"` (this tests the multi-ABI case), and test on the
-following devices, with at least one being a clean install:
+Set `abiFilters` to `"x86", "x86_64"` (this tests the multi-ABI case), and test on those
+ABIs, with at least one device being each of the following:
-* x86 emulator with minSdkVersion
-* x86_64 emulator with minSdkVersion (TensorFlow is expected to fail because of #669)
-* x86_64 emulator with targetSdkVersion
+* minSdk
+* targetSdk
+* A clean install
## Public release
diff --git a/server/pypi/README-internal.md b/server/pypi/README-internal.md
index d134f4284f..fb70d8fda5 100644
--- a/server/pypi/README-internal.md
+++ b/server/pypi/README-internal.md
@@ -22,14 +22,12 @@ which may be less stable. Include these packages in all the remaining tests.
Once everything's working on this ABI, save any edits in the package's `patches`
directory. Then run build-wheel for all other ABIs.
-Restore `abiFilters` to include all ABIs. Then test the app on the following devices, with
-at least one device being a clean install:
-
-* x86 emulator with minSdkVersion
-* x86_64 emulator with minSdkVersion
-* x86_64 emulator with targetSdkVersion
-* Any armeabi-v7a device
-* Any arm64-v8a device
+Restore `abiFilters` to include all ABIs, and test them all, with at least one device
+being each of the following:
+
+* minSdk
+* targetSdk
+* A clean install
Repeat the build and test on all other Python versions.
diff --git a/server/pypi/build-wheel.py b/server/pypi/build-wheel.py
index fbb6c6ce50..12ad017e4e 100755
--- a/server/pypi/build-wheel.py
+++ b/server/pypi/build-wheel.py
@@ -38,10 +38,11 @@
# Libraries are grouped by minimum API level and listed under their SONAMEs.
STANDARD_LIBS = [
# Android native APIs (https://developer.android.com/ndk/guides/stable_apis)
- (16, ["libandroid.so", "libc.so", "libdl.so", "libEGL.so", "libGLESv1_CM.so", "libGLESv2.so",
- "libjnigraphics.so", "liblog.so", "libm.so", "libOpenMAXAL.so", "libOpenSLES.so",
- "libz.so"]),
+ (16, ["libandroid.so", "libc.so", "libdl.so", "libEGL.so", "libGLESv1_CM.so",
+ "libGLESv2.so", "libjnigraphics.so", "liblog.so", "libm.so",
+ "libOpenMAXAL.so", "libOpenSLES.so", "libz.so"]),
(21, ["libmediandk.so"]),
+ (24, ["libcamera2ndk.so", "libvulkan.so"]),
# Chaquopy-provided libraries
(0, ["libcrypto_chaquopy.so", "libsqlite3_chaquopy.so", "libssl_chaquopy.so"]),
@@ -233,10 +234,25 @@ def find_target(self):
raise ERROR
target_dir = abspath(f"{PYPI_DIR}/../../maven/com/chaquo/python/target")
- versions = [ver for ver in os.listdir(target_dir) if ver.startswith(self.python)]
+ versions = [
+ ver for ver in os.listdir(target_dir)
+ if ver.startswith(f"{self.python}.")]
if not versions:
raise CommandError(f"Can't find Python {self.python} in {target_dir}")
- max_ver = max(versions, key=lambda ver: [int(x) for x in re.split(r"[.-]", ver)])
+
+ def version_key(ver):
+ # Use the same "releaselevel" notation as sys.version_info so it sorts in
+ # the correct order.
+ groups = list(
+ re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:([a-z]+)(\d+))?-(\d+)", ver).groups())
+ groups[3] = {
+ "a": "alpha", "b": "beta", "rc": "candidate", None: "final"
+ }[groups[3]]
+ if groups[4] is None:
+ groups[4] = 0
+ return groups
+
+ max_ver = max(versions, key=version_key)
target_version_dir = f"{target_dir}/{max_ver}"
zips = glob(f"{target_version_dir}/target-*-{self.abi}.zip")
@@ -459,9 +475,11 @@ def create_host_env(self):
# doesn't support it, and the links wouldn't survive on Windows anyway. So our library
# wheels include external shared libraries only under their SONAMEs, and we need to
# create links from the other names so the compiler can find them.
- SONAME_PATTERNS = [(r"^(lib.*)\.so\..*$", r"\1.so"),
- (r"^(lib.*?)\d+\.so$", r"\1.so"), # e.g. libpng
- (r"^(lib.*)_chaquopy\.so$", r"\1.so")] # e.g. libjpeg
+ SONAME_PATTERNS = [
+ (r"^(lib.*)\.so\..*$", r"\1.so"),
+ (r"^(lib.*?)[\d.]+\.so$", r"\1.so"), # e.g. libpng
+ (r"^(lib.*)_(chaquopy|python)\.so$", r"\1.so"), # e.g. libssl, libjpeg
+ ]
reqs_lib_dir = f"{self.host_env}/chaquopy/lib"
for filename in os.listdir(reqs_lib_dir):
for pattern, repl in SONAME_PATTERNS:
@@ -502,9 +520,15 @@ def build_with_pep517(self):
raise CommandError(e)
def get_common_env_vars(self, env):
+ # HOST is set by conda-forge's compiler activation scripts, e.g.
+ # https://github.com/conda-forge/clang-compiler-activation-feedstock/blob/main/recipe/activate-clang.sh
+ tool_prefix = ABIS[self.abi].tool_prefix
build_common_output = run(
- f"abi={self.abi}; api_level={self.api_level}; prefix={self.host_env}/chaquopy; "
- f". {PYPI_DIR}/../../target/build-common.sh; export",
+ f"export HOST={tool_prefix}; "
+ f"api_level={self.api_level}; "
+ f"PREFIX={self.host_env}/chaquopy; "
+ f". {PYPI_DIR}/../../target/android-env.sh; "
+ f"export",
shell=True, executable="bash", text=True, stdout=subprocess.PIPE
).stdout
for line in build_common_output.splitlines():
@@ -517,14 +541,9 @@ def get_common_env_vars(self, env):
if os.environ.get(key) != value:
env[key] = value
if not env:
- raise CommandError("Found no variables in build-common.sh output:\n"
+ raise CommandError("Found no variables in android-env.sh output:\n"
+ build_common_output)
- # This flag often catches errors in .so files which would otherwise be delayed
- # until runtime. (Some of the more complex build.sh scripts need to remove this, or
- # use it more selectively.)
- env["LDFLAGS"] += " -Wl,--no-undefined"
-
# Set all other variables used by distutils to prevent the host Python values (if
# any) from taking effect.
env["CPPFLAGS"] = ""
@@ -533,7 +552,6 @@ def get_common_env_vars(self, env):
compiler_vars = ["CC", "CXX", "LD"]
if "fortran" in self.non_python_build_reqs:
- tool_prefix = ABIS[self.abi].tool_prefix
toolchain = self.abi if self.abi in ["x86", "x86_64"] else tool_prefix
gfortran = f"{PYPI_DIR}/fortran/{toolchain}-4.9/bin/{tool_prefix}-gfortran"
if not exists(gfortran):
@@ -602,13 +620,9 @@ def env_vars(self):
# TODO: make everything use HOST instead, and remove this.
"CHAQUOPY_ABI": self.abi,
- # Set by conda-forge's compiler activation scripts, e.g.
- # https://github.com/conda-forge/clang-compiler-activation-feedstock/blob/main/recipe/activate-clang.sh
- "HOST": ABIS[self.abi].tool_prefix,
-
# conda-build variable names defined at
# https://docs.conda.io/projects/conda-build/en/latest/user-guide/environment-variables.html
- # CPU_COUNT is now in build-common.sh, so the target scripts can use it.
+ # CPU_COUNT is now in android-env.sh, so the target scripts can use it.
"PKG_BUILDNUM": self.build_num,
"PKG_NAME": self.package,
"PKG_VERSION": self.version,
@@ -627,7 +641,7 @@ def env_vars(self):
if self.verbose:
log("Environment set as follows:\n" +
"\n".join(f"export {key}={shlex.quote(value)}"
- for key, value in env.items()))
+ for key, value in sorted(env.items())))
original_env = {key: os.environ.get(key) for key in env}
os.environ.update(env)
@@ -720,6 +734,8 @@ def fix_wheel(self, in_filename):
if fixed_path != original_path:
run(f"mv {original_path} {fixed_path}")
+ # Strip before patching, otherwise the libraries may be corrupted:
+ # https://github.com/NixOS/patchelf/issues?q=is%3Aissue+strip+in%3Atitle
run(f"chmod +w {fixed_path}")
run(f"{os.environ['STRIP']} --strip-unneeded {fixed_path}")
diff --git a/server/pypi/packages/cryptography/meta.yaml b/server/pypi/packages/cryptography/meta.yaml
index 70aed87a2d..23167805c7 100644
--- a/server/pypi/packages/cryptography/meta.yaml
+++ b/server/pypi/packages/cryptography/meta.yaml
@@ -21,7 +21,8 @@ requirements:
# loads on startup.
#
# Instead, we link against OpenSSL 1.1 statically, as follows:
- # * Run the OpenSSL 1.1 build command from target/build-all.sh.
+ # * Download an OpenSSL 1.1 build from
+ # https://github.com/beeware/cpython-android-source-deps/releases.
# * For each combination of Python version and ABI, run build-with-static-openssl.sh
# in this directory.
#
diff --git a/server/pypi/pkgtest/app/build.gradle b/server/pypi/pkgtest/app/build.gradle
index 5e8f63a901..48df0b09a6 100644
--- a/server/pypi/pkgtest/app/build.gradle
+++ b/server/pypi/pkgtest/app/build.gradle
@@ -31,12 +31,12 @@ ext.TEST_PACKAGES = [
android {
namespace "com.chaquo.python.pkgtest3"
- compileSdk = 34
+ compileSdk = 35
defaultConfig {
applicationId "com.chaquo.python.pkgtest3"
minSdk = 24
- targetSdk = 34
+ targetSdk = 35
String pluginVersion = null
for (art in rootProject.buildscript.configurations.getByName("classpath")
@@ -57,9 +57,7 @@ android {
// To test packages, edit the following line to list their names, separated by
// spaces. Each name must be a subdirectory of PACKAGES_DIR.
def PACKAGES = ""
- if (!PACKAGES.isEmpty()) {
- addPackages(delegate, PACKAGES.trim().split(/\s+/).toList())
- }
+ addPackages(delegate, PACKAGES.trim().split(/\s+/).toList())
python {
version "3.8"
@@ -108,6 +106,10 @@ def addPackages(flavor, List packages) {
mkdir(outputDir)
String suiteSrc = ""
for (req in packages) {
+ // Splitting an empty string returns a list containing one empty string.
+ if (req.isEmpty()) {
+ continue
+ }
def pkgName = req.split("==")[0]
def pkgDir = file("${ext.PACKAGES_DIR}/$pkgName")
def testPaths = ["test.py", "test"]
diff --git a/target/README.md b/target/README.md
index a4cde39c89..bc6bdcb4d5 100644
--- a/target/README.md
+++ b/target/README.md
@@ -1,36 +1,28 @@
# Chaquopy target
-This directory contains scripts to build Python for Android.
+This directory contains scripts to build Python for Android. They can be run on Linux or
+macOS.
-## Supporting libraries
-
-Before building Python, the correct versions of the supporting libraries must already be
-present in the `prefix` subdirectory:
-
-* Bzip2, libffi and xz use static libraries, so you must build them yourself, using the
- commands from build-all.sh.
-* SQLite and OpenSSL use dynamic libraries, so you may either build them yourself in the
- same way, or get pre-built copies using the download-target.sh and unpackage-target.sh
- scripts, as shown in ci.yml.
-
-
-## Python
+## Building and testing
Update Common.java with the version you want to build, and the build number you want to
give it.
-Run build-and-package.sh, as shown in build-all.sh. This will create a release in the
-`maven` directory in the root of this repository. If the packaging phase fails, e.g.
-because the version already exists, then rather than doing the whole build again, you
-can re-run package-target.sh directly.
+Make sure the build machine has `pythonX.Y` on the PATH, where `X.Y` is the Python
+major.minor version you want to build (e.g. `3.13`).
+
+Run `python/build-and-package.sh X.Y`. This will create a release in the `maven`
+directory in the root of this repository. If the packaging phase fails, e.g. because the
+version already exists, then rather than doing the whole build again, you can re-run
+package-target.sh directly.
If this is a new major.minor version, do the "Adding a Python version" checklist below.
-Run the PythonVersion integration tests.
+Run the integration tests, starting with PythonVersion.
-Use the demo app to run the Python and Java unit tests on the full set of pre-release
-devices (see release/README.md).
+Temporarily change the Python version of the demo app, and run the Python and Java unit
+tests on the full set of pre-release devices (see release/README.md).
To publish the build, follow the "Public release" instructions in release/README.md.
Once a version has been published on Maven Central, it cannot be changed, so any fixes
@@ -39,16 +31,14 @@ must be released under a different build number.
## Adding a Python version
-* First add buildPython support for the same version (see gradle-plugin/README.md).
-* Add the version to Common.java.
-* Add the version to build-all.sh.
+* Add buildPython support for the same version (see gradle-plugin/README.md).
* In test_gradle_plugin.py, update the `PYTHON_VERSIONS` assertion.
* Update the `MAGIC` lists in test_gradle_plugin.py and pyc.py.
+* Add a directory under integration/data/PythonVersion.
* Update android.rst and versions.rst.
* Build any packages used by the demo app.
-* Temporarily change the Python version of the demo app, and run all tests.
-When building the other packages:
+When building wheels for other packages:
* For each package, in dependency order:
* Update to the current stable version, unless it's been updated recently, or updating
diff --git a/target/abi-to-host.sh b/target/abi-to-host.sh
new file mode 100644
index 0000000000..6e717aac28
--- /dev/null
+++ b/target/abi-to-host.sh
@@ -0,0 +1,18 @@
+case ${abi:?} in
+ armeabi-v7a)
+ HOST=arm-linux-androideabi
+ ;;
+ arm64-v8a)
+ HOST=aarch64-linux-android
+ ;;
+ x86)
+ HOST=i686-linux-android
+ ;;
+ x86_64)
+ HOST=x86_64-linux-android
+ ;;
+ *)
+ echo "Unknown ABI: '$abi'"
+ exit 1
+ ;;
+esac
diff --git a/target/android-env.sh b/target/android-env.sh
new file mode 100644
index 0000000000..31700e31f0
--- /dev/null
+++ b/target/android-env.sh
@@ -0,0 +1,93 @@
+# This script must be sourced with the following variables already set:
+: ${ANDROID_HOME:?} # Path to Android SDK
+: ${HOST:?} # GNU target triplet
+
+# You may also override the following:
+: ${api_level:=24} # Minimum Android API level the build will run on
+: ${PREFIX:-} # Path in which to find required libraries
+
+
+# Print all messages on stderr so they're visible when running within build-wheel.
+log() {
+ echo "$1" >&2
+}
+
+fail() {
+ log "$1"
+ exit 1
+}
+
+# When moving to a new version of the NDK, carefully review the following:
+#
+# * https://developer.android.com/ndk/downloads/revision_history
+#
+# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
+# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
+# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
+ndk_version=27.1.12297006
+
+ndk=$ANDROID_HOME/ndk/$ndk_version
+if ! [ -e $ndk ]; then
+ log "Installing NDK - this may take several minutes"
+ yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version"
+fi
+
+if [ $HOST = "arm-linux-androideabi" ]; then
+ clang_triplet=armv7a-linux-androideabi
+else
+ clang_triplet=$HOST
+fi
+
+# These variables are based on BuildSystemMaintainers.md above, and
+# $ndk/build/cmake/android.toolchain.cmake.
+toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*)
+export AR="$toolchain/bin/llvm-ar"
+export AS="$toolchain/bin/llvm-as"
+export CC="$toolchain/bin/${clang_triplet}${api_level}-clang"
+export CXX="${CC}++"
+export LD="$toolchain/bin/ld"
+export NM="$toolchain/bin/llvm-nm"
+export RANLIB="$toolchain/bin/llvm-ranlib"
+export READELF="$toolchain/bin/llvm-readelf"
+export STRIP="$toolchain/bin/llvm-strip"
+
+# The quotes make sure the wildcard in the `toolchain` assignment has been expanded.
+for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do
+ if ! [ -e "$path" ]; then
+ fail "$path does not exist"
+ fi
+done
+
+export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO"
+export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384"
+
+# Unlike Linux, Android does not implicitly use a dlopened library to resolve
+# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used
+# (https://github.com/android/ndk/issues/1244). So any library that fails to
+# build with this flag, would also fail to load at runtime.
+LDFLAGS="$LDFLAGS -Wl,--no-undefined"
+
+# Many packages get away with omitting -lm on Linux, but Android is stricter.
+LDFLAGS="$LDFLAGS -lm"
+
+# -mstackrealign is included where necessary in the clang launcher scripts which are
+# pointed to by $CC, so we don't need to include it here.
+if [ $HOST = "arm-linux-androideabi" ]; then
+ CFLAGS="$CFLAGS -march=armv7-a -mthumb"
+fi
+
+if [ -n "${PREFIX:-}" ]; then
+ abs_prefix=$(realpath $PREFIX)
+ CFLAGS="$CFLAGS -I$abs_prefix/include"
+ LDFLAGS="$LDFLAGS -L$abs_prefix/lib"
+
+ export PKG_CONFIG="pkg-config --define-prefix"
+ export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig"
+fi
+
+# Use the same variable name as conda-build
+if [ $(uname) = "Darwin" ]; then
+ export CPU_COUNT=$(sysctl -n hw.ncpu)
+else
+ export CPU_COUNT=$(nproc)
+fi
diff --git a/target/build-all.sh b/target/build-all.sh
deleted file mode 100755
index ad3850400d..0000000000
--- a/target/build-all.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-set -eu
-
-cd $(dirname $(realpath $0))
-
-echo "This script needs to be updated to use https://github.com/beeware/cpython-android-source-deps"
-exit 1
-
-# Build libraries shared by all Python versions.
-./for-each-abi.sh bzip2/build.sh 1.0.8
-./for-each-abi.sh libffi/build.sh 3.4.4
-./for-each-abi.sh sqlite/build.sh 2022 3390200
-./for-each-abi.sh xz/build.sh 5.4.5
-
-# Build all supported versions of Python, and generate `target` artifacts for Maven.
-#
-# For a given Python version, we can't change the OpenSSL major version after we've made
-# the first release, because that would break binary compatibility with our existing
-# builds of the `cryptography` package. Also, multiple OpenSSL versions can't coexist
-# within the same include directory, because they use the same header file names. So we
-# build each OpenSSL version immediately before all the Python versions that use it.
-
-./for-each-abi.sh openssl/build.sh 1.1.1s
-python/build-and-package.sh 3.8
-
-./for-each-abi.sh openssl/build.sh 3.0.5
-python/build-and-package.sh 3.9
-python/build-and-package.sh 3.10
-python/build-and-package.sh 3.11
-python/build-and-package.sh 3.12
diff --git a/target/build-common.sh b/target/build-common.sh
deleted file mode 100644
index 85266bdcc8..0000000000
--- a/target/build-common.sh
+++ /dev/null
@@ -1,104 +0,0 @@
-# This script must be sourced with the following variables already set:
-# * ANDROID_HOME: path to Android SDK
-# * prefix: path with `include` and `lib` subdirectories to add to CFLAGS and LDFLAGS.
-#
-# You may also override the following:
-: ${abi:=$(basename $prefix)}
-: ${api_level:=24} # Should match MIN_SDK_VERSION in Common.java.
-
-# Print all messages on stderr so they're visible when running within build-wheel.
-log() {
- echo "$1" >&2
-}
-
-fail() {
- log "$1"
- exit 1
-}
-
-# When moving to a new version of the NDK, carefully review the following:
-#
-# * The release notes (https://developer.android.com/ndk/downloads/revision_history)
-#
-# * https://android.googlesource.com/platform/ndk/+/ndk-release-rXX/docs/BuildSystemMaintainers.md,
-# where XX is the NDK version. Do a diff against the version you're upgrading from.
-#
-# * According to https://github.com/kivy/python-for-android/pull/2615, the mzakharo
-# build of gfortran is not compatible with NDK version 23, which is the version in
-# which they removed the GNU binutils.
-ndk_version=22.1.7171670 # See ndkDir in product/runtime/build.gradle.
-
-ndk=${ANDROID_HOME:?}/ndk/$ndk_version
-if ! [ -e $ndk ]; then
- log "Installing NDK: this may take several minutes"
- yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version"
-fi
-
-case $abi in
- armeabi-v7a)
- host_triplet=arm-linux-androideabi
- clang_triplet=armv7a-linux-androideabi
- ;;
- arm64-v8a)
- host_triplet=aarch64-linux-android
- ;;
- x86)
- host_triplet=i686-linux-android
- ;;
- x86_64)
- host_triplet=x86_64-linux-android
- ;;
- *)
- fail "Unknown ABI: '$abi'"
- ;;
-esac
-
-# These variables are based on BuildSystemMaintainers.md above, and
-# $ndk/build/cmake/android.toolchain.cmake.
-toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*)
-export AR="$toolchain/bin/llvm-ar"
-export AS="$toolchain/bin/$host_triplet-as"
-export CC="$toolchain/bin/${clang_triplet:-$host_triplet}$api_level-clang"
-export CXX="${CC}++"
-export LD="$toolchain/bin/ld"
-export NM="$toolchain/bin/llvm-nm"
-export RANLIB="$toolchain/bin/llvm-ranlib"
-export READELF="$toolchain/bin/llvm-readelf"
-export STRIP="$toolchain/bin/llvm-strip"
-
-# The quotes make sure the wildcard in the `toolchain` assignment has been expanded.
-for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do
- if ! [ -e "$path" ]; then
- fail "$path does not exist"
- fi
-done
-
-# Use -idirafter so that package-specified -I directories take priority. For example,
-# grpcio provides its own BoringSSL headers which must be used rather than our OpenSSL.
-export CFLAGS="-idirafter ${prefix:?}/include"
-export LDFLAGS="-L${prefix:?}/lib \
--Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libgcc_real.a -Wl,--exclude-libs,libunwind.a \
--Wl,--build-id=sha1 -Wl,--no-rosegment"
-
-# Many packages get away with omitting this on standard Linux, but Android is stricter.
-LDFLAGS+=" -lm"
-
-case $abi in
- armeabi-v7a)
- CFLAGS+=" -march=armv7-a -mthumb"
- ;;
- x86)
- # -mstackrealign is unnecessary because it's included in the clang launcher script
- # which is pointed to by $CC.
- ;;
-esac
-
-export PKG_CONFIG="pkg-config --define-prefix"
-export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig"
-
-# conda-build variable name
-if [ $(uname) = "Darwin" ]; then
- export CPU_COUNT=$(sysctl -n hw.ncpu)
-else
- export CPU_COUNT=$(nproc)
-fi
diff --git a/target/for-each-abi.sh b/target/for-each-abi.sh
deleted file mode 100755
index a05ce184ff..0000000000
--- a/target/for-each-abi.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-set -eu
-
-target_dir=$(dirname $(realpath $0))
-script=$(realpath ${1:?})
-
-shift
-for prefix in $target_dir/prefix/*; do
- $script $prefix "$@"
-done
diff --git a/target/list-versions.py b/target/list-versions.py
index 21a8eb5733..04667d65f8 100755
--- a/target/list-versions.py
+++ b/target/list-versions.py
@@ -11,6 +11,7 @@
mode_group.add_argument("--micro", action="store_true")
mode_group.add_argument("--build", action="store_true")
args = parser.parse_args()
+versions_seen = set()
product_dir = abspath(f"{dirname(__file__)}/../product")
lines = []
@@ -22,16 +23,24 @@
if match:
lines.append(match[1])
break
- else:
- match = re.search(r'PYTHON_VERSIONS.put\("(\d+)\.(\d+)\.(\d+)", "(\d+)"\)', line)
- if match:
- major, minor, micro, build = match.groups()
- version = f"{major}.{minor}"
- if args.micro or args.build:
- version += f".{micro}"
- if args.build:
- version += f"-{build}"
- lines.append(version)
+
+ elif "PYTHON_VERSIONS.put" in line:
+ if not (match := re.search(
+ r'PYTHON_VERSIONS.put\("(\d+)\.(\d+)\.(\w+)", "(\d+)"\)', line
+ )):
+ raise ValueError(f"couldn't parse {line!r}")
+ major, minor, micro, build = match.groups()
+
+ version = f"{major}.{minor}"
+ if version in versions_seen:
+ raise ValueError(f"more than one entry for Python {version}")
+ versions_seen.add(version)
+
+ if args.micro or args.build:
+ version += f".{micro}"
+ if args.build:
+ version += f"-{build}"
+ lines.append(version)
assert lines
print("\n".join(lines))
diff --git a/target/package-target.sh b/target/package-target.sh
index c883687d5a..687a630696 100755
--- a/target/package-target.sh
+++ b/target/package-target.sh
@@ -25,9 +25,14 @@ mkdir "$target_dir" # Fail if it already exists: we don't want to overwrite thi
target_dir=$(realpath $target_dir)
full_ver=$(basename $target_dir)
-short_ver=$(echo $full_ver | sed -E 's/^([0-9]+\.[0-9]+).*/\1/')
-target_prefix="$target_dir/target-$full_ver"
+version=$(echo $full_ver | sed 's/-.*//')
+read version_major version_minor version_micro < <(
+ echo $version | sed -E 's/^([0-9]+)\.([0-9]+)\.([0-9]+).*/\1 \2 \3/'
+)
+version_short=$version_major.$version_minor
+version_int=$(($version_major * 100 + $version_minor))
+target_prefix="$target_dir/target-$full_ver"
cat > "$target_prefix.pom" <
&2
+ exit 1
+fi
case $version_short in
3.8|3.9|3.10|3.11)
diff --git a/target/python/build.sh b/target/python/build.sh
index 3fc2ac1275..b10f031fb4 100755
--- a/target/python/build.sh
+++ b/target/python/build.sh
@@ -1,76 +1,131 @@
#!/bin/bash
-set -eu
+set -eu -o pipefail
recipe_dir=$(dirname $(realpath $0))
-prefix=$(realpath ${1:?})
+PREFIX=${1:?}
+mkdir -p "$PREFIX"
+PREFIX=$(realpath "$PREFIX")
+
version=${2:?}
-read version_major version_minor < <(echo $version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/')
+read version_major version_minor version_micro < <(
+ echo $version | sed -E 's/^([0-9]+)\.([0-9]+)\.([0-9]+).*/\1 \2 \3/'
+)
version_short=$version_major.$version_minor
+version_no_pre=$version_major.$version_minor.$version_micro
version_int=$(($version_major * 100 + $version_minor))
+abi=$(basename $PREFIX)
cd $recipe_dir
-. ../build-common.sh
+. ../abi-to-host.sh
+. ../android-env.sh
+# Download and unpack Python source code.
version_dir=$recipe_dir/build/$version
mkdir -p $version_dir
cd $version_dir
src_filename=Python-$version.tgz
-wget -c https://www.python.org/ftp/python/$version/$src_filename
+wget -c https://www.python.org/ftp/python/$version_no_pre/$src_filename
build_dir=$version_dir/$abi
rm -rf $build_dir
-mkdir $build_dir
-cd $build_dir
-tar -xf $version_dir/$src_filename
-cd $(basename $src_filename .tgz)
+tar -xf "$src_filename"
+mv "Python-$version" "$build_dir"
+cd "$build_dir"
-patches="soname"
+# Apply patches.
+patches=""
if [ $version_int -le 311 ]; then
patches+=" sysroot_paths"
fi
-if [ $version_int -ge 311 ]; then
- # Although this patch applies cleanly to 3.12, it no longer has the intended effect,
- # because the stdlib extension modules are now built using autoconf rather than
- # distutils. Replace it with the fix we upstreamed to 3.13.
- patches+=" python_for_build_deps_REMOVED"
+if [ $version_int -eq 311 ]; then
+ patches+=" python_for_build_deps"
+fi
+if [ $version_int -le 312 ]; then
+ patches+=" soname"
fi
-if [ $version_int -ge 312 ]; then
+if [ $version_int -eq 312 ]; then
patches+=" bldlibrary grp"
fi
+if [ $version_int -eq 313 ]; then
+ # TODO: remove this once it's merged upstream.
+ patches+=" 3.13_pending"
+fi
for name in $patches; do
- patch -p1 -i $recipe_dir/patches/$name.patch
+ patch_file="$recipe_dir/patches/$name.patch"
+ echo "$patch_file"
+ patch -p1 -i "$patch_file"
done
-# Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable.
-CFLAGS+=" -I$toolchain/sysroot/usr/include"
-LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$host_triplet/$api_level"
-
-# The configure script omits -fPIC on Android, because it was unnecessary on older versions of
-# the NDK (https://bugs.python.org/issue26851). But it's definitely necessary on the current
-# version, otherwise we get linker errors like "Parser/myreadline.o: relocation R_386_GOTOFF
-# against preemptible symbol PyOS_InputHook cannot be used when making a shared object".
-export CCSHARED="-fPIC"
-
-# Override some tests.
-cat > config.site < config.site <<-EOF
+ # Things that can't be autodetected when cross-compiling.
+ ac_cv_aligned_required=no # Default of "yes" changes hash function to FNV, which breaks Numba.
+ ac_cv_file__dev_ptmx=no
+ ac_cv_file__dev_ptc=no
+ EOF
+ export CONFIG_SITE=$(pwd)/config.site
-make -j $CPU_COUNT
-make install prefix=$prefix
+ configure_args="--host=$HOST --build=$(./config.guess) \
+ --enable-shared --without-ensurepip --with-openssl=$PREFIX"
+
+ # This prevents the "getaddrinfo bug" test, which can't be run when cross-compiling.
+ configure_args+=" --enable-ipv6"
+
+ # Some of the patches involve missing Makefile dependencies, which allowed extension
+ # modules to be built before libpython3.x.so in parallel builds. In case this happens
+ # again, make sure there's no libpython3.x.a, otherwise the modules may end up silently
+ # linking with that instead.
+ if [ $version_int -ge 310 ]; then
+ configure_args+=" --without-static-libpython"
+ fi
+
+ if [ $version_int -ge 311 ]; then
+ configure_args+=" --with-build-python=yes"
+ fi
+
+ ./configure $configure_args
+
+ make -j $CPU_COUNT
+ make install prefix=$PREFIX
+
+# Python 3.13 and later comes with an official Android build script.
+else
+ mkdir -p cross-build/build
+ ln -s "$(which python$version_short)" cross-build/build/python
+
+ Android/android.py configure-host "$HOST"
+ Android/android.py make-host "$HOST"
+ cp -a "cross-build/$HOST/prefix/"* "$PREFIX"
+fi
diff --git a/target/python/patches/3.13_pending.patch b/target/python/patches/3.13_pending.patch
new file mode 100644
index 0000000000..83ab8e7198
--- /dev/null
+++ b/target/python/patches/3.13_pending.patch
@@ -0,0 +1,39 @@
+diff --git a/Android/android-env.sh b/Android/android-env.sh
+index 93372e3fe1c..94712602a23 100644
+--- a/Android/android-env.sh
++++ b/Android/android-env.sh
+@@ -24,7 +24,7 @@ fail() {
+ # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
+ # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
+ # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
+-ndk_version=26.2.11394342
++ndk_version=27.1.12297006
+
+ ndk=$ANDROID_HOME/ndk/$ndk_version
+ if ! [ -e $ndk ]; then
+@@ -58,8 +58,8 @@ for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP";
+ fi
+ done
+
+-export CFLAGS=""
+-export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment"
++export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO"
++export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384"
+
+ # Unlike Linux, Android does not implicitly use a dlopened library to resolve
+ # relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used
+diff --git a/Android/android.py b/Android/android.py
+index 8696d9eaeca..b3ee449ba43 100755
+--- a/Android/android.py
++++ b/Android/android.py
+@@ -138,8 +138,8 @@ def make_build_python(context):
+
+ def unpack_deps(host):
+ deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
+- for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.15-0",
+- "sqlite-3.45.1-0", "xz-5.4.6-0"]:
++ for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
++ "sqlite-3.45.3-3", "xz-5.4.6-1"]:
+ filename = f"{name_ver}-{host}.tar.gz"
+ download(f"{deps_url}/{name_ver}/{filename}")
+ run(["tar", "-xf", filename])
diff --git a/target/python/patches/bldlibrary.patch b/target/python/patches/bldlibrary.patch
index 4dda79fe1c..72b0bac89b 100644
--- a/target/python/patches/bldlibrary.patch
+++ b/target/python/patches/bldlibrary.patch
@@ -1,44 +1,55 @@
---- Python-3.12.0-original/configure 2023-11-22 09:33:49
-+++ Python-3.12.0/configure 2023-11-22 10:13:05
-@@ -7476,6 +7476,7 @@
- case $ac_sys_system in
- CYGWIN*)
- LDLIBRARY='libpython$(LDVERSION).dll.a'
-+ BLDLIBRARY='-L. -lpython$(LDVERSION)'
- DLLLIBRARY='libpython$(LDVERSION).dll'
- ;;
- SunOS*)
-@@ -24374,7 +24375,7 @@
- # On Android and Cygwin the shared libraries must be linked with libpython.
+diff --git a/configure b/configure
+index 1c75810d9e8..d883a00d548 100755
+--- a/configure
++++ b/configure
+@@ -841,6 +841,7 @@ PY_ENABLE_SHARED
+ PLATLIBDIR
+ BINLIBDEST
+ LIBPYTHON
++MODULE_DEPS_SHARED
+ EXT_SUFFIX
+ ALT_SOABI
+ SOABI
+@@ -24402,12 +24403,17 @@ LDVERSION='$(VERSION)$(ABIFLAGS)'
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5
+ printf "%s\n" "$LDVERSION" >&6; }
+-# On Android and Cygwin the shared libraries must be linked with libpython.
++# Configure the flags and dependencies used when compiling shared modules.
++# Do not rename LIBPYTHON - it's accessed via sysconfig by package build
++# systems (e.g. Meson) to decide whether to link extension modules against
++# libpython.
++MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)'
++LIBPYTHON=''
+
++# On Android and Cygwin the shared libraries must be linked with libpython.
if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then
- LIBPYTHON="-lpython${VERSION}${ABIFLAGS}"
-+ LIBPYTHON="$BLDLIBRARY"
- else
- LIBPYTHON=''
+-else
+- LIBPYTHON=''
++ MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)"
++ LIBPYTHON="\$(BLDLIBRARY)"
fi
---- Python-3.12.0-original/Modules/makesetup 2023-10-02 12:48:14
-+++ Python-3.12.0/Modules/makesetup 2023-11-22 10:11:40
-@@ -86,18 +86,6 @@
- # Newline for sed i and a commands
- NL='\
- '
--
--# Setup to link with extra libraries when making shared extensions.
--# Currently, only Cygwin needs this baggage.
--case `uname -s` in
--CYGWIN*) if test $libdir = .
-- then
-- ExtraLibDir=.
-- else
-- ExtraLibDir='$(LIBPL)'
-- fi
-- ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";;
--esac
- # Main loop
- for i in ${*-Setup}
-@@ -286,7 +274,7 @@
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index 0e64ccc5c21..c4217424508 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -2797,7 +2797,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
+
+ # force rebuild when header file or module build flavor (static/shared) is changed
+ MODULE_DEPS_STATIC=Modules/config.c
+-MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS)
++MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@
+
+ MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h
+ MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h
+diff --git a/Modules/makesetup b/Modules/makesetup
+index f000c9cd673..3231044230e 100755
+--- a/Modules/makesetup
++++ b/Modules/makesetup
+@@ -286,7 +286,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
;;
esac
rule="$file: $objs"
diff --git a/target/python/patches/python_for_build_deps.patch b/target/python/patches/python_for_build_deps.patch
new file mode 100644
index 0000000000..820dd58dba
--- /dev/null
+++ b/target/python/patches/python_for_build_deps.patch
@@ -0,0 +1,16 @@
+--- a/Makefile.pre.in 2022-10-24 17:35:39.000000000 +0000
++++ b/Makefile.pre.in 2022-11-01 18:20:18.472102145 +0000
+@@ -292,7 +292,12 @@
+ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
+ # Single-platform builds depend on $(BUILDPYTHON). Cross builds use an
+ # external "build Python" and have an empty PYTHON_FOR_BUILD_DEPS.
+-PYTHON_FOR_BUILD_DEPS=@PYTHON_FOR_BUILD_DEPS@
++#
++# Chaquopy: Was PYTHON_FOR_BUILD_DEPS from the configure script, which is empty when
++# cross-compiling (https://github.com/python/cpython/pull/93977). But this means that in
++# parallel builds, the sharedmods target can start running before libpython is available
++# (https://github.com/beeware/briefcase-android-gradle-template/pull/55).
++PYTHON_FOR_BUILD_DEPS=$(LDLIBRARY)
+
+ # Single-platform builds use Programs/_freeze_module.c for bootstrapping and
+ # ./_bootstrap_python Programs/_freeze_module.py for remaining modules