Merge branch '3.13' into backport-a10152f-3.13

Author: colesbury

.github/workflows/build.yml

@@ -310,7 +310,7 @@ jobs:
         # Keep 1.1.1w in our list despite it being upstream EOL and otherwise
         # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
         # supported by important vendors such as AWS-LC.
-        openssl_ver: [1.1.1w, 3.0.15, 3.1.7, 3.2.3, 3.3.2]
+        openssl_ver: [1.1.1w, 3.0.18, 3.1.7, 3.2.6, 3.3.5]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -382,7 +382,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Build and test
-        run: ./Android/android.py ci ${{ matrix.arch }}-linux-android
+        run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android
 
   build-wasi:
     name: 'WASI'
@@ -399,7 +399,7 @@ jobs:
     needs: build-context
     if: needs.build-context.outputs.run-tests == 'true'
     env:
-      OPENSSL_VER: 3.0.15
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -518,7 +518,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
     env:
-      OPENSSL_VER: 3.0.15
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:

.github/workflows/reusable-ubuntu.yml

@@ -25,7 +25,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04, ubuntu-24.04-arm]
     env:
-      OPENSSL_VER: 3.0.15
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       TERM: linux
     steps:

Android/android.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import argparse
+import json
 import os
 import platform
 import re
@@ -184,10 +185,16 @@ def make_build_python(context):
     run(["make", "-j", str(os.cpu_count())])
 
 
+# To create new builds of these dependencies, usually all that's necessary is to
+# push a tag to the cpython-android-source-deps repository, and GitHub Actions
+# will do the rest.
+#
+# If you're a member of the Python core team, and you'd like to be able to push
+# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
 def unpack_deps(host, prefix_dir):
     os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4",
+    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0",
                      "sqlite-3.50.4-0", "xz-5.4.6-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
@@ -546,27 +553,33 @@ async def gradle_task(context):
         task_prefix = "connected"
         env["ANDROID_SERIAL"] = context.connected
 
-    if context.command:
-        mode = "-c"
-        module = context.command
-    else:
-        mode = "-m"
-        module = context.module or "test"
+    if context.ci_mode:
+        context.args[0:0] = [
+            # See _add_ci_python_opts in libregrtest/main.py.
+            "-W", "error", "-bb", "-E",
+
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
+            "-m", "test",
+            f"--{context.ci_mode}-ci", "--single-process", "--no-randomize"
+        ]
+
+    if not any(arg in context.args for arg in ["-c", "-m"]):
+        context.args[0:0] = ["-m", "test"]
 
     args = [
         gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
     ] + [
-        # Build-time properties
-        f"-Ppython.{name}={value}"
-        for name, value in [
-            ("sitePackages", context.site_packages), ("cwd", context.cwd)
-        ] if value
-    ] + [
-        # Runtime properties
-        f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}"
+        f"-P{name}={value}"
         for name, value in [
-            ("Mode", mode), ("Module", module), ("Args", join_command(context.args))
-        ] if value
+            ("python.sitePackages", context.site_packages),
+            ("python.cwd", context.cwd),
+            (
+                "android.testInstrumentationRunnerArguments.pythonArgs",
+                json.dumps(context.args),
+            ),
+        ]
+        if value
     ]
     if context.verbose >= 2:
         args.append("--info")
@@ -734,15 +747,14 @@ def ci(context):
     else:
         with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
             print("::group::Tests")
+
             # Prove the package is self-contained by using it to run the tests.
             shutil.unpack_archive(package_path, temp_dir)
-
-            # Randomization is disabled because order-dependent failures are
-            # much less likely to pass on a rerun in single-process mode.
-            launcher_args = ["--managed", "maxVersion", "-v"]
-            test_args = ["--fast-ci", "--single-process", "--no-randomize"]
+            launcher_args = [
+                "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci"
+            ]
             run(
-                ["./android.py", "test", *launcher_args, "--", *test_args],
+                ["./android.py", "test", *launcher_args],
                 cwd=temp_dir
             )
             print("::endgroup::")
@@ -825,25 +837,28 @@ def add_parser(*args, **kwargs):
     test.add_argument(
         "--cwd", metavar="DIR", type=abspath,
         help="Directory to copy as the app's working directory.")
-
-    mode_group = test.add_mutually_exclusive_group()
-    mode_group.add_argument(
-        "-c", dest="command", help="Execute the given Python code.")
-    mode_group.add_argument(
-        "-m", dest="module", help="Execute the module with the given name.")
-    test.epilog = (
-        "If neither -c nor -m are passed, the default is '-m test', which will "
-        "run Python's own test suite.")
     test.add_argument(
-        "args", nargs="*", help=f"Arguments to add to sys.argv. "
-        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.")
+        "args", nargs="*", help=f"Python command-line arguments. "
+        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. "
+        f"If neither -c nor -m are included, `-m test` will be prepended, "
+        f"which will run Python's own test suite.")
 
     # Package arguments.
     for subcommand in [package, ci]:
         subcommand.add_argument(
             "-g", action="store_true", default=False, dest="debug",
             help="Include debug information in package")
 
+    # CI arguments
+    for subcommand in [test, ci]:
+        group = subcommand.add_mutually_exclusive_group(required=subcommand is ci)
+        group.add_argument(
+            "--fast-ci", action="store_const", dest="ci_mode", const="fast",
+            help="Add test arguments for GitHub Actions")
+        group.add_argument(
+            "--slow-ci", action="store_const", dest="ci_mode", const="slow",
+            help="Add test arguments for buildbots")
+
     return parser.parse_args()
 
 

Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt

@@ -20,7 +20,7 @@ class PythonSuite {
             val status = PythonTestRunner(
                 InstrumentationRegistry.getInstrumentation().targetContext
             ).run(
-                InstrumentationRegistry.getArguments()
+                InstrumentationRegistry.getArguments().getString("pythonArgs")!!,
             )
             assertEquals(0, status)
         } finally {

Android/testbed/app/src/main/c/main_activity.c

@@ -3,6 +3,7 @@
 #include <jni.h>
 #include <pthread.h>
 #include <Python.h>
+#include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
@@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) {
         message);
 }
 
+static void throw_errno(JNIEnv *env, const char *error_prefix) {
+    char error_message[1024];
+    snprintf(error_message, sizeof(error_message),
+             "%s: %s", error_prefix, strerror(errno));
+    throw_runtime_exception(env, error_message);
+}
+
 
 // --- Stdio redirection ------------------------------------------------------
 
@@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
     for (StreamInfo *si = STREAMS; si->file; si++) {
         char *error_prefix;
         if ((error_prefix = redirect_stream(si))) {
-            char error_message[1024];
-            snprintf(error_message, sizeof(error_message),
-                     "%s: %s", error_prefix, strerror(errno));
-            throw_runtime_exception(env, error_message);
+            throw_errno(env, error_prefix);
             return;
         }
     }
@@ -107,41 +112,86 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
 
 // --- Python initialization ---------------------------------------------------
 
-static PyStatus set_config_string(
-    JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value
-) {
-    const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL);
-    PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8);
-    (*env)->ReleaseStringUTFChars(env, value, value_utf8);
-    return status;
+static char *init_signals() {
+    // Some tests use SIGUSR1, but that's blocked by default in an Android app in
+    // order to make it available to `sigwait` in the Signal Catcher thread.
+    // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc).
+    // That thread's functionality is only useful for debugging the JVM, so disabling
+    // it should not weaken the tests.
+    //
+    // There's no safe way of stopping the thread completely (#123982), but simply
+    // unblocking SIGUSR1 is enough to fix most tests.
+    //
+    // However, in tests that generate multiple different signals in quick
+    // succession, it's possible for SIGUSR1 to arrive while the main thread is busy
+    // running the C-level handler for a different signal. In that case, the SIGUSR1
+    // may be sent to the Signal Catcher thread instead, which will generate a log
+    // message containing the text "reacting to signal".
+    //
+    // Such tests may need to be changed in one of the following ways:
+    //   * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in
+    //     test_signal.py).
+    //   * Send the signal to a specific thread rather than the whole process (e.g.
+    //     test_signals in test_threadsignals.py.
+    sigset_t set;
+    if (sigemptyset(&set)) {
+        return "sigemptyset";
+    }
+    if (sigaddset(&set, SIGUSR1)) {
+        return "sigaddset";
+    }
+    if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) {
+        return "pthread_sigmask";
+    }
+    return NULL;
 }
 
 static void throw_status(JNIEnv *env, PyStatus status) {
     throw_runtime_exception(env, status.err_msg ? status.err_msg : "");
 }
 
 JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython(
-    JNIEnv *env, jobject obj, jstring home, jstring runModule
+    JNIEnv *env, jobject obj, jstring home, jarray args
 ) {
+    const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL);
+    char cwd[PATH_MAX];
+    snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd");
+    if (chdir(cwd)) {
+        throw_errno(env, "chdir");
+        return 1;
+    }
+
+    char *error_prefix;
+    if ((error_prefix = init_signals())) {
+        throw_errno(env, error_prefix);
+        return 1;
+    }
+
     PyConfig config;
     PyStatus status;
-    PyConfig_InitIsolatedConfig(&config);
+    PyConfig_InitPythonConfig(&config);
 
-    status = set_config_string(env, &config, &config.home, home);
-    if (PyStatus_Exception(status)) {
+    jsize argc = (*env)->GetArrayLength(env, args);
+    const char *argv[argc + 1];
+    for (int i = 0; i < argc; i++) {
+        jobject arg = (*env)->GetObjectArrayElement(env, args, i);
+        argv[i] = (*env)->GetStringUTFChars(env, arg, NULL);
+    }
+    argv[argc] = NULL;
+
+    // PyConfig_SetBytesArgv "must be called before other methods, since the
+    // preinitialization configuration depends on command line arguments"
+    if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) {
         throw_status(env, status);
         return 1;
     }
 
-    status = set_config_string(env, &config, &config.run_module, runModule);
+    status = PyConfig_SetBytesString(&config, &config.home, home_utf8);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);
         return 1;
     }
 
-    // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored.
-    config.install_signal_handlers = 1;
-
     status = Py_InitializeFromConfig(&config);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);