Merge remote-tracking branch 'upstream/main' into mmap-flush-flags

Author: aisk

.github/CODEOWNERS

@@ -86,6 +86,10 @@ Modules/makesetup                @erlend-aasland @AA-Turner @emmatyping
 Modules/Setup*                   @erlend-aasland @AA-Turner @emmatyping
 Tools/build/regen-configure.sh   @AA-Turner
 
+# generate-build-details
+Tools/build/generate-build-details.py   @FFY00
+Lib/test/test_build_details.py          @FFY00
+
 
 # ----------------------------------------------------------------------------
 # Documentation

.github/workflows/build.yml

@@ -202,23 +202,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Cirrus and macos-14 are M1, macos-15-intel is default GHA Intel.
-        # macOS 13 only runs tests against the GIL-enabled CPython.
-        # Cirrus used for upstream, macos-14 for forks.
+        # macos-14 is M1, macos-15-intel is Intel.
+        # macos-15-intel only runs tests against the GIL-enabled CPython.
         os:
-        - ghcr.io/cirruslabs/macos-runner:sonoma
         - macos-14
         - macos-15-intel
-        is-fork:  # only used for the exclusion trick
-        - ${{ github.repository_owner != 'python' }}
         free-threading:
         - false
         - true
         exclude:
-        - os: ghcr.io/cirruslabs/macos-runner:sonoma
-          is-fork: true
-        - os: macos-14
-          is-fork: false
         - os: macos-15-intel
           free-threading: true
     uses: ./.github/workflows/reusable-macos.yml
@@ -273,7 +265,7 @@ jobs:
         # Keep 1.1.1w in our list despite it being upstream EOL and otherwise
         # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
         # supported by important vendors such as AWS-LC.
-        openssl_ver: [1.1.1w, 3.0.17, 3.2.5, 3.3.4, 3.4.2, 3.5.2]
+        openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4]
         # See Tools/ssl/make_ssl_data.py for notes on adding a new version
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -409,9 +401,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Use the same runs-on configuration as build-macos and build-ubuntu.
           - arch: aarch64
-            runs-on: ${{ github.repository_owner == 'python' && 'ghcr.io/cirruslabs/macos-runner:sonoma' || 'macos-14' }}
+            runs-on: macos-14
           - arch: x86_64
             runs-on: ubuntu-24.04
 
@@ -421,7 +412,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Build and test
-        run: ./Android/android.py ci ${{ matrix.arch }}-linux-android
+        run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android
 
   build-wasi:
     name: 'WASI'
@@ -438,7 +429,7 @@ jobs:
     needs: build-context
     if: needs.build-context.outputs.run-tests == 'true'
     env:
-      OPENSSL_VER: 3.0.16
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -558,7 +549,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
     env:
-      OPENSSL_VER: 3.0.16
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:

.github/workflows/reusable-ubuntu.yml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     timeout-minutes: 60
     env:
-      OPENSSL_VER: 3.0.15
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       TERM: linux
     steps:

.pre-commit-config.yaml

@@ -26,6 +26,10 @@ repos:
         name: Run Ruff (lint) on Tools/peg_generator/
         args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml]
         files: ^Tools/peg_generator/
+      - id: ruff-check
+        name: Run Ruff (lint) on Tools/wasm/
+        args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml]
+        files: ^Tools/wasm/
       - id: ruff-format
         name: Run Ruff (format) on Doc/
         args: [--check]
@@ -34,6 +38,10 @@ repos:
         name: Run Ruff (format) on Tools/build/check_warnings.py
         args: [--check, --config=Tools/build/.ruff.toml]
         files: ^Tools/build/check_warnings.py
+      - id: ruff-format
+        name: Run Ruff (format) on Tools/wasm/
+        args: [--check, --config=Tools/wasm/.ruff.toml]
+        files: ^Tools/wasm/
 
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 25.9.0

Android/android.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import argparse
+import json
 import os
 import platform
 import re
@@ -184,10 +185,16 @@ def make_build_python(context):
     run(["make", "-j", str(os.cpu_count())])
 
 
+# To create new builds of these dependencies, usually all that's necessary is to
+# push a tag to the cpython-android-source-deps repository, and GitHub Actions
+# will do the rest.
+#
+# If you're a member of the Python core team, and you'd like to be able to push
+# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
 def unpack_deps(host, prefix_dir):
     os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4",
+    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0",
                      "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
@@ -546,27 +553,33 @@ async def gradle_task(context):
         task_prefix = "connected"
         env["ANDROID_SERIAL"] = context.connected
 
-    if context.command:
-        mode = "-c"
-        module = context.command
-    else:
-        mode = "-m"
-        module = context.module or "test"
+    if context.ci_mode:
+        context.args[0:0] = [
+            # See _add_ci_python_opts in libregrtest/main.py.
+            "-W", "error", "-bb", "-E",
+
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
+            "-m", "test",
+            f"--{context.ci_mode}-ci", "--single-process", "--no-randomize"
+        ]
+
+    if not any(arg in context.args for arg in ["-c", "-m"]):
+        context.args[0:0] = ["-m", "test"]
 
     args = [
         gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
     ] + [
-        # Build-time properties
-        f"-Ppython.{name}={value}"
-        for name, value in [
-            ("sitePackages", context.site_packages), ("cwd", context.cwd)
-        ] if value
-    ] + [
-        # Runtime properties
-        f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}"
+        f"-P{name}={value}"
         for name, value in [
-            ("Mode", mode), ("Module", module), ("Args", join_command(context.args))
-        ] if value
+            ("python.sitePackages", context.site_packages),
+            ("python.cwd", context.cwd),
+            (
+                "android.testInstrumentationRunnerArguments.pythonArgs",
+                json.dumps(context.args),
+            ),
+        ]
+        if value
     ]
     if context.verbose >= 2:
         args.append("--info")
@@ -734,15 +747,14 @@ def ci(context):
     else:
         with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
             print("::group::Tests")
+
             # Prove the package is self-contained by using it to run the tests.
             shutil.unpack_archive(package_path, temp_dir)
-
-            # Randomization is disabled because order-dependent failures are
-            # much less likely to pass on a rerun in single-process mode.
-            launcher_args = ["--managed", "maxVersion", "-v"]
-            test_args = ["--fast-ci", "--single-process", "--no-randomize"]
+            launcher_args = [
+                "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci"
+            ]
             run(
-                ["./android.py", "test", *launcher_args, "--", *test_args],
+                ["./android.py", "test", *launcher_args],
                 cwd=temp_dir
             )
             print("::endgroup::")
@@ -825,25 +837,28 @@ def add_parser(*args, **kwargs):
     test.add_argument(
         "--cwd", metavar="DIR", type=abspath,
         help="Directory to copy as the app's working directory.")
-
-    mode_group = test.add_mutually_exclusive_group()
-    mode_group.add_argument(
-        "-c", dest="command", help="Execute the given Python code.")
-    mode_group.add_argument(
-        "-m", dest="module", help="Execute the module with the given name.")
-    test.epilog = (
-        "If neither -c nor -m are passed, the default is '-m test', which will "
-        "run Python's own test suite.")
     test.add_argument(
-        "args", nargs="*", help=f"Arguments to add to sys.argv. "
-        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.")
+        "args", nargs="*", help=f"Python command-line arguments. "
+        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. "
+        f"If neither -c nor -m are included, `-m test` will be prepended, "
+        f"which will run Python's own test suite.")
 
     # Package arguments.
     for subcommand in [package, ci]:
         subcommand.add_argument(
             "-g", action="store_true", default=False, dest="debug",
             help="Include debug information in package")
 
+    # CI arguments
+    for subcommand in [test, ci]:
+        group = subcommand.add_mutually_exclusive_group(required=subcommand is ci)
+        group.add_argument(
+            "--fast-ci", action="store_const", dest="ci_mode", const="fast",
+            help="Add test arguments for GitHub Actions")
+        group.add_argument(
+            "--slow-ci", action="store_const", dest="ci_mode", const="slow",
+            help="Add test arguments for buildbots")
+
     return parser.parse_args()
 
 

Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt

@@ -20,7 +20,7 @@ class PythonSuite {
             val status = PythonTestRunner(
                 InstrumentationRegistry.getInstrumentation().targetContext
             ).run(
-                InstrumentationRegistry.getArguments()
+                InstrumentationRegistry.getArguments().getString("pythonArgs")!!,
             )
             assertEquals(0, status)
         } finally {

Android/testbed/app/src/main/c/main_activity.c

@@ -3,6 +3,7 @@
 #include <jni.h>
 #include <pthread.h>
 #include <Python.h>
+#include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
@@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) {
         message);
 }
 
+static void throw_errno(JNIEnv *env, const char *error_prefix) {
+    char error_message[1024];
+    snprintf(error_message, sizeof(error_message),
+             "%s: %s", error_prefix, strerror(errno));
+    throw_runtime_exception(env, error_message);
+}
+
 
 // --- Stdio redirection ------------------------------------------------------
 
@@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
     for (StreamInfo *si = STREAMS; si->file; si++) {
         char *error_prefix;
         if ((error_prefix = redirect_stream(si))) {
-            char error_message[1024];
-            snprintf(error_message, sizeof(error_message),
-                     "%s: %s", error_prefix, strerror(errno));
-            throw_runtime_exception(env, error_message);
+            throw_errno(env, error_prefix);
             return;
         }
     }
@@ -107,41 +112,86 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
 
 // --- Python initialization ---------------------------------------------------
 
-static PyStatus set_config_string(
-    JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value
-) {
-    const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL);
-    PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8);
-    (*env)->ReleaseStringUTFChars(env, value, value_utf8);
-    return status;
+static char *init_signals() {
+    // Some tests use SIGUSR1, but that's blocked by default in an Android app in
+    // order to make it available to `sigwait` in the Signal Catcher thread.
+    // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc).
+    // That thread's functionality is only useful for debugging the JVM, so disabling
+    // it should not weaken the tests.
+    //
+    // There's no safe way of stopping the thread completely (#123982), but simply
+    // unblocking SIGUSR1 is enough to fix most tests.
+    //
+    // However, in tests that generate multiple different signals in quick
+    // succession, it's possible for SIGUSR1 to arrive while the main thread is busy
+    // running the C-level handler for a different signal. In that case, the SIGUSR1
+    // may be sent to the Signal Catcher thread instead, which will generate a log
+    // message containing the text "reacting to signal".
+    //
+    // Such tests may need to be changed in one of the following ways:
+    //   * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in
+    //     test_signal.py).
+    //   * Send the signal to a specific thread rather than the whole process (e.g.
+    //     test_signals in test_threadsignals.py.
+    sigset_t set;
+    if (sigemptyset(&set)) {
+        return "sigemptyset";
+    }
+    if (sigaddset(&set, SIGUSR1)) {
+        return "sigaddset";
+    }
+    if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) {
+        return "pthread_sigmask";
+    }
+    return NULL;
 }
 
 static void throw_status(JNIEnv *env, PyStatus status) {
     throw_runtime_exception(env, status.err_msg ? status.err_msg : "");
 }
 
 JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython(
-    JNIEnv *env, jobject obj, jstring home, jstring runModule
+    JNIEnv *env, jobject obj, jstring home, jarray args
 ) {
+    const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL);
+    char cwd[PATH_MAX];
+    snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd");
+    if (chdir(cwd)) {
+        throw_errno(env, "chdir");
+        return 1;
+    }
+
+    char *error_prefix;
+    if ((error_prefix = init_signals())) {
+        throw_errno(env, error_prefix);
+        return 1;
+    }
+
     PyConfig config;
     PyStatus status;
-    PyConfig_InitIsolatedConfig(&config);
+    PyConfig_InitPythonConfig(&config);
 
-    status = set_config_string(env, &config, &config.home, home);
-    if (PyStatus_Exception(status)) {
+    jsize argc = (*env)->GetArrayLength(env, args);
+    const char *argv[argc + 1];
+    for (int i = 0; i < argc; i++) {
+        jobject arg = (*env)->GetObjectArrayElement(env, args, i);
+        argv[i] = (*env)->GetStringUTFChars(env, arg, NULL);
+    }
+    argv[argc] = NULL;
+
+    // PyConfig_SetBytesArgv "must be called before other methods, since the
+    // preinitialization configuration depends on command line arguments"
+    if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) {
         throw_status(env, status);
         return 1;
     }
 
-    status = set_config_string(env, &config, &config.run_module, runModule);
+    status = PyConfig_SetBytesString(&config, &config.home, home_utf8);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);
         return 1;
     }
 
-    // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored.
-    config.install_signal_handlers = 1;
-
     status = Py_InitializeFromConfig(&config);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);