Merge remote-tracking branch 'upstream/main' into msvc-tailcall-take-two

Author: Fidget-Spinner

.devcontainer/wasi/devcontainer.json

@@ -0,0 +1,73 @@
+{
+    "image": "ghcr.io/python/wasicontainer:latest",
+    "onCreateCommand": [
+        // Install common tooling.
+        "dnf",
+        "install",
+        "-y",
+        // For umask fix below.
+        "/usr/bin/setfacl"
+    ],
+    "updateContentCommand": {
+        // Using the shell for `nproc` usage.
+        "python": "python3 Tools/wasm/wasi build --quiet -- --with-pydebug -C"
+    },
+    "postCreateCommand": {
+        // https://github.com/orgs/community/discussions/26026
+        "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."],
+        "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"]
+    },
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                // Highlighting for Parser/Python.asdl.
+                "brettcannon.zephyr-asdl",
+                // Highlighting for configure.ac.
+                "maelvalais.autoconf",
+                // C auto-complete.
+                "ms-vscode.cpptools",
+                // Python auto-complete.
+                "ms-python.python"
+            ],
+            "settings": {
+                "C_Cpp.default.compilerPath": "/usr/bin/clang",
+                "C_Cpp.default.cStandard": "c11",
+                "C_Cpp.default.defines": [
+                    "CONFIG_64",
+                    "Py_BUILD_CORE"
+                ],
+                "C_Cpp.default.includePath": [
+                    "${workspaceFolder}/*",
+                    "${workspaceFolder}/Include/**"
+                ],
+                // https://github.com/microsoft/vscode-cpptools/issues/10732
+                "C_Cpp.errorSquiggles": "disabled",
+                "editor.insertSpaces": true,
+                "editor.rulers": [
+                    80
+                ],
+                "editor.tabSize": 4,
+                "editor.trimAutoWhitespace": true,
+                "files.associations": {
+                    "*.h": "c"
+                },
+                "files.encoding": "utf8",
+                "files.eol": "\n",
+                "files.insertFinalNewline": true,
+                "files.trimTrailingWhitespace": true,
+                "python.analysis.diagnosticSeverityOverrides": {
+                    // Complains about shadowing the stdlib w/ the stdlib.
+                    "reportShadowedImports": "none",
+                    // Doesn't like _frozen_importlib.
+                    "reportMissingImports": "none"
+                },
+                "python.analysis.extraPaths": [
+                    "Lib"
+                ],
+                "[restructuredtext]": {
+                    "editor.tabSize": 3
+                }
+            }
+        }
+    }
+}

.github/workflows/build.yml

@@ -414,6 +414,29 @@ jobs:
       - name: Build and test
         run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android
 
+  build-ios:
+    name: iOS
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
+    timeout-minutes: 60
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      # GitHub recommends explicitly selecting the desired Xcode version:
+      # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140
+      # This became a necessity as a result of
+      # https://github.com/actions/runner-images/issues/12541 and
+      # https://github.com/actions/runner-images/issues/12751.
+      - name: Select Xcode version
+        run: |
+          sudo xcode-select --switch /Applications/Xcode_16.4.app
+
+      - name: Build and test
+        run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5'
+
   build-wasi:
     name: 'WASI'
     needs: build-context
@@ -723,6 +746,7 @@ jobs:
     - build-ubuntu-ssltests-awslc
     - build-ubuntu-ssltests-openssl
     - build-android
+    - build-ios
     - build-wasi
     - test-hypothesis
     - build-asan
@@ -759,6 +783,7 @@ jobs:
             build-ubuntu-ssltests-awslc,
             build-ubuntu-ssltests-openssl,
             build-android,
+            build-ios,
             build-wasi,
             test-hypothesis,
             build-asan,

.github/workflows/jit.yml

@@ -68,7 +68,7 @@ jobs:
           - true
           - false
         llvm:
-          - 19
+          - 20
         include:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
@@ -138,7 +138,7 @@ jobs:
       fail-fast: false
       matrix:
         llvm:
-          - 19
+          - 20
     steps:
       - uses: actions/checkout@v4
         with:
@@ -166,7 +166,7 @@ jobs:
       fail-fast: false
       matrix:
         llvm:
-          - 19
+          - 20
     steps:
       - uses: actions/checkout@v4
         with:
@@ -183,3 +183,27 @@ jobs:
       - name: Run tests without optimizations
         run: |
           PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+  tail-call-jit:
+    name: JIT with tail calling interpreter
+    needs: interpreter
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        llvm:
+          - 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Build with JIT and tailcall
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug
+          make all --jobs 4

.github/workflows/mypy.yml

@@ -16,6 +16,7 @@ on:
       - "Tools/build/check_extension_modules.py"
       - "Tools/build/check_warnings.py"
       - "Tools/build/compute-changes.py"
+      - "Tools/build/consts_getter.py"
       - "Tools/build/deepfreeze.py"
       - "Tools/build/generate-build-details.py"
       - "Tools/build/generate_sbom.py"

.github/workflows/reusable-macos.yml

@@ -43,9 +43,9 @@ jobs:
         key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
       run: |
-        brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make
+        brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make
         # Because alternate versions are not symlinked into place by default:
-        brew link --overwrite tcl-tk@8
+        brew link --overwrite tcl-tk@9
     - name: Configure CPython
       run: |
         MACOSX_DEPLOYMENT_TARGET=10.15 \

.github/workflows/reusable-wasi.yml

@@ -13,11 +13,11 @@ env:
 jobs:
   build-wasi-reusable:
     name: 'build and test'
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-24.04-arm
     timeout-minutes: 60
     env:
-      WASMTIME_VERSION: 22.0.0
-      WASI_SDK_VERSION: 24
+      WASMTIME_VERSION: 38.0.3
+      WASI_SDK_VERSION: 25
       WASI_SDK_PATH: /opt/wasi-sdk
       CROSS_BUILD_PYTHON: cross-build/build
       CROSS_BUILD_WASI: cross-build/wasm32-wasip1
@@ -40,7 +40,7 @@ jobs:
       if: steps.cache-wasi-sdk.outputs.cache-hit != 'true'
       run: |
         mkdir "${WASI_SDK_PATH}" && \
-        curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \
+        curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \
         tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip
     - name: "Configure ccache action"
       uses: hendrikmuhs/ccache-action@v1.2
@@ -60,24 +60,24 @@ jobs:
       with:
         path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
         # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python.
-        # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables.
+        # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables.
         # (Make sure to keep the key in sync with the other config.cache step below.)
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }}
     - name: "Configure build Python"
-      run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
+      run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug
     - name: "Make build Python"
-      run: python3 Tools/wasm/wasi.py make-build-python
+      run: python3 Tools/wasm/wasi make-build-python
     - name: "Restore host config.cache"
       uses: actions/cache@v4
       with:
         path: ${{ env.CROSS_BUILD_WASI }}/config.cache
         # Should be kept in sync with the other config.cache step above.
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }}
     - name: "Configure host"
       # `--with-pydebug` inferred from configure-build-python
-      run: python3 Tools/wasm/wasi.py configure-host -- --config-cache
+      run: python3 Tools/wasm/wasi configure-host -- --config-cache
     - name: "Make host"
-      run: python3 Tools/wasm/wasi.py make-host
+      run: python3 Tools/wasm/wasi make-host
     - name: "Display build info"
       run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
     - name: "Test"

Android/testbed/app/build.gradle.kts

@@ -47,7 +47,7 @@ for ((i, prefix) in prefixes.withIndex()) {
     val libDir = file("$prefix/lib")
     val version = run {
         for (filename in libDir.list()!!) {
-            """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let {
+            """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let {
                 return@run it.groupValues[1]
             }
         }
@@ -64,9 +64,10 @@ for ((i, prefix) in prefixes.withIndex()) {
     val libPythonDir = file("$libDir/python$pythonVersion")
     val triplet = run {
         for (filename in libPythonDir.list()!!) {
-            """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let {
-                return@run it.groupValues[1]
-            }
+            """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex()
+                .matchEntire(filename)?.let {
+                    return@run it.groupValues[1]
+                }
         }
         throw GradleException("Failed to find Python triplet in $libPythonDir")
     }

Apple/__main__.py

@@ -823,7 +823,7 @@ def test(context: argparse.Namespace, host: str | None = None) -> None:
             + [
                 "--",
                 "test",
-                "--slow-ci" if context.slow else "--fast-ci",
+                f"--{context.ci_mode}-ci",
                 "--single-process",
                 "--no-randomize",
                 # Timeout handling requires subprocesses; explicitly setting
@@ -836,11 +836,39 @@ def test(context: argparse.Namespace, host: str | None = None) -> None:
         )
 
 
+def apple_sim_host(platform_name: str) -> str:
+    """Determine the native simulator target for this platform."""
+    for _, slice_parts in HOSTS[platform_name].items():
+        for host_triple in slice_parts:
+            parts = host_triple.split('-')
+            if parts[0] == platform.machine() and parts[-1] == "simulator":
+                return host_triple
+
+    raise KeyError(platform_name)
+
+
 def ci(context: argparse.Namespace) -> None:
-    """The implementation of the "ci" command."""
+    """The implementation of the "ci" command.
+
+    In "Fast" mode, this compiles the build python, and the simulator for the
+    build machine's architecture; and runs the test suite with `--fast-ci`
+    configuration.
+
+    In "Slow" mode, it compiles the build python, plus all candidate
+    architectures (both device and simulator); then runs the test suite with
+    `--slow-ci` configuration.
+    """
     clean(context, "all")
-    build(context, host="all")
-    test(context, host="all")
+    if context.ci_mode == "slow":
+        # In slow mode, build and test the full XCframework
+        build(context, host="all")
+        test(context, host="all")
+    else:
+        # In fast mode, just build the simulator platform.
+        sim_host = apple_sim_host(context.platform)
+        build(context, host="build")
+        build(context, host=sim_host)
+        test(context, host=sim_host)
 
 
 def parse_args() -> argparse.Namespace:
@@ -947,11 +975,13 @@ def parse_args() -> argparse.Namespace:
                 "an ARM64 iPhone 16 Pro simulator running iOS 26.0."
             ),
         )
-        cmd.add_argument(
-            "--slow",
-            action="store_true",
-            help="Run tests with --slow-ci options.",
-        )
+        group = cmd.add_mutually_exclusive_group()
+        group.add_argument(
+            "--fast-ci", action="store_const", dest="ci_mode", const="fast",
+            help="Add test arguments for GitHub Actions")
+        group.add_argument(
+            "--slow-ci", action="store_const", dest="ci_mode", const="slow",
+            help="Add test arguments for buildbots")
 
     for subcommand in [configure_build, configure_host, build, ci]:
         subcommand.add_argument(
@@ -1012,4 +1042,10 @@ def signal_handler(*args):
 
 
 if __name__ == "__main__":
+    # Under the buildbot, stdout is not a TTY, but we must still flush after
+    # every line to make sure our output appears in the correct order relative
+    # to the output of our subprocesses.
+    for stream in [sys.stdout, sys.stderr]:
+        stream.reconfigure(line_buffering=True)
+
     main()

Apple/iOS/README.md

@@ -224,6 +224,17 @@ Once you have a built an XCframework, you can test that framework by running:
 
   $ python Apple test iOS
 
+This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or
+iPhone 16e, or similar), and run the test suite on the most recent version of
+iOS that is available. You can specify a simulator using the `--simulator`
+command line argument, providing the name of the simulator (e.g., `--simulator
+'iPhone 16 Pro'`). You can also use this argument to control the OS version used
+for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the
+tests on an iPhone 16 Pro running iOS 18.2.
+
+If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS`
+environment variable will be exposed to the iOS process at runtime.
+
 ### Testing a single-architecture framework
 
 The `Apple/testbed` folder that contains an Xcode project that is able to run

Apple/testbed/TestbedTests/TestbedTests.m

@@ -35,6 +35,9 @@ - (void)testPython {
     setenv("NO_COLOR", "1", true);
     setenv("PYTHON_COLORS", "0", true);
 
+    if (getenv("GITHUB_ACTIONS")) {
+        NSLog(@"Running in a GitHub Actions environment");
+    }
     // Arguments to pass into the test suite runner.
     // argv[0] must identify the process; any subsequent arg
     // will be handled as if it were an argument to `python -m test`