Merge remote-tracking branch 'origin/3.13' into backport-a10152f-3.13

Author: colesbury

.github/workflows/build.yml

@@ -208,20 +208,20 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - windows-latest
+          - windows-2022
         arch:
           - x64
         free-threading:
           - false
           - true
         include:
-          - os: windows-latest # FIXME(diegorusso): change to os: windows-aarch64
+          - os: windows-2022 # FIXME(diegorusso): change to os: windows-aarch64
             arch: arm64
             free-threading: false
-          - os: windows-latest # FIXME(diegorusso): change to os: windows-aarch64
+          - os: windows-2022 # FIXME(diegorusso): change to os: windows-aarch64
             arch: arm64
             free-threading: true
-          - os: windows-latest
+          - os: windows-2022
             arch: Win32
             free-threading: false
     uses: ./.github/workflows/reusable-windows.yml
@@ -307,7 +307,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04]
-        openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
+        # Keep 1.1.1w in our list despite it being upstream EOL and otherwise
+        # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
+        # supported by important vendors such as AWS-LC.
+        openssl_ver: [1.1.1w, 3.0.15, 3.1.7, 3.2.3, 3.3.2]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl

.github/workflows/jit.yml

@@ -70,15 +70,15 @@ jobs:
         include:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
-            runner: windows-latest
+            runner: windows-2022
             compiler: msvc
           - target: x86_64-pc-windows-msvc/msvc
             architecture: x64
-            runner: windows-latest
+            runner: windows-2022
             compiler: msvc
           - target: aarch64-pc-windows-msvc/msvc
             architecture: ARM64
-            runner: windows-latest
+            runner: windows-2022
             compiler: msvc
           - target: x86_64-apple-darwin/clang
             architecture: x86_64

.github/workflows/reusable-windows-msi.yml

@@ -17,7 +17,7 @@ env:
 jobs:
   build:
     name: installer for ${{ inputs.arch }}
-    runs-on: windows-latest
+    runs-on: windows-2022
     timeout-minutes: 60
     env:
       ARCH: ${{ inputs.arch }}

.gitignore

@@ -80,7 +80,6 @@ iOS/testbed/Python.xcframework/ios-*/lib
 iOS/testbed/Python.xcframework/ios-*/Python.framework
 iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace
 iOS/testbed/iOSTestbed.xcodeproj/xcuserdata
-iOS/testbed/iOSTestbed.xcodeproj/xcshareddata
 Mac/Makefile
 Mac/PythonLauncher/Info.plist
 Mac/PythonLauncher/Makefile

.pre-commit-config.yaml

@@ -10,10 +10,18 @@ repos:
         name: Run Ruff (lint) on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff (lint) on Tools/i18n/
+        args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml]
+        files: ^Tools/i18n/
       - id: ruff
         name: Run Ruff (lint) on Argument Clinic
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
         files: ^Tools/clinic/|Lib/test/test_clinic.py
+      - id: ruff
+        name: Run Ruff (lint) on Tools/peg_generator/
+        args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml]
+        files: ^Tools/peg_generator/
       - id: ruff-format
         name: Run Ruff (format) on Doc/
         args: [--check]

.ruff.toml

@@ -0,0 +1,12 @@
+# Default settings for Ruff in CPython
+
+# PYTHON_FOR_REGEN
+target-version = "py310"
+
+# PEP 8
+line-length = 79
+
+# Enable automatic fixes by default.
+# To override this, use ``fix = false`` in a subdirectory's config file
+# or ``--no-fix`` on the command line.
+fix = true

Android/android.py

@@ -737,12 +737,10 @@ def ci(context):
             # Prove the package is self-contained by using it to run the tests.
             shutil.unpack_archive(package_path, temp_dir)
 
-            # Arguments are similar to --fast-ci, but in single-process mode.
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
             launcher_args = ["--managed", "maxVersion", "-v"]
-            test_args = [
-                "--single-process", "--fail-env-changed", "--rerun", "--slowest",
-                "--verbose3", "-u", "all,-cpu", "--timeout=600"
-            ]
+            test_args = ["--fast-ci", "--single-process", "--no-randomize"]
             run(
                 ["./android.py", "test", *launcher_args, "--", *test_args],
                 cwd=temp_dir

Doc/.ruff.toml

@@ -1,7 +1,6 @@
+extend = "../.ruff.toml"  # Inherit the project-wide settings
+
 target-version = "py312"  # Align with the version in oldest_supported_sphinx
-fix = true
-output-format = "full"
-line-length = 79
 extend-exclude = [
     "includes/*",
     # Temporary exclusions:

Doc/c-api/init.rst

@@ -447,7 +447,11 @@ Initializing and finalizing the interpreter
    freed.  Some memory allocated by extension modules may not be freed.  Some
    extensions may not work properly if their initialization routine is called more
    than once; this can happen if an application calls :c:func:`Py_Initialize` and
-   :c:func:`Py_FinalizeEx` more than once.
+   :c:func:`Py_FinalizeEx` more than once.  :c:func:`Py_FinalizeEx` must not be
+   called recursively from within itself.  Therefore, it must not be called by
+   any code that may be run as part of the interpreter shutdown process, such
+   as :py:mod:`atexit` handlers, object finalizers, or any code that may be run
+   while flushing the stdout and stderr files.
 
    .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx
 
@@ -1074,6 +1078,37 @@ thread, where the CPython global runtime was originally initialized.
 The only exception is if :c:func:`exec` will be called immediately
 after.
 
+.. _cautions-regarding-runtime-finalization:
+
+Cautions regarding runtime finalization
+---------------------------------------
+
+In the late stage of :term:`interpreter shutdown`, after attempting to wait for
+non-daemon threads to exit (though this can be interrupted by
+:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime
+is marked as *finalizing*: :c:func:`Py_IsFinalizing` and
+:func:`sys.is_finalizing` return true.  At this point, only the *finalization
+thread* that initiated finalization (typically the main thread) is allowed to
+acquire the :term:`GIL`.
+
+If any thread, other than the finalization thread, attempts to acquire the GIL
+during finalization, either explicitly via :c:func:`PyGILState_Ensure`,
+:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or
+:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to
+reacquire it after having yielded it, the thread enters **a permanently blocked
+state** where it remains until the program exits.  In most cases this is
+harmless, but this can result in deadlock if a later stage of finalization
+attempts to acquire a lock owned by the blocked thread, or otherwise waits on
+the blocked thread.
+
+Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++
+finalizations further up the call stack when such threads were forcibly exited
+here in CPython 3.13.7 and earlier. The CPython runtime GIL acquiring C APIs
+have never had any error reporting or handling expectations at GIL acquisition
+time that would've allowed for graceful exit from this situation. Changing that
+would require new stable C APIs and rewriting the majority of C code in the
+CPython ecosystem to use those with error handling.
+
 
 High-level API
 --------------
@@ -1093,6 +1128,12 @@ code, or when embedding the Python interpreter:
    interpreter lock is also shared by all threads, regardless of to which
    interpreter they belong.
 
+   .. versionchanged:: 3.12
+
+      :pep:`684` introduced the possibility
+      of a :ref:`per-interpreter GIL <per-interpreter-gil>`.
+      See :c:func:`Py_NewInterpreterFromConfig`.
+
 
 .. c:type:: PyThreadState
 
@@ -1147,11 +1188,14 @@ code, or when embedding the Python interpreter:
    ensues.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
+
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
 
 .. c:function:: PyThreadState* PyThreadState_Get()
 
@@ -1207,11 +1251,14 @@ with sub-interpreters:
    to call arbitrary Python code.  Failure is a fatal error.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
+
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
 
 .. c:function:: void PyGILState_Release(PyGILState_STATE)
 
@@ -1503,17 +1550,20 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    If this thread already has the lock, deadlock ensues.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
 
    .. versionchanged:: 3.8
       Updated to be consistent with :c:func:`PyEval_RestoreThread`,
       :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`,
       and terminate the current thread if called while the interpreter is finalizing.
 
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
+
    :c:func:`PyEval_RestoreThread` is a higher-level function which is always
    available (even when threads have not been initialized).
 
@@ -1773,6 +1823,8 @@ function. You can create and destroy them using the following functions:
    haven't been explicitly destroyed at that point.
 
 
+.. _per-interpreter-gil:
+
 A Per-Interpreter GIL
 ---------------------
 
@@ -1784,7 +1836,7 @@ being blocked by other interpreters or blocking any others.  Thus a
 single Python process can truly take advantage of multiple CPU cores
 when running Python code.  The isolation also encourages a different
 approach to concurrency than that of just using threads.
-(See :pep:`554`.)
+(See :pep:`554` and :pep:`684`.)
 
 Using an isolated interpreter requires vigilance in preserving that
 isolation.  That especially means not sharing any objects or mutable
@@ -2352,12 +2404,20 @@ per-object locks for :term:`free-threaded <free threading>` CPython.  They are
 intended to replace reliance on the :term:`global interpreter lock`, and are
 no-ops in versions of Python with the global interpreter lock.
 
+Critical sections are intended to be used for custom types implemented
+in C-API extensions. They should generally not be used with built-in types like
+:class:`list` and :class:`dict` because their public C-APIs
+already use critical sections internally, with the notable
+exception of :c:func:`PyDict_Next`, which requires critical section
+to be acquired externally.
+
 Critical sections avoid deadlocks by implicitly suspending active critical
-sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`.
-When :c:func:`PyEval_RestoreThread` is called, the most recent critical section
-is resumed, and its locks reacquired.  This means the critical section API
-provides weaker guarantees than traditional locks -- they are useful because
-their behavior is similar to the :term:`GIL`.
+sections, hence, they do not provide exclusive access such as provided by
+traditional locks like :c:type:`PyMutex`.  When a critical section is started,
+the per-object lock for the object is acquired. If the code executed inside the
+critical section calls C-API functions then it can suspend the critical section thereby
+releasing the per-object lock, so other threads can acquire the per-object lock
+for the same object.
 
 The functions and structs used by the macros are exposed for cases
 where C macros are not available. They should only be used as in the

Doc/c-api/long.rst

@@ -40,9 +40,11 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
 
    Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure.
 
-   The current implementation keeps an array of integer objects for all integers
-   between ``-5`` and ``256``. When you create an int in that range you actually
-   just get back a reference to the existing object.
+   .. impl-detail::
+
+      CPython keeps an array of integer objects for all integers
+      between ``-5`` and ``256``.  When you create an int in that range
+      you actually just get back a reference to the existing object.
 
 
 .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v)