Merge branch 'main' into remove-__cached__

Author: brettcannon

.github/CODEOWNERS

@@ -126,6 +126,9 @@ Doc/howto/clinic.rst          @erlend-aasland @AA-Turner
 # C Analyser
 Tools/c-analyzer/             @ericsnowcurrently
 
+# C API Documentation Checks
+Tools/check-c-api-docs/       @ZeroIntensity
+
 # Fuzzing
 Modules/_xxtestfuzz/          @ammaraskar
 

.github/ISSUE_TEMPLATE/config.yml

@@ -5,3 +5,6 @@ contact_links:
   - name: "Proposing new features"
     about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first."
     url: "https://discuss.python.org/c/ideas/6"
+  - name: "Python Install Manager issues"
+    about: "Report issues with the Python Install Manager (for Windows)"
+    url: "https://github.com/python/pymanager/issues"

.github/workflows/build.yml

@@ -142,6 +142,9 @@ jobs:
       - name: Check for unsupported C global variables
         if: github.event_name == 'pull_request'  # $GITHUB_EVENT_NAME
         run: make check-c-globals
+      - name: Check for undocumented C APIs
+        run: make check-c-api-docs
+
 
   build-windows:
     name: >-
@@ -188,7 +191,7 @@ jobs:
       macOS
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-macos == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -214,7 +217,7 @@ jobs:
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
       ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }}
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -245,7 +248,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -301,7 +304,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -365,7 +368,7 @@ jobs:
   build-android:
     name: Android (${{ matrix.arch }})
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-android == 'true'
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -387,9 +390,9 @@ jobs:
   build-ios:
     name: iOS
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ios == 'true'
     timeout-minutes: 60
-    runs-on: macos-15
+    runs-on: macos-14
     steps:
       - uses: actions/checkout@v4
         with:
@@ -402,23 +405,23 @@ jobs:
       # https://github.com/actions/runner-images/issues/12751.
       - name: Select Xcode version
         run: |
-          sudo xcode-select --switch /Applications/Xcode_16.4.app
+          sudo xcode-select --switch /Applications/Xcode_15.4.app
 
       - name: Build and test
-        run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5'
+        run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5'
 
   build-wasi:
     name: 'WASI'
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-wasi == 'true'
     uses: ./.github/workflows/reusable-wasi.yml
 
   test-hypothesis:
     name: "Hypothesis tests on Ubuntu"
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     env:
       OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
@@ -525,7 +528,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -578,7 +581,7 @@ jobs:
     # ${{ '' } is a hack to nest jobs under the same sidebar category.
     name: Sanitizers${{ '' }}  # zizmor: ignore[obfuscation]
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -603,7 +606,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     needs: build-context
-    if: needs.build-context.outputs.run-tests == 'true'
+    if: needs.build-context.outputs.run-ubuntu == 'true'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -703,50 +706,39 @@ jobs:
       uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
       with:
         allowed-failures: >-
+          build-ios,
           build-windows-msi,
           build-ubuntu-ssltests-awslc,
           build-ubuntu-ssltests-openssl,
           test-hypothesis,
           cifuzz,
         allowed-skips: >-
+          ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }}
           ${{
-            !fromJSON(needs.build-context.outputs.run-docs)
+            needs.build-context.outputs.run-tests != 'true'
             && '
-            check-docs,
+            check-autoconf-regen,
+            check-generated-files,
             '
             || ''
           }}
+          ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }}
+          ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }}
+          ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }}
           ${{
-            needs.build-context.outputs.run-tests != 'true'
+            !fromJSON(needs.build-context.outputs.run-ubuntu)
             && '
-            check-autoconf-regen,
-            check-generated-files,
-            build-macos,
             build-ubuntu,
             build-ubuntu-ssltests-awslc,
             build-ubuntu-ssltests-openssl,
-            build-android,
-            build-ios,
-            build-wasi,
             test-hypothesis,
             build-asan,
             build-san,
             cross-build-linux,
             '
             || ''
           }}
-          ${{
-            !fromJSON(needs.build-context.outputs.run-windows-tests)
-            && '
-            build-windows,
-            '
-            || ''
-          }}
-          ${{
-            !fromJSON(needs.build-context.outputs.run-ci-fuzz)
-            && '
-            cifuzz,
-            '
-            || ''
-          }}
+          ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }}
+          ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }}
+          ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }}
         jobs: ${{ toJSON(needs) }}

.github/workflows/reusable-context.yml

@@ -17,31 +17,51 @@ on:  # yamllint disable-line rule:truthy
       #        || 'falsy-branch'
       #   }}
       #
+      run-android:
+        description: Whether to run the Android tests
+        value: ${{ jobs.compute-changes.outputs.run-android }}  # bool
+      run-ci-fuzz:
+        description: Whether to run the CIFuzz job
+        value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }}  # bool
       run-docs:
         description: Whether to build the docs
         value: ${{ jobs.compute-changes.outputs.run-docs }}  # bool
+      run-ios:
+        description: Whether to run the iOS tests
+        value: ${{ jobs.compute-changes.outputs.run-ios }}  # bool
+      run-macos:
+        description: Whether to run the macOS tests
+        value: ${{ jobs.compute-changes.outputs.run-macos }}  # bool
       run-tests:
         description: Whether to run the regular tests
         value: ${{ jobs.compute-changes.outputs.run-tests  }}  # bool
-      run-windows-tests:
-        description: Whether to run the Windows tests
-        value: ${{ jobs.compute-changes.outputs.run-windows-tests }}  # bool
+      run-ubuntu:
+        description: Whether to run the Ubuntu tests
+        value: ${{ jobs.compute-changes.outputs.run-ubuntu }}  # bool
+      run-wasi:
+        description: Whether to run the WASI tests
+        value: ${{ jobs.compute-changes.outputs.run-wasi }}  # bool
       run-windows-msi:
         description: Whether to run the MSI installer smoke tests
         value: ${{ jobs.compute-changes.outputs.run-windows-msi }}  # bool
-      run-ci-fuzz:
-        description: Whether to run the CIFuzz job
-        value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }}  # bool
+      run-windows-tests:
+        description: Whether to run the Windows tests
+        value: ${{ jobs.compute-changes.outputs.run-windows-tests }}  # bool
 
 jobs:
   compute-changes:
     name: Create context from changed files
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
+      run-android: ${{ steps.changes.outputs.run-android }}
       run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
       run-docs: ${{ steps.changes.outputs.run-docs }}
+      run-ios: ${{ steps.changes.outputs.run-ios }}
+      run-macos: ${{ steps.changes.outputs.run-macos }}
       run-tests: ${{ steps.changes.outputs.run-tests }}
+      run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }}
+      run-wasi: ${{ steps.changes.outputs.run-wasi }}
       run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }}
       run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }}
     steps:

Android/android.py

@@ -29,6 +29,7 @@
     ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
 )
 
+ENV_SCRIPT = ANDROID_DIR / "android-env.sh"
 TESTBED_DIR = ANDROID_DIR / "testbed"
 CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
 
@@ -129,12 +130,11 @@ def android_env(host):
         sysconfig_filename = next(sysconfig_files).name
         host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
 
-    env_script = ANDROID_DIR / "android-env.sh"
     env_output = subprocess.run(
         f"set -eu; "
         f"HOST={host}; "
         f"PREFIX={prefix}; "
-        f". {env_script}; "
+        f". {ENV_SCRIPT}; "
         f"export",
         check=True, shell=True, capture_output=True, encoding='utf-8',
     ).stdout
@@ -151,7 +151,7 @@ def android_env(host):
                 env[key] = value
 
     if not env:
-        raise ValueError(f"Found no variables in {env_script.name} output:\n"
+        raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n"
                          + env_output)
     return env
 
@@ -281,15 +281,30 @@ def clean_all(context):
 
 
 def setup_ci():
-    # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/
-    if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux":
-        run(
-            ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
-            input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
-            text=True,
-        )
-        run(["sudo", "udevadm", "control", "--reload-rules"])
-        run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
+    if "GITHUB_ACTIONS" in os.environ:
+        # Enable emulator hardware acceleration
+        # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/).
+        if platform.system() == "Linux":
+            run(
+                ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
+                input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
+                text=True,
+            )
+            run(["sudo", "udevadm", "control", "--reload-rules"])
+            run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
+
+        # Free up disk space by deleting unused versions of the NDK
+        # (https://github.com/freakboy3742/pyspamsum/pull/108).
+        for line in ENV_SCRIPT.read_text().splitlines():
+            if match := re.fullmatch(r"ndk_version=(.+)", line):
+                ndk_version = match[1]
+                break
+        else:
+            raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}")
+
+        for item in (android_home / "ndk").iterdir():
+            if item.name[0].isdigit() and item.name != ndk_version:
+                delete_glob(item)
 
 
 def setup_sdk():

Android/testbed/app/build.gradle.kts

@@ -79,7 +79,7 @@ android {
     val androidEnvFile = file("../../android-env.sh").absoluteFile
 
     namespace = "org.python.testbed"
-    compileSdk = 34
+    compileSdk = 35
 
     defaultConfig {
         applicationId = "org.python.testbed"
@@ -92,7 +92,7 @@ android {
             }
             throw GradleException("Failed to find API level in $androidEnvFile")
         }
-        targetSdk = 34
+        targetSdk = 35
 
         versionCode = 1
         versionName = "1.0"

Doc/c-api/gcsupport.rst

@@ -232,6 +232,10 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type:
    object argument.  If *visit* returns a non-zero value that value should be
    returned immediately.
 
+   The traversal function must not have any side effects.  Implementations
+   may not modify the reference counts of any Python objects nor create or
+   destroy any Python objects.
+
 To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is
 provided.  In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation
 must name its arguments exactly *visit* and *arg*:

Doc/c-api/set.rst

@@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
 
    Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an
    error is encountered.  Does not raise :exc:`KeyError` for missing keys.  Raise a
-   :exc:`TypeError` if the *key* is unhashable.  Unlike the Python :meth:`~frozenset.discard`
+   :exc:`TypeError` if the *key* is unhashable.  Unlike the Python :meth:`~set.discard`
    method, this function does not automatically convert unhashable sets into
    temporary frozensets. Raise :exc:`SystemError` if *set* is not an
    instance of :class:`set` or its subtype.

Doc/c-api/typeobj.rst

@@ -1569,6 +1569,11 @@ and :c:data:`PyType_Type` effectively act as defaults.)
        but the instance has no strong reference to the elements inside it, as they
        are allowed to be removed even if the instance is still alive).
 
+   .. warning::
+      The traversal function must not have any side effects.  It must not
+      modify the reference counts of any Python objects nor create or destroy
+      any Python objects.
+
    Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to
    :c:func:`!local_traverse` to have these specific names; don't name them just
    anything.