Merge branch 'main' into branch-unique-reference-tracking

Author: reidenong

.gitattributes

@@ -106,6 +106,7 @@ Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
 Python/optimizer_cases.c.h                          generated
 Python/opcode_targets.h                             generated
+Python/record_functions.c.h                         generated
 Python/stdlib_module_names.h                        generated
 Tools/peg_generator/pegen/grammar_parser.py         generated
 aclocal.m4                                          generated

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,15 @@
+name: Documentation
+description: Report a problem with the documentation
+labels: ["docs"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        > [!NOTE]
+        > Trivial changes (for example typos) don’t require an issue before opening a PR.
+  - type: textarea
+    attributes:
+      label: "Documentation"
+      description: "A clear and concise description of the issue."
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/documentation.yml

@@ -655,11 +655,14 @@ jobs:
       matrix:
         sanitizer:
         - address
-        - undefined
-        - memory
         oss-fuzz-project-name:
         - cpython3
         - python3-libraries
+        include:
+        - sanitizer: undefined
+          oss-fuzz-project-name: cpython3
+        - sanitizer: memory
+          oss-fuzz-project-name: cpython3
         exclude:
         # Note that the 'no-exclude' sentinel below is to prevent
         # an empty string value from excluding all jobs and causing

.github/workflows/build.yml

@@ -15,6 +15,7 @@
 from asyncio import wait_for
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
+from enum import IntEnum, auto
 from glob import glob
 from os.path import abspath, basename, relpath
 from pathlib import Path
@@ -61,6 +62,19 @@
 hidden_output = []
 
 
+# Based on android/log.h in the NDK.
+class LogPriority(IntEnum):
+    UNKNOWN = 0
+    DEFAULT = auto()
+    VERBOSE = auto()
+    DEBUG = auto()
+    INFO = auto()
+    WARN = auto()
+    ERROR = auto()
+    FATAL = auto()
+    SILENT = auto()
+
+
 def log_verbose(context, line, stream=sys.stdout):
     if context.verbose:
         stream.write(line)
@@ -505,47 +519,47 @@ async def logcat_task(context, initial_devices):
     pid = await wait_for(find_pid(serial), startup_timeout)
 
     # `--pid` requires API level 24 or higher.
-    args = [adb, "-s", serial, "logcat", "--pid", pid,  "--format", "tag"]
+    #
+    # `--binary` mode is used in order to detect which messages end with a
+    # newline, which most of the other modes don't indicate (except `--format
+    # long`). For example, every time pytest runs a test, it prints a "." and
+    # flushes the stream. Each "." becomes a separate log message, but we should
+    # show them all on the same line.
+    args = [adb, "-s", serial, "logcat", "--pid", pid,  "--binary"]
     logcat_started = False
     async with async_process(
-        *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        *args, stdout=subprocess.PIPE, stderr=None
     ) as process:
-        while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
-            if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL):
+        while True:
+            try:
+                priority, tag, message = await read_logcat(process.stdout)
                 logcat_started = True
-                level, message = match.groups()
-            else:
-                # If the regex doesn't match, this is either a logcat startup
-                # error, or the second or subsequent line of a multi-line
-                # message. Python won't produce multi-line messages, but other
-                # components might.
-                level, message = None, line
+            except asyncio.IncompleteReadError:
+                break
 
             # Exclude high-volume messages which are rarely useful.
             if context.verbose < 2 and "from python test_syslog" in message:
                 continue
 
             # Put high-level messages on stderr so they're highlighted in the
             # buildbot logs. This will include Python's own stderr.
-            stream = (
-                sys.stderr
-                if level in ["W", "E", "F"]  # WARNING, ERROR, FATAL (aka ASSERT)
-                else sys.stdout
-            )
-
-            # To simplify automated processing of the output, e.g. a buildbot
-            # posting a failure notice on a GitHub PR, we strip the level and
-            # tag indicators from Python's stdout and stderr.
-            for prefix in ["python.stdout: ", "python.stderr: "]:
-                if message.startswith(prefix):
-                    global python_started
-                    python_started = True
-                    stream.write(message.removeprefix(prefix))
-                    break
+            stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout
+
+            # The app's stdout and stderr should be passed through transparently
+            # to our own corresponding streams.
+            if tag in ["python.stdout", "python.stderr"]:
+                global python_started
+                python_started = True
+                stream.write(message)
+                stream.flush()
             else:
                 # Non-Python messages add a lot of noise, but they may
-                # sometimes help explain a failure.
-                log_verbose(context, line, stream)
+                # sometimes help explain a failure. Format them in the same way
+                # as `logcat --format tag`.
+                formatted = f"{priority.name[0]}/{tag}: {message}"
+                if not formatted.endswith("\n"):
+                    formatted += "\n"
+                log_verbose(context, formatted, stream)
 
         # If the device disconnects while logcat is running, which always
         # happens in --managed mode, some versions of adb return non-zero.
@@ -556,6 +570,44 @@ async def logcat_task(context, initial_devices):
             raise CalledProcessError(status, args)
 
 
+# Read one binary log message from the given StreamReader. The message format is
+# described at https://android.stackexchange.com/a/74660. All supported versions
+# of Android use format version 2 or later.
+async def read_logcat(stream):
+    async def read_bytes(size):
+        return await stream.readexactly(size)
+
+    async def read_int(size):
+        return int.from_bytes(await read_bytes(size), "little")
+
+    payload_len = await read_int(2)
+    if payload_len < 2:
+        # 1 byte for priority, 1 byte for null terminator of tag.
+        raise ValueError(f"payload length {payload_len} is too short")
+
+    header_len = await read_int(2)
+    if header_len < 4:
+        raise ValueError(f"header length {header_len} is too short")
+    await read_bytes(header_len - 4)  # Ignore other header fields.
+
+    priority_int = await read_int(1)
+    try:
+        priority = LogPriority(priority_int)
+    except ValueError:
+        priority = LogPriority.UNKNOWN
+
+    payload_fields = (await read_bytes(payload_len - 1)).split(b"\0")
+    if len(payload_fields) < 2:
+        raise ValueError(
+            f"payload {payload!r} does not contain at least 2 "
+            f"null-separated fields"
+        )
+    tag, message, *_ = [
+        field.decode(*DECODE_ARGS) for field in payload_fields
+    ]
+    return priority, tag, message
+
+
 def stop_app(serial):
     run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False)
 

Android/android.py

@@ -94,10 +94,13 @@ android {
         }
 
         // This controls the API level of the maxVersion managed emulator, which is used
-        // by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has
-        // issues connecting to the internet (#142387), and 36 and later are not
-        // available as aosp_atd images yet.
-        targetSdk = 33
+        // by CI and cibuildwheel.
+        //  * 33 has excessive buffering in the logcat client
+        //    (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826).
+        //  * 34 consumes too much disk space on GitHub Actions (#142289).
+        //  * 35 has issues connecting to the internet (#142387).
+        //  * 36 and later are not available as aosp_atd images yet.
+        targetSdk = 32
 
         versionCode = 1
         versionName = "1.0"
@@ -130,9 +133,10 @@ android {
         path("src/main/c/CMakeLists.txt")
     }
 
-    // Set this property to something non-empty, otherwise it'll use the default
-    // list, which ignores asset directories beginning with an underscore.
-    aaptOptions.ignoreAssetsPattern = ".git"
+    // Set this property to something nonexistent but non-empty. Otherwise it'll use the
+    // default list, which ignores asset directories beginning with an underscore, and
+    // maybe also other files required by tests.
+    aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything"
 
     compileOptions {
         sourceCompatibility = JavaVersion.VERSION_1_8
@@ -234,6 +238,12 @@ androidComponents.onVariants { variant ->
                     from(cwd)
                 }
             }
+
+            // A filename ending with .gz will be automatically decompressed
+            // while building the APK. Avoid this by adding a dash to the end,
+            // and add an extra dash to any filenames that already end with one.
+            // This will be undone in MainActivity.kt.
+            rename(""".*(\.gz|-)""", "$0-")
         }
     }
 

Android/testbed/app/build.gradle.kts

@@ -80,7 +80,9 @@ class PythonTestRunner(val context: Context) {
                 continue
             }
             input.use {
-                File(targetSubdir, name).outputStream().use { output ->
+                // Undo the .gz workaround from build.gradle.kts.
+                val outputName = name.replace(Regex("""(.*)-"""), "$1")
+                File(targetSubdir, outputName).outputStream().use { output ->
                     input.copyTo(output)
                 }
             }

Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt

@@ -167,6 +167,35 @@ complete listing.
 
    .. versionadded:: 3.3
 
+.. c:macro:: Py_ALIGNED(num)
+
+   Specify alignment to *num* bytes on compilers that support it.
+
+   Consider using the C11 standard ``_Alignas`` specifier over this macro.
+
+.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions)
+
+   Similar to ``integer >> positions``, but forces sign extension, as the C
+   standard does not define whether a right-shift of a signed integer will
+   perform sign extension or a zero-fill.
+
+   *integer* should be any signed integer type.
+   *positions* is the number of positions to shift to the right.
+
+   Both *integer* and *positions* can be evaluated more than once;
+   consequently, avoid directly passing a function call or some other
+   operation with side-effects to this macro. Instead, store the result as a
+   variable and then pass it.
+
+   *type* is unused and only kept for backwards compatibility. Historically,
+   *type* was used to cast *integer*.
+
+   .. versionchanged:: 3.1
+
+      This macro is now valid for all signed integer types, not just those for
+      which ``unsigned type`` is legal. As a result, *type* is no longer
+      used.
+
 .. c:macro:: Py_ALWAYS_INLINE
 
    Ask the compiler to always inline a static inline function. The compiler can
@@ -189,6 +218,15 @@ complete listing.
 
    .. versionadded:: 3.11
 
+.. c:macro:: Py_CAN_START_THREADS
+
+   If this macro is defined, then the current system is able to start threads.
+
+   Currently, all systems supported by CPython (per :pep:`11`), with the
+   exception of some WebAssembly platforms, support starting threads.
+
+   .. versionadded:: 3.13
+
 .. c:macro:: Py_CHARMASK(c)
 
    Argument must be a character or an integer in the range [-128, 127] or [0,
@@ -206,11 +244,35 @@ complete listing.
    .. versionchanged:: 3.8
       MSVC support was added.
 
+.. c:macro:: Py_FORCE_EXPANSION(X)
+
+   This is equivalent to ``X``, which is useful for token-pasting in
+   macros, as macro expansions in *X* are forcefully evaluated by the
+   preprocessor.
+
+.. c:macro:: Py_GCC_ATTRIBUTE(name)
+
+   Use a GCC attribute *name*, hiding it from compilers that don't support GCC
+   attributes (such as MSVC).
+
+   This expands to ``__attribute__((name))`` on a GCC compiler, and expands
+   to nothing on compilers that don't support GCC attributes.
+
 .. c:macro:: Py_GETENV(s)
 
    Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the
    command line (see :c:member:`PyConfig.use_environment`).
 
+.. c:macro:: Py_LL(number)
+
+   Use *number* as a ``long long`` integer literal.
+
+   This usally expands to *number* followed by ``LL``, but will expand to some
+   compiler-specific suffixes (such as ``I64``) on older compilers.
+
+   In modern versions of Python, this macro is not very useful, as C99 and
+   later require the ``LL`` suffix to be valid for an integer.
+
 .. c:macro:: Py_LOCAL(type)
 
    Declare a function returning the specified *type* using a fast-calling
@@ -268,13 +330,37 @@ complete listing.
 
    .. versionadded:: 3.11
 
+.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller)
+
+   Cast *value* to type *smaller* from type *larger*, validating that no
+   information was lost.
+
+   On release builds of Python, this is roughly equivalent to
+   ``(smaller) value`` (in C++, ``static_cast<smaller>(value)`` will be
+   used instead).
+
+   On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts
+   that no information was lost with the cast from *larger* to *smaller*.
+
+   *value*, *larger*, and *smaller* may all be evaluated more than once in the
+   expression; consequently, do not pass an expression with side-effects directly to
+   this macro.
+
 .. c:macro:: Py_STRINGIFY(x)
 
    Convert ``x`` to a C string.  E.g. ``Py_STRINGIFY(123)`` returns
    ``"123"``.
 
    .. versionadded:: 3.4
 
+.. c:macro:: Py_ULL(number)
+
+   Similar to :c:macro:`Py_LL`, but *number* will be an ``unsigned long long``
+   literal instead. This is done by appending ``U`` to the result of ``Py_LL``.
+
+   In modern versions of Python, this macro is not very useful, as C99 and
+   later require the ``ULL``/``LLU`` suffixes to be valid for an integer.
+
 .. c:macro:: Py_UNREACHABLE()
 
    Use this when you have a code path that cannot be reached by design.
@@ -415,6 +501,16 @@ complete listing.
    This macro is intended for defining CPython's C API itself;
    extension modules should not use it for their own symbols.
 
+.. c:macro:: Py_VA_COPY
+
+   This is a :term:`soft deprecated` alias to the C99-standard ``va_copy``
+   function.
+
+   Historically, this would use a compiler-specific method to copy a ``va_list``.
+
+   .. versionchanged:: 3.6
+      This is now an alias to ``va_copy``.
+
 
 .. _api-objects: