Merge branch 'main' into lazy

Author: pablogsal

Doc/library/asyncio-queue.rst

@@ -107,7 +107,7 @@ Queue
       The queue can no longer grow.
       Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`.
       Currently blocked callers of :meth:`~Queue.put` will be unblocked
-      and will raise :exc:`QueueShutDown` in the formerly blocked thread.
+      and will raise :exc:`QueueShutDown` in the formerly awaiting task.
 
       If *immediate* is false (the default), the queue can be wound
       down normally with :meth:`~Queue.get` calls to extract tasks

Lib/test/_test_atexit.py

@@ -135,6 +135,19 @@ def func():
         finally:
             atexit.unregister(func)
 
+    def test_eq_unregister_clear(self):
+        # Issue #112127: callback's __eq__ may call unregister or _clear
+        class Evil:
+            def __eq__(self, other):
+                action(other)
+                return NotImplemented
+
+        for action in atexit.unregister, lambda o: atexit._clear():
+            with self.subTest(action=action):
+                atexit.register(lambda: None)
+                atexit.unregister(Evil())
+                atexit._clear()
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_generators.py

@@ -290,6 +290,33 @@ def __iter__(self):
 
         self.assertEqual([1,2], list(i for i in C()))
 
+    def test_close_clears_frame(self):
+        # gh-142766: Test that closing a generator clears its frame
+        class DetectDelete:
+            def __init__(self):
+                DetectDelete.deleted = False
+
+            def __del__(self):
+                DetectDelete.deleted = True
+
+        def generator(arg):
+            yield
+
+        # Test a freshly created generator (not suspended)
+        g = generator(DetectDelete())
+        g.close()
+        self.assertTrue(DetectDelete.deleted)
+
+        # Test a suspended generator
+        g = generator(DetectDelete())
+        next(g)
+        g.close()
+        self.assertTrue(DetectDelete.deleted)
+
+        # Clear via gi_frame.clear()
+        g = generator(DetectDelete())
+        g.gi_frame.clear()
+        self.assertTrue(DetectDelete.deleted)
 
 class ModifyUnderlyingIterableTest(unittest.TestCase):
     iterables = [

Lib/test/test_profiling/test_sampling_profiler/test_integration.py

@@ -863,3 +863,98 @@ def test_async_aware_running_sees_only_cpu_task(self):
         self.assertGreater(cpu_percentage, 90.0,
             f"cpu_leaf should dominate samples in 'running' mode, "
             f"got {cpu_percentage:.1f}% ({cpu_leaf_samples}/{total})")
+
+
+def _generate_deep_generators_script(chain_depth=20, recurse_depth=150):
+    """Generate a script with deep nested generators for stress testing."""
+    lines = [
+        'import sys',
+        'sys.setrecursionlimit(5000)',
+        '',
+    ]
+    # Generate chain of yield-from functions
+    for i in range(chain_depth - 1):
+        lines.extend([
+            f'def deep_yield_chain_{i}(n):',
+            f'    yield ("L{i}", n)',
+            f'    yield from deep_yield_chain_{i + 1}(n)',
+            '',
+        ])
+    # Last chain function calls recursive_diver
+    lines.extend([
+        f'def deep_yield_chain_{chain_depth - 1}(n):',
+        f'    yield ("L{chain_depth - 1}", n)',
+        f'    yield from recursive_diver(n, {chain_depth})',
+        '',
+        'def recursive_diver(n, depth):',
+        '    yield (f"DIVE_{depth}", n)',
+        f'    if depth < {recurse_depth}:',
+        '        yield from recursive_diver(n, depth + 1)',
+        '    else:',
+        '        for i in range(5):',
+        '            yield (f"BOTTOM_{depth}", i)',
+        '',
+        'def oscillating_generator(iterations=1000):',
+        '    for i in range(iterations):',
+        '        yield ("OSCILLATE", i)',
+        '        yield from deep_yield_chain_0(i)',
+        '',
+        'def run_forever():',
+        '    while True:',
+        '        for _ in oscillating_generator(10):',
+        '            pass',
+        '',
+        '_test_sock.sendall(b"working")',
+        'run_forever()',
+    ])
+    return '\n'.join(lines)
+
+
+@requires_remote_subprocess_debugging()
+class TestDeepGeneratorFrameCache(unittest.TestCase):
+    """Test frame cache consistency with deep oscillating generator stacks."""
+
+    def test_all_stacks_share_same_base_frame(self):
+        """Verify all sampled stacks reach the entry point function.
+
+        When profiling deep generators that oscillate up and down the call
+        stack, every sample should include the entry point function
+        (run_forever) in its call chain. If the frame cache stores
+        incomplete stacks, some samples will be missing this base function,
+        causing broken flamegraphs.
+        """
+        script = _generate_deep_generators_script()
+        with test_subprocess(script, wait_for_working=True) as subproc:
+            collector = CollapsedStackCollector(sample_interval_usec=1, skip_idle=False)
+
+            with (
+                io.StringIO() as captured_output,
+                mock.patch("sys.stdout", captured_output),
+            ):
+                profiling.sampling.sample.sample(
+                    subproc.process.pid,
+                    collector,
+                    duration_sec=2,
+                )
+
+        samples_with_entry_point = 0
+        samples_without_entry_point = 0
+        total_samples = 0
+
+        for (call_tree, _thread_id), count in collector.stack_counter.items():
+            total_samples += count
+            if call_tree:
+                has_entry_point = call_tree and call_tree[0][2] == "<module>"
+                if has_entry_point:
+                    samples_with_entry_point += count
+                else:
+                    samples_without_entry_point += count
+
+        self.assertGreater(total_samples, 100,
+            f"Expected at least 100 samples, got {total_samples}")
+
+        self.assertEqual(samples_without_entry_point, 0,
+            f"Found {samples_without_entry_point}/{total_samples} samples "
+            f"missing the entry point function 'run_forever'. This indicates "
+            f"incomplete stacks are being returned, likely due to frame cache "
+            f"storing partial stack traces.")

Misc/ACKS

@@ -908,6 +908,7 @@ Jim Jewett
 Pedro Diaz Jimenez
 Orjan Johansen
 Fredrik Johansson
+Benjamin Johnson
 Benjamin K. Johnson
 Gregory K. Johnson
 Kent Johnson

Misc/NEWS.d/next/Core_and_Builtins/2025-12-16-11-56-20.gh-issue-142766.Uy2HTm.rst

@@ -0,0 +1 @@
+Clear the frame of a generator when :meth:`generator.close` is called.

Misc/NEWS.d/next/Library/2025-12-17-03-03-12.gh-issue-138122.m3EF9E.rst

@@ -0,0 +1,4 @@
+Fix incomplete stack traces in the Tachyon profiler's frame cache when
+profiling code with deeply nested generators. The frame cache now validates
+that stack traces reach the base frame before caching, preventing broken
+flamegraphs. Patch by Pablo Galindo.

Misc/NEWS.d/next/Library/2025-12-17-14-41-09.gh-issue-112127.13OHQk.rst

@@ -0,0 +1,2 @@
+Fix possible use-after-free in :func:`atexit.unregister` when the callback
+is unregistered during comparison.

Modules/_remote_debugging/_remote_debugging.h

@@ -279,6 +279,35 @@ typedef struct {
     size_t count;
 } StackChunkList;
 
+/*
+ * Context for frame chain traversal operations.
+ */
+typedef struct {
+    /* Inputs */
+    uintptr_t frame_addr;           // Starting frame address
+    uintptr_t base_frame_addr;      // Sentinel at bottom (for validation)
+    uintptr_t gc_frame;             // GC frame address (0 if not tracking)
+    uintptr_t last_profiled_frame;  // Last cached frame (0 if no cache)
+    StackChunkList *chunks;         // Pre-copied stack chunks
+
+    /* Outputs */
+    PyObject *frame_info;           // List to append FrameInfo objects
+    uintptr_t *frame_addrs;         // Array of visited frame addresses
+    Py_ssize_t num_addrs;           // Count of addresses collected
+    Py_ssize_t max_addrs;           // Capacity of frame_addrs array
+    uintptr_t last_frame_visited;   // Last frame address visited
+    int stopped_at_cached_frame;    // Whether we stopped at cached frame
+} FrameWalkContext;
+
+/*
+ * Context for code object parsing.
+ */
+typedef struct {
+    uintptr_t code_addr;            // Code object address in remote process
+    uintptr_t instruction_pointer;  // Current instruction pointer
+    int32_t tlbc_index;             // Thread-local bytecode index (free-threading)
+} CodeObjectContext;
+
 /* Function pointer types for iteration callbacks */
 typedef int (*thread_processor_func)(
     RemoteUnwinderObject *unwinder,
@@ -343,10 +372,7 @@ extern long read_py_long(RemoteUnwinderObject *unwinder, uintptr_t address);
 extern int parse_code_object(
     RemoteUnwinderObject *unwinder,
     PyObject **result,
-    uintptr_t address,
-    uintptr_t instruction_pointer,
-    uintptr_t *previous_frame,
-    int32_t tlbc_index
+    const CodeObjectContext *ctx
 );
 
 extern PyObject *make_location_info(
@@ -420,16 +446,7 @@ extern void *find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr);
 
 extern int process_frame_chain(
     RemoteUnwinderObject *unwinder,
-    uintptr_t initial_frame_addr,
-    StackChunkList *chunks,
-    PyObject *frame_info,
-    uintptr_t base_frame_addr,
-    uintptr_t gc_frame,
-    uintptr_t last_profiled_frame,
-    int *stopped_at_cached_frame,
-    uintptr_t *frame_addrs,
-    Py_ssize_t *num_addrs,
-    Py_ssize_t max_addrs
+    FrameWalkContext *ctx
 );
 
 /* Frame cache functions */
@@ -447,20 +464,19 @@ extern int frame_cache_lookup_and_extend(
     Py_ssize_t *num_addrs,
     Py_ssize_t max_addrs);
 // Returns: 1 = stored, 0 = not stored (graceful), -1 = error
+// Only stores complete stacks that reach base_frame_addr
 extern int frame_cache_store(
     RemoteUnwinderObject *unwinder,
     uint64_t thread_id,
     PyObject *frame_list,
     const uintptr_t *addrs,
-    Py_ssize_t num_addrs);
+    Py_ssize_t num_addrs,
+    uintptr_t base_frame_addr,
+    uintptr_t last_frame_visited);
 
 extern int collect_frames_with_cache(
     RemoteUnwinderObject *unwinder,
-    uintptr_t frame_addr,
-    StackChunkList *chunks,
-    PyObject *frame_info,
-    uintptr_t gc_frame,
-    uintptr_t last_profiled_frame,
+    FrameWalkContext *ctx,
     uint64_t thread_id);
 
 /* ============================================================================

Modules/_remote_debugging/code_objects.c

@@ -76,6 +76,7 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t
         PyErr_SetString(PyExc_RuntimeError, "TLBC array size exceeds maximum limit");
         return 0; // Invalid size
     }
+    assert(tlbc_size > 0 && tlbc_size <= MAX_TLBC_SIZE);
 
     // Allocate and read the entire TLBC array
     size_t array_data_size = tlbc_size * sizeof(void*);
@@ -156,8 +157,11 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L
     const uint8_t* ptr = (const uint8_t*)(linetable);
     uintptr_t addr = 0;
     int computed_line = firstlineno;  // Running accumulator, separate from output
+    const size_t MAX_LINETABLE_ENTRIES = 65536;
+    size_t entry_count = 0;
 
-    while (*ptr != '\0') {
+    while (*ptr != '\0' && entry_count < MAX_LINETABLE_ENTRIES) {
+        entry_count++;
         uint8_t first_byte = *(ptr++);
         uint8_t code = (first_byte >> 3) & 15;
         size_t length = (first_byte & 7) + 1;
@@ -277,12 +281,9 @@ make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *locati
 int
 parse_code_object(RemoteUnwinderObject *unwinder,
                   PyObject **result,
-                  uintptr_t address,
-                  uintptr_t instruction_pointer,
-                  uintptr_t *previous_frame,
-                  int32_t tlbc_index)
+                  const CodeObjectContext *ctx)
 {
-    void *key = (void *)address;
+    void *key = (void *)ctx->code_addr;
     CachedCodeMetadata *meta = NULL;
     PyObject *func = NULL;
     PyObject *file = NULL;
@@ -291,9 +292,9 @@ parse_code_object(RemoteUnwinderObject *unwinder,
 #ifdef Py_GIL_DISABLED
     // In free threading builds, code object addresses might have the low bit set
     // as a flag, so we need to mask it off to get the real address
-    uintptr_t real_address = address & (~1);
+    uintptr_t real_address = ctx->code_addr & (~1);
 #else
-    uintptr_t real_address = address;
+    uintptr_t real_address = ctx->code_addr;
 #endif
 
     if (unwinder && unwinder->code_object_cache != NULL) {
@@ -360,12 +361,12 @@ parse_code_object(RemoteUnwinderObject *unwinder,
         linetable = NULL;
     }
 
-    uintptr_t ip = instruction_pointer;
+    uintptr_t ip = ctx->instruction_pointer;
     ptrdiff_t addrq;
 
 #ifdef Py_GIL_DISABLED
     // Handle thread-local bytecode (TLBC) in free threading builds
-    if (tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) {
+    if (ctx->tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) {
         // No TLBC or no unwinder - use main bytecode directly
         addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive;
         goto done_tlbc;
@@ -383,10 +384,12 @@ parse_code_object(RemoteUnwinderObject *unwinder,
         tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation);
     }
 
-    if (tlbc_entry && tlbc_index < tlbc_entry->tlbc_array_size) {
+    if (tlbc_entry && ctx->tlbc_index < tlbc_entry->tlbc_array_size) {
+        assert(ctx->tlbc_index >= 0);
+        assert(tlbc_entry->tlbc_array_size > 0);
         // Use cached TLBC data
         uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t));
-        uintptr_t tlbc_bytecode_addr = entries[tlbc_index];
+        uintptr_t tlbc_bytecode_addr = entries[ctx->tlbc_index];
 
         if (tlbc_bytecode_addr != 0) {
             // Calculate offset from TLBC bytecode
@@ -401,8 +404,6 @@ parse_code_object(RemoteUnwinderObject *unwinder,
 done_tlbc:
 #else
     // Non-free-threaded build, always use the main bytecode
-    (void)tlbc_index; // Suppress unused parameter warning
-    (void)unwinder;   // Suppress unused parameter warning
     addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive;
 #endif
     ;  // Empty statement to avoid C23 extension warning