Merge branch 'main' into deprecate-struct-init/78724

Author: skirpichev

Include/internal/pycore_optimizer.h

@@ -222,7 +222,7 @@ extern void _PyExecutor_Free(_PyExecutorObject *self);
 
 PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 #ifdef _Py_TIER2
-extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
+PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
 #endif
 
 int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode);
@@ -231,7 +231,7 @@ PyAPI_FUNC(int)
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,
     _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr,
     _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit,
-    int oparg);
+    int oparg, _PyExecutorObject *current_executor);
 
 void _PyJit_FinalizeTracing(PyThreadState *tstate);
 void _PyJit_TracerFree(_PyThreadStateImpl *_tstate);

Include/internal/pycore_tstate.h

@@ -31,6 +31,7 @@ typedef struct _PyJitTracerInitialState {
     struct _PyExitData *exit;
     PyCodeObject *code; // Strong
     PyFunctionObject *func; // Strong
+    struct _PyExecutorObject *executor; // Strong
     _Py_CODEUNIT *start_instr;
     _Py_CODEUNIT *close_loop_instr;
     _Py_CODEUNIT *jump_backward_instr;

Lib/test/support/import_helper.py

@@ -305,8 +305,8 @@ def ready_to_import(name=None, source=""):
         try:
             sys.path.insert(0, tempdir)
             yield name, path
-            sys.path.remove(tempdir)
         finally:
+            sys.path.remove(tempdir)
             if old_module is not None:
                 sys.modules[name] = old_module
             else:

Lib/test/test_capi/test_opt.py

@@ -139,6 +139,19 @@ def f():
         exe = get_first_executor(f)
         self.assertIsNone(exe)
 
+    def test_prev_executor_freed_while_tracing(self):
+        def f(start, end, way):
+            for x in range(start, end):
+                # For the first trace, create a bad branch on purpose to trace into.
+                # A side exit will form from here on the second trace.
+                y = way + way
+                if x >= TIER2_THRESHOLD:
+                    # Invalidate the first trace while tracing the second.
+                    _testinternalcapi.invalidate_executors(f.__code__)
+                    _testinternalcapi.clear_executor_deletion_list()
+        f(0, TIER2_THRESHOLD, 1)
+        f(1, TIER2_THRESHOLD + 1, 1.0)
+
 
 @requires_specialization
 @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds")

Lib/test/test_crossinterp.py

@@ -1,10 +1,12 @@
 import contextlib
 import itertools
 import sys
+import traceback
 import types
 import unittest
 import warnings
 
+from test import support
 from test.support import import_helper
 
 _testinternalcapi = import_helper.import_module('_testinternalcapi')
@@ -1491,5 +1493,54 @@ def test_builtin_objects(self):
         ])
 
 
+class CaptureExceptionTests(unittest.TestCase):
+
+    # Prevent crashes with incompatible TracebackException.format().
+    # Regression test for https://github.com/python/cpython/issues/143377.
+
+    def capture_with_formatter(self, exc, formatter):
+        with support.swap_attr(traceback.TracebackException, "format", formatter):
+            return _interpreters.capture_exception(exc)
+
+    def test_capture_exception(self):
+        captured = _interpreters.capture_exception(ValueError("hello"))
+
+        self.assertEqual(captured.type.__name__, "ValueError")
+        self.assertEqual(captured.type.__qualname__, "ValueError")
+        self.assertEqual(captured.type.__module__, "builtins")
+
+        self.assertEqual(captured.msg, "hello")
+        self.assertEqual(captured.formatted, "ValueError: hello")
+
+    def test_capture_exception_custom_format(self):
+        exc = ValueError("good bye!")
+        formatter = lambda self: ["hello\n", "world\n"]
+        captured = self.capture_with_formatter(exc, formatter)
+        self.assertEqual(captured.msg, "good bye!")
+        self.assertEqual(captured.formatted, "ValueError: good bye!")
+        self.assertEqual(captured.errdisplay, "hello\nworld")
+
+    @support.subTests("exc_lines", ([], ["x-no-nl"], ["x-no-nl", "y-no-nl"]))
+    def test_capture_exception_invalid_format(self, exc_lines):
+        formatter = lambda self: exc_lines
+        captured = self.capture_with_formatter(ValueError(), formatter)
+        self.assertEqual(captured.msg, "")
+        self.assertEqual(captured.formatted, "ValueError: ")
+        self.assertEqual(captured.errdisplay, "".join(exc_lines))
+
+    @unittest.skipUnless(
+        support.Py_DEBUG,
+        "printing subinterpreter unraisable exceptions requires DEBUG build",
+    )
+    def test_capture_exception_unraisable_exception(self):
+        formatter = lambda self: 1
+        with support.catch_unraisable_exception() as cm:
+            captured = self.capture_with_formatter(ValueError(), formatter)
+            self.assertNotHasAttr(captured, "errdisplay")
+            self.assertEqual(cm.unraisable.exc_type, TypeError)
+            self.assertEqual(str(cm.unraisable.exc_value),
+                             "can only join an iterable")
+
+
 if __name__ == '__main__':
     unittest.main()

Misc/NEWS.d/next/Core_and_Builtins/2026-01-04-16-56-17.gh-issue-143377.YJqMCa.rst

@@ -0,0 +1,2 @@
+Fix a crash in :func:`!_interpreters.capture_exception` when
+the exception is incorrectly formatted. Patch by Bénédikt Tran.

Misc/NEWS.d/next/Core_and_Builtins/2026-01-10-17-13-04.gh-issue-143604.BygbVT.rst

@@ -0,0 +1,2 @@
+Fix a reference counting issue in the JIT tracer where the current executor
+could be prematurely freed during tracing.

Modules/_testinternalcapi.c

@@ -1245,6 +1245,14 @@ invalidate_executors(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+clear_executor_deletion_list(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    _Py_ClearExecutorDeletionList(interp);
+    Py_RETURN_NONE;
+}
+
 static PyObject *
 get_exit_executor(PyObject *self, PyObject *arg)
 {
@@ -2562,6 +2570,7 @@ static PyMethodDef module_functions[] = {
 #ifdef _Py_TIER2
     {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},
     {"invalidate_executors", invalidate_executors, METH_O, NULL},
+    {"clear_executor_deletion_list", clear_executor_deletion_list, METH_NOARGS, NULL},
     {"get_exit_executor", get_exit_executor, METH_O, NULL},
 #endif
     {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc),

Python/bytecodes.c

@@ -2990,7 +2990,7 @@ dummy_func(
                     oparg >>= 8;
                     insert_exec_at--;
                 }
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg);
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg, NULL);
                 if (succ) {
                     ENTER_TRACING();
                 }
@@ -5525,7 +5525,7 @@ dummy_func(
                 // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG.
                 // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG.
                 // So setting it to anything else is wrong.
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg);
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg, previous_executor);
                 exit->temperature = restart_backoff_counter(exit->temperature);
                 if (succ) {
                     GOTO_TIER_ONE_CONTINUE_TRACING(target);

Python/ceval.c

@@ -1475,7 +1475,7 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
             tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&_tstate->policy);
         }
     }
-    else {
+    else if (tracer->initial_state.executor->vm_data.valid) {
         // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
         // to be valid to access.
         if (err <= 0) {