Clean up labels

Author: Fidget-Spinner

Include/internal/pycore_opcode_metadata.h

@@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
 #endif
 
-int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr);
+int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing);
 
 int
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,

Include/internal/pycore_optimizer.h

@@ -1218,33 +1218,19 @@ dummy_func(
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
-            if (IS_JIT_TRACING()) {
-#if _Py_TIER2
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL);
-                LEAVE_TRACING();
-                int err = bail_tracing_and_jit(tstate, frame);
-                if (err < 0) {
-                    Py_DECREF(result);
-                    ERROR_IF(true);
-                }
-                return result;
-#endif
-            }
-            else {
 #if !_Py_TAIL_CALL_INTERP
-                assert(frame == &entry.frame);
+            assert(frame == &entry.frame);
 #endif
 #ifdef _Py_TIER2
-                _PyStackRef executor = frame->localsplus[0];
-                assert(tstate->current_executor == NULL);
-                if (!PyStackRef_IsNull(executor)) {
-                    tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
-                    PyStackRef_CLOSE(executor);
-                }
-#endif
-                LLTRACE_RESUME_FRAME();
-                return result;
+            _PyStackRef executor = frame->localsplus[0];
+            assert(tstate->current_executor == NULL);
+            if (!PyStackRef_IsNull(executor)) {
+                tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+                PyStackRef_CLOSE(executor);
             }
+#endif
+            LLTRACE_RESUME_FRAME();
+            return result;
         }
 
         // The stack effect here is a bit misleading.
@@ -3028,10 +3014,8 @@ dummy_func(
         tier1 inst(ENTER_EXECUTOR, (--)) {
             #ifdef _Py_TIER2
             if (IS_JIT_TRACING()) {
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
-                LEAVE_TRACING();
-                int err = bail_tracing_and_jit(tstate, frame);
-                ERROR_IF(err < 0);
+                next_instr = this_instr;
+                goto stop_tracing;
             }
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
@@ -5657,7 +5641,10 @@ dummy_func(
 #if _Py_TIER2
             assert(IS_JIT_TRACING());
             int opcode = next_instr->op.code;
-            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
+            bool stop_tracing = (opcode == WITH_EXCEPT_START ||
+                opcode == RERAISE || opcode == CLEANUP_THROW ||
+                opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT);
+            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing);
             if (full) {
                 LEAVE_TRACING();
                 int err = bail_tracing_and_jit(tstate, frame);
@@ -5677,9 +5664,9 @@ dummy_func(
                 _tstate->jit_state.prev_state.instr = next_instr;
             }
             _tstate->jit_state.prev_state.specialize_counter = 0;
-            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable));
-            if (_tstate->jit_state.prev_state.instr_code != prev_code) {
-                Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code);
+            PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
+            if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) {
+                Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code)));
             }
 
             _tstate->jit_state.prev_state.instr_frame = frame;
@@ -5691,6 +5678,19 @@ dummy_func(
 #endif
         }
 
+        label(stop_tracing) {
+#if _Py_TIER2
+            assert(IS_JIT_TRACING());
+            _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true);
+            LEAVE_TRACING();
+            int err = bail_tracing_and_jit(tstate, frame);
+            ERROR_IF(err < 0);
+            DISPATCH_GOTO_NON_TRACING();
+#else
+            Py_FatalError("JIT label executed in non-jit build.");
+#endif
+        }
+
 
 // END BYTECODES //
 

Python/bytecodes.c

@@ -1017,13 +1017,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
         // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
         // to be valid to access.
         if (err <= 0) {
-            // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again.
-            if (_tstate->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) {
-                exit->temperature = initial_unreachable_backoff_counter();
-            }
-            else {
-                exit->temperature = restart_backoff_counter(exit->temperature);
-            }
+            exit->temperature = restart_backoff_counter(exit->temperature);
         }
         else {
             exit->temperature = initial_temperature_backoff_counter();

Python/ceval.c

@@ -552,7 +552,8 @@ int
 _PyJit_translate_single_bytecode_to_trace(
     PyThreadState *tstate,
     _PyInterpreterFrame *frame,
-    _Py_CODEUNIT *next_instr)
+    _Py_CODEUNIT *next_instr,
+    bool stop_tracing)
 {
 
 #ifdef Py_DEBUG
@@ -571,11 +572,6 @@ _PyJit_translate_single_bytecode_to_trace(
     _PyUOpInstruction *trace = _tstate->jit_state.code_buffer;
     int max_length = _tstate->jit_state.prev_state.code_max_size;
 
-    int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL);
-    if (is_sys_tracing) {
-        goto full;
-    }
-
     _Py_CODEUNIT *this_instr =  _tstate->jit_state.prev_state.instr;
     _Py_CODEUNIT *target_instr = this_instr;
     uint32_t target = 0;
@@ -619,6 +615,17 @@ _PyJit_translate_single_bytecode_to_trace(
         DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]);
         goto unsupported;
     }
+
+    int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL);
+    if (is_sys_tracing) {
+        goto full;
+    }
+
+    if (stop_tracing) {
+        ADD_TO_TRACE(_DEOPT, 0, 0, target);
+        goto done;
+    }
+
     DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level);
 
 #ifdef Py_DEBUG
@@ -655,11 +662,6 @@ _PyJit_translate_single_bytecode_to_trace(
 
     // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom.
     if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) {
-        goto unsupported;
-    }
-
-    if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) {
-        DPRINTF(2, "Unsupported: strange control-flow\n");
         unsupported:
         {
             // Rewind to previous instruction and replace with _EXIT_TRACE.
@@ -773,9 +775,6 @@ _PyJit_translate_single_bytecode_to_trace(
              *  start with RESUME_CHECK */
             ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
             break;
-        case INTERPRETER_EXIT:
-            ADD_TO_TRACE(_DEOPT, 0, 0, target);
-            goto done;;
         default:
         {
             const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];