Faster side exits.

Author: markshannon

Include/cpython/pystate.h

@@ -198,6 +198,9 @@ struct _ts {
 
     PyObject *current_executor;
 
+    /* Internal to the JIT */
+    struct _PyExitData *jit_exit;
+
     uint64_t dict_global_version;
 
     /* Used to store/retrieve `threading.local` keys/values for this thread */

Include/internal/pycore_interp_structs.h

@@ -944,6 +944,7 @@ struct _is {
     bool jit;
     struct _PyExecutorObject *executor_list_head;
     struct _PyExecutorObject *executor_deletion_list_head;
+    struct _PyExecutorObject *cold_executor;
     int executor_deletion_list_remaining_capacity;
     size_t trace_run_counter;
     _rare_events rare_events;

Include/internal/pycore_optimizer.h

@@ -67,8 +67,9 @@ typedef struct {
 #endif
 } _PyUOpInstruction;
 
-typedef struct {
+typedef struct _PyExitData {
     uint32_t target;
+    uint16_t index;
     _Py_BackoffCounter temperature;
     struct _PyExecutorObject *executor;
 } _PyExitData;
@@ -354,6 +355,14 @@ PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 
 PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth);
 
+static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit)
+{
+    _PyExitData *exit0 = exit - exit->index;
+    return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits));
+}
+
+extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void);
+
 static inline int is_terminator(const _PyUOpInstruction *uop)
 {
     int opcode = uop->opcode;