From fa5c6fd3b9244e81ed5627bd3b045fcca03923c6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Jan 2025 10:06:26 +0000 Subject: [PATCH 1/8] Make PyStackRef_CLOSE escaping. --- Include/internal/pycore_uop_metadata.h | 10 +- Lib/test/test_generated_cases.py | 27 +- Python/bytecodes.c | 65 +- Python/executor_cases.c.h | 852 ++++++++++++++---- Python/generated_cases.c.h | 983 +++++++++++++++------ Python/optimizer.c | 1 - Python/optimizer_bytecodes.c | 15 +- Python/optimizer_cases.c.h | 46 +- Tools/cases_generator/analyzer.py | 44 +- Tools/cases_generator/generators_common.py | 41 +- Tools/cases_generator/tier2_generator.py | 6 +- 11 files changed, 1493 insertions(+), 597 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 59740dbb57072e..82a4d26da0424e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -964,7 +964,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_METHOD_VERSION: return 0; case _EXPAND_METHOD: - return 2 + oparg; + return 0; case _CHECK_IS_NOT_PY_CALLABLE: return 0; case _CALL_NON_PY_GENERAL: @@ -972,7 +972,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return 0; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return 2 + oparg; + return 0; case _CHECK_PEP_523: return 0; case _CHECK_FUNCTION_EXACT_ARGS: @@ -1036,7 +1036,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_METHOD_VERSION_KW: return 0; case _EXPAND_METHOD_KW: - return 3 + oparg; + return 0; case _CHECK_IS_NOT_PY_CALLABLE_KW: return 0; case _CALL_KW_NON_PY: @@ -1062,7 +1062,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_OP: return 2; case _SWAP: - return 2 + (oparg-2); + return 0; case _GUARD_IS_TRUE_POP: return 1; case _GUARD_IS_FALSE_POP: @@ -1110,7 +1110,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _DEOPT: return 0; case _ERROR_POP_N: - return oparg; + return 0; case _TIER2_RESUME_CHECK: return 0; default: diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index bb78dd9af83b62..0932c41a0a3265 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -538,7 +538,9 @@ def test_error_if_plain(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) goto label; + if (cond) { + goto label; + } DISPATCH(); } """ @@ -555,7 +557,9 @@ def test_error_if_plain_with_comment(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) goto label; + if (cond) { + goto label; + } // Comment is ok DISPATCH(); } @@ -582,7 +586,11 @@ def test_error_if_pop(self): right = stack_pointer[-1]; left = stack_pointer[-2]; SPAM(left, right); - if (cond) goto pop_2_label; + if (cond) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto label; + } res = 0; stack_pointer[-2] = res; stack_pointer += -1; @@ -611,7 +619,12 @@ def test_error_if_pop_with_result(self): right = stack_pointer[-1]; left = stack_pointer[-2]; res = SPAM(left, right); - if (cond) goto pop_2_label; + if (cond) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto label; + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1392,7 +1405,11 @@ def test_pop_on_error_peeks(self): // THIRD { // Mark j and k as used - if (cond) goto pop_2_error; + if (cond) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5f0be8d3feefd4..e6ae7026ee4dcb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2120,8 +2120,7 @@ dummy_func( PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; } - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(global_super_st); + DECREF_INPUTS(); attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -3635,15 +3634,14 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD, (callable[1], null[1], unused[oparg] -- method[1], self[1], unused[oparg])) { + op(_EXPAND_METHOD, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); - DEAD(null); + assert(PyStackRef_IsNull(self_or_null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); PyStackRef_CLOSE(temp); } @@ -3704,13 +3702,13 @@ dummy_func( EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], null[1], unused[oparg] -- func[1], self[1], unused[oparg])) { - DEAD(null); + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { + assert(PyStackRef_IsNull(self_or_null[0])); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); PyStackRef_CLOSE(temp); } @@ -4135,8 +4133,9 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } @@ -4147,8 +4146,8 @@ dummy_func( EXIT_IF(meth->ml_flags != METH_O); // CPython promises to check all non-vectorcall function calls. EXIT_IF(tstate->c_recursion_remaining <= 0); - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; EXIT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)); STAT_INC(CALL, hit); @@ -4159,11 +4158,7 @@ dummy_func( PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); - DEAD(args); - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4450,15 +4445,14 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD_KW, (callable[1], null[1], unused[oparg], unused -- method[1], self[1], unused[oparg], unused)) { + op(_EXPAND_METHOD_KW, (callable[1], self_or_null[1], unused[oparg], unused -- callable[1], self_or_null[1], unused[oparg], unused)) { + assert(PyStackRef_IsNull(self_or_null[0])); _PyStackRef callable_s = callable[0]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - - assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); PyStackRef_CLOSE(callable_s); } @@ -4564,7 +4558,8 @@ dummy_func( } } - op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st -- result)) { + op(_DO_CALL_FUNCTION_EX, (func_st, null, callargs_st, kwargs_st -- result)) { + (void)null; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); // DICT_MERGE is called before this opcode if there are kwargs. @@ -4635,8 +4630,8 @@ dummy_func( result_o = PyObject_Call(func, callargs, kwargs); } PyStackRef_XCLOSE(kwargs_st); - DEAD(kwargs_st); PyStackRef_CLOSE(callargs_st); + DEAD(null); PyStackRef_CLOSE(func_st); ERROR_IF(result_o == NULL, error); result = PyStackRef_FromPyObjectSteal(result_o); @@ -4773,12 +4768,11 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; - pure inst(SWAP, (bottom_in, unused[oparg-2], top_in -- - top_out, unused[oparg-2], bottom_out)) { - bottom_out = bottom_in; - DEAD(bottom_in); - top_out = top_in; - DEAD(top_in); + pure inst(SWAP, (bottom[1], unused[oparg-2], top[1] -- + bottom[1], unused[oparg-2], top[1])) { + _PyStackRef temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; assert(oparg >= 2); } @@ -5138,7 +5132,8 @@ dummy_func( EXIT_TO_TIER1(); } - tier2 op(_ERROR_POP_N, (target/2, unused[oparg] --)) { + tier2 op(_ERROR_POP_N, (target/2 --)) { + assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); GOTO_UNWIND(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4addfbcf6d6419..a8533578860934 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -19,7 +19,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) JUMP_TO_ERROR(); + if (err != 0) { + JUMP_TO_ERROR(); + } } break; } @@ -33,7 +35,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) JUMP_TO_ERROR(); + if (err != 0) { + JUMP_TO_ERROR(); + } } } break; @@ -81,7 +85,9 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) JUMP_TO_ERROR(); + if (1) { + JUMP_TO_ERROR(); + } } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -422,9 +428,11 @@ * This has the benign side effect that if value is * finalized it will see the location as the FOR_ITER's. */ - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -451,7 +459,11 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; @@ -476,7 +488,11 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (err < 0) JUMP_TO_ERROR(); + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; @@ -587,7 +603,11 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; @@ -647,7 +667,11 @@ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -669,7 +693,11 @@ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -691,7 +719,11 @@ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -754,7 +786,11 @@ ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -777,7 +813,11 @@ ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -800,7 +840,11 @@ ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -840,7 +884,11 @@ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -887,7 +935,11 @@ PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) JUMP_TO_ERROR(); + if (PyStackRef_IsNull(*target_local)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -957,7 +1009,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -993,11 +1049,17 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1031,7 +1093,11 @@ } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); break; @@ -1079,10 +1145,14 @@ Py_INCREF(res_o); #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1121,10 +1191,14 @@ STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1160,10 +1234,14 @@ assert(res_o != NULL); Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(tuple_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1192,7 +1270,11 @@ } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); - if (rc <= 0) JUMP_TO_ERROR(); + if (rc <= 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1262,7 +1344,11 @@ list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); - if (err < 0) JUMP_TO_ERROR(); + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1279,7 +1365,11 @@ PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1299,7 +1389,11 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; @@ -1346,10 +1440,10 @@ assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); Py_DECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); break; @@ -1373,10 +1467,14 @@ PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); - if (err) JUMP_TO_ERROR(); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -1392,7 +1490,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1408,7 +1510,11 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; @@ -1429,7 +1535,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -1481,13 +1591,21 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) JUMP_TO_ERROR(); + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { stack_pointer += -1; @@ -1499,7 +1617,9 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -1532,7 +1652,11 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) JUMP_TO_ERROR(); + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; @@ -1653,13 +1777,17 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) JUMP_TO_ERROR(); + if (err < 0) { + JUMP_TO_ERROR(); + } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -1681,7 +1809,11 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1694,7 +1826,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(v); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1738,7 +1874,11 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) JUMP_TO_ERROR(); + if (res == 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1842,7 +1982,11 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) JUMP_TO_ERROR(); + if (res == 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); break; @@ -1861,7 +2005,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1876,7 +2024,11 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1891,7 +2043,11 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) JUMP_TO_ERROR(); + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1925,7 +2081,9 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -1943,7 +2101,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) JUMP_TO_ERROR(); + if (v_o == NULL) { + JUMP_TO_ERROR(); + } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -1959,7 +2119,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR(); + if (PyStackRef_IsNull(*res)) { + JUMP_TO_ERROR(); + } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2103,7 +2265,9 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) JUMP_TO_ERROR(); + if (1) { + JUMP_TO_ERROR(); + } } SETLOCAL(oparg, PyStackRef_NULL); break; @@ -2167,9 +2331,15 @@ JUMP_TO_ERROR(); } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2182,7 +2352,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -2230,14 +2402,22 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - if (str_o == NULL) JUMP_TO_ERROR(); + if (str_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -2303,7 +2483,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -2323,7 +2507,11 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (err < 0) JUMP_TO_ERROR(); + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2341,7 +2529,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } int err = 0; for (int i = 0; i < oparg; i++) { @@ -2360,7 +2552,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -2379,7 +2573,11 @@ for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -2391,7 +2589,11 @@ for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (map_o == NULL) JUMP_TO_ERROR(); + if (map_o == NULL) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -2406,24 +2608,32 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) JUMP_TO_ERROR(); + if (err < 0) { + JUMP_TO_ERROR(); + } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) JUMP_TO_ERROR(); + if (ann_dict == NULL) { + JUMP_TO_ERROR(); + } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) JUMP_TO_ERROR(); + if (err) { + JUMP_TO_ERROR(); + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2456,7 +2666,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -2483,7 +2697,11 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -2510,7 +2728,11 @@ PyStackRef_AsPyObjectSteal(value) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) JUMP_TO_ERROR(); + if (err != 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -2547,7 +2769,11 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) JUMP_TO_ERROR(); + if (attr == NULL) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -2591,11 +2817,17 @@ if (method_found) { self_or_null = self_st; // transfer ownership } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); self_or_null = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -2635,7 +2867,11 @@ meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - if (attr_o == NULL) JUMP_TO_ERROR(); + if (attr_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } self_or_null[0] = PyStackRef_NULL; } } @@ -2645,7 +2881,11 @@ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (attr_o == NULL) JUMP_TO_ERROR(); + if (attr_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; @@ -2784,8 +3024,10 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -3001,10 +3243,10 @@ _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); break; @@ -3071,10 +3313,10 @@ // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); break; @@ -3096,10 +3338,10 @@ PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); break; @@ -3120,7 +3362,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } if (oparg & 16) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3128,7 +3374,9 @@ int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) JUMP_TO_ERROR(); + if (res_bool < 0) { + JUMP_TO_ERROR(); + } res = res_bool ? PyStackRef_True : PyStackRef_False; } else { @@ -3256,7 +3504,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) JUMP_TO_ERROR(); + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3284,7 +3536,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) JUMP_TO_ERROR(); + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3311,7 +3567,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) JUMP_TO_ERROR(); + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3334,7 +3594,11 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3344,9 +3608,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (res < 0) JUMP_TO_ERROR(); + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) JUMP_TO_ERROR(); + if (match_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3377,7 +3649,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3403,7 +3679,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -3420,7 +3700,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -3455,9 +3737,13 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) JUMP_TO_ERROR(); + if (len_i < 0) { + JUMP_TO_ERROR(); + } PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) JUMP_TO_ERROR(); + if (len_o == NULL) { + JUMP_TO_ERROR(); + } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; @@ -3491,7 +3777,11 @@ attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); + if (_PyErr_Occurred(tstate)) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } // Error! attrs = PyStackRef_None; // Failure! } @@ -3536,7 +3826,9 @@ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) JUMP_TO_ERROR(); + if (values_or_none_o == NULL) { + JUMP_TO_ERROR(); + } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -3553,7 +3845,11 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) JUMP_TO_ERROR(); + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; @@ -3773,7 +4069,9 @@ r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); - if (res == NULL) JUMP_TO_ERROR(); + if (res == NULL) { + JUMP_TO_ERROR(); + } next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; @@ -3833,7 +4131,9 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - if (true) JUMP_TO_ERROR(); + if (true) { + JUMP_TO_ERROR(); + } } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -3886,7 +4186,9 @@ PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -4069,7 +4371,9 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } break; } @@ -4175,23 +4479,21 @@ } case _EXPAND_METHOD: { - _PyStackRef *null; + _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *method; - _PyStackRef *self; oparg = CURRENT_OPARG(); - null = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); + assert(PyStackRef_IsNull(self_or_null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -4238,7 +4540,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -4253,7 +4559,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4279,21 +4589,20 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _PyStackRef *null; + _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *func; - _PyStackRef *self; oparg = CURRENT_OPARG(); - null = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; + assert(PyStackRef_IsNull(self_or_null[0])); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -4525,10 +4834,12 @@ } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - PyStackRef_CLOSE(arg); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -4556,11 +4867,17 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4589,11 +4906,17 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4645,7 +4968,9 @@ self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; init[0] = PyStackRef_FromPyObjectNew(init_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -4738,7 +5063,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -4749,7 +5078,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4799,12 +5132,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4844,7 +5185,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -4859,7 +5204,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4906,7 +5255,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -4918,7 +5271,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4965,11 +5322,17 @@ if (res_o == NULL) { GOTO_ERROR(error); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5050,17 +5413,25 @@ STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); - if (err) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -5075,8 +5446,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5098,8 +5470,8 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); @@ -5115,10 +5487,16 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) JUMP_TO_ERROR(); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -5167,7 +5545,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -5181,7 +5563,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -5238,12 +5624,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5289,7 +5683,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -5303,7 +5701,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -5335,7 +5737,9 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } kwnames_out = kwnames_in; stack_pointer[-1] = kwnames_out; @@ -5374,10 +5778,14 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { JUMP_TO_ERROR(); @@ -5436,23 +5844,21 @@ } case _EXPAND_METHOD_KW: { - _PyStackRef *null; + _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *method; - _PyStackRef *self; oparg = CURRENT_OPARG(); - null = &stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = &stack_pointer[-3 - oparg]; - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; + assert(PyStackRef_IsNull(self_or_null[0])); _PyStackRef callable_s = callable[0]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -5502,7 +5908,11 @@ PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - if (true) JUMP_TO_ERROR(); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -5512,7 +5922,11 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); @@ -5520,10 +5934,14 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5558,8 +5976,14 @@ JUMP_TO_ERROR(); } kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-2] = tuple; stack_pointer[-1] = kwargs_out; @@ -5577,12 +6001,20 @@ PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + JUMP_TO_ERROR(); + } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -5615,7 +6047,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) JUMP_TO_ERROR(); + if (gen == NULL) { + JUMP_TO_ERROR(); + } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -5650,7 +6084,11 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (slice_o == NULL) JUMP_TO_ERROR(); + if (slice_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; @@ -5669,10 +6107,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (result_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_ERROR(); + } result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -5687,14 +6133,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (res_o == NULL) JUMP_TO_ERROR(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -5709,7 +6165,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -5745,7 +6205,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); - if (res_o == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -5754,18 +6218,15 @@ } case _SWAP: { - _PyStackRef top_in; - _PyStackRef bottom_in; - _PyStackRef top_out; - _PyStackRef bottom_out; + _PyStackRef *top; + _PyStackRef *bottom; oparg = CURRENT_OPARG(); - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; + top = &stack_pointer[-1]; + bottom = &stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; break; } @@ -5818,9 +6279,11 @@ val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); if (!is_none) { - PyStackRef_CLOSE(val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5835,9 +6298,11 @@ _PyStackRef val; val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); - PyStackRef_CLOSE(val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); if (is_none) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5975,9 +6440,15 @@ _PyStackRef value; pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); + stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectImmortal(ptr); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -6144,8 +6615,7 @@ case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + assert(oparg == 0); _PyFrame_SetStackPointer(frame, stack_pointer); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ad044e62a38b1c..e84e71ae451777 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -57,7 +57,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -95,7 +99,11 @@ ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -132,7 +140,11 @@ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -169,7 +181,11 @@ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -274,7 +290,11 @@ PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + if (PyStackRef_IsNull(*target_local)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -316,7 +336,11 @@ ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -353,7 +377,11 @@ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -391,7 +419,11 @@ ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -428,7 +460,11 @@ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -477,12 +513,18 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -525,7 +567,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -560,7 +606,11 @@ } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); - if (rc <= 0) goto pop_2_error; + if (rc <= 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -660,10 +710,14 @@ Py_INCREF(res_o); #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -692,10 +746,14 @@ STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -724,10 +782,14 @@ assert(res_o != NULL); Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(tuple_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -762,11 +824,9 @@ for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -804,11 +864,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } int err = 0; for (int i = 0; i < oparg; i++) { @@ -874,11 +932,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -968,7 +1024,9 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } } // _DO_CALL @@ -1016,11 +1074,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -1074,7 +1130,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1132,7 +1190,9 @@ self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; init[0] = PyStackRef_FromPyObjectNew(init_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // _CREATE_INIT_FRAME { @@ -1190,8 +1250,6 @@ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *null; - _PyStackRef *func; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -1209,19 +1267,20 @@ } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _CHECK_FUNCTION_VERSION { - callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); @@ -1230,7 +1289,6 @@ } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -1293,8 +1351,6 @@ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *null; - _PyStackRef *method; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -1317,23 +1373,22 @@ } // _EXPAND_METHOD { - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; + self_or_null = null; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); + assert(PyStackRef_IsNull(self_or_null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _PY_FRAME_GENERAL { args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -1419,11 +1474,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -1452,7 +1505,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1499,11 +1554,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -1536,7 +1589,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1587,11 +1642,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -1621,7 +1674,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1669,11 +1724,15 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1683,19 +1742,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1713,6 +1774,7 @@ _PyStackRef tuple; _PyStackRef kwargs_out; _PyStackRef func_st; + _PyStackRef null; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef result; @@ -1740,15 +1802,23 @@ goto error; } kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } // _DO_CALL_FUNCTION_EX { kwargs_st = kwargs_out; callargs_st = tuple; + null = stack_pointer[-3]; func_st = func; + (void)null; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. @@ -1806,7 +1876,7 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -1814,7 +1884,7 @@ nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -1833,12 +1903,24 @@ result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -1846,19 +1928,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 3; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1875,7 +1959,11 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -1898,7 +1986,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -2001,7 +2093,9 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } kwnames_out = kwnames_in; } @@ -2035,9 +2129,13 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2057,11 +2155,9 @@ PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2119,8 +2215,6 @@ _PyStackRef *callable; _PyStackRef *null; _PyStackRef kwnames; - _PyStackRef *method; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2143,24 +2237,23 @@ } // _EXPAND_METHOD_KW { - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); _PyStackRef callable_s = callable[0]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _PY_FRAME_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -2180,10 +2273,14 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { goto error; @@ -2260,11 +2357,9 @@ PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -2274,7 +2369,11 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); @@ -2283,7 +2382,7 @@ PyStackRef_CLOSE(args[_i]); } if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } @@ -2294,19 +2393,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2359,10 +2460,14 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { goto error; @@ -2434,11 +2539,17 @@ if (res_o == NULL) { GOTO_ERROR(error); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2467,17 +2578,25 @@ STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); - if (err) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2520,11 +2639,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -2556,7 +2673,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2606,11 +2725,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -2642,7 +2759,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2694,11 +2813,15 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2708,19 +2831,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2743,8 +2868,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -2754,8 +2880,8 @@ DEOPT_IF(meth->ml_flags != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type), CALL); STAT_INC(CALL, hit); @@ -2768,9 +2894,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2789,7 +2917,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2840,11 +2970,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -2877,7 +3005,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3072,8 +3202,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3081,19 +3217,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3123,8 +3261,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3132,19 +3276,21 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3170,10 +3316,12 @@ DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - PyStackRef_CLOSE(arg); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -3195,7 +3343,9 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3205,9 +3355,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (res < 0) goto pop_2_error; + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) goto pop_2_error; + if (match_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3241,7 +3399,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3330,7 +3490,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } if (oparg & 16) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3338,7 +3502,9 @@ int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) goto error; + if (res_bool < 0) { + goto error; + } res = res_bool ? PyStackRef_True : PyStackRef_False; } else { @@ -3509,7 +3675,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } stack_pointer[-2] = b; @@ -3538,7 +3708,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3567,7 +3741,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3588,10 +3766,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (result_o == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3639,7 +3825,11 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (err) goto pop_1_error; + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3749,7 +3939,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_2_error; + if (err) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3776,7 +3970,9 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3809,7 +4005,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3858,9 +4056,11 @@ * This has the benign side effect that if value is * finalized it will see the location as the FOR_ITER's. */ - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -3960,14 +4160,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3985,7 +4195,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -4188,7 +4402,9 @@ r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); - if (res == NULL) goto error; + if (res == NULL) { + goto error; + } next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; @@ -4267,13 +4483,19 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { stack_pointer += -1; @@ -4323,7 +4545,11 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4341,7 +4567,11 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4358,9 +4588,13 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) goto error; + if (len_i < 0) { + goto error; + } PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; + if (len_o == NULL) { + goto error; + } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; @@ -4422,7 +4656,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -4447,7 +4683,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -4481,7 +4721,9 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } } // _MONITOR_CALL @@ -4509,7 +4751,9 @@ frame, this_instr, function, arg0 ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } // _DO_CALL { @@ -4555,11 +4799,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -4613,7 +4855,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4651,7 +4895,9 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); goto PREDICTED_CALL_KW; } @@ -4701,10 +4947,12 @@ } } val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4754,7 +5002,9 @@ int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) goto error; + if (next_opcode < 0) { + goto error; + } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); @@ -4778,7 +5028,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _MONITOR_JUMP_BACKWARD @@ -4867,9 +5119,11 @@ _PyStackRef iter; iter = stack_pointer[-1]; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - PyStackRef_CLOSE(iter); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4902,7 +5156,9 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); } @@ -4917,7 +5173,9 @@ int jump = !PyStackRef_IsNone(value_stackref); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } DISPATCH(); @@ -4953,7 +5211,9 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4992,7 +5252,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } } @@ -5002,7 +5264,9 @@ int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; @@ -5027,7 +5291,9 @@ tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } // _RETURN_VALUE { @@ -5172,7 +5438,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _JUMP_BACKWARD @@ -5197,7 +5465,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) goto error; + if (optimized < 0) { + goto error; + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5248,7 +5518,11 @@ list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); - if (err < 0) goto pop_1_error; + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5282,7 +5556,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -5347,7 +5623,11 @@ meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } self_or_null[0] = PyStackRef_NULL; } } @@ -5357,7 +5637,11 @@ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -5708,14 +5992,16 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -5986,7 +6272,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, @@ -6190,9 +6478,15 @@ goto error; } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6209,7 +6503,11 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) goto pop_1_error; + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -6241,13 +6539,17 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (v_o == NULL) { /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( @@ -6302,7 +6604,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; + if (PyStackRef_IsNull(*res)) { + goto error; + } } // _PUSH_NULL_CONDITIONAL { @@ -6443,7 +6747,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) goto error; + if (v_o == NULL) { + goto error; + } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -6550,7 +6856,9 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + goto error; } } // we make no attempt to optimize here; specializations should @@ -6582,7 +6890,11 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; + if (super == NULL) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -6590,7 +6902,9 @@ PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; + if (attr_o == NULL) { + goto error; + } attr = PyStackRef_FromPyObjectSteal(attr_o); } // _PUSH_NULL_CONDITIONAL @@ -6631,7 +6945,11 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) goto pop_3_error; + if (attr == NULL) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -6673,11 +6991,17 @@ if (method_found) { self_or_null = self_st; // transfer ownership } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); self_or_null = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -6713,12 +7037,20 @@ PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + goto error; + } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6743,7 +7075,11 @@ PyStackRef_AsPyObjectSteal(value) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto pop_2_error; + if (err != 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6777,7 +7113,11 @@ attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; + if (_PyErr_Occurred(tstate)) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } // Error! attrs = PyStackRef_None; // Failure! } @@ -6801,7 +7141,9 @@ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) goto error; + if (values_or_none_o == NULL) { + goto error; + } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -7124,7 +7466,9 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7171,7 +7515,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } } @@ -7208,7 +7554,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) goto error; + if (gen == NULL) { + goto error; + } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -7339,13 +7687,21 @@ } else { PyStackRef_CLOSE(v); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[-1] = retval; + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7417,18 +7773,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) goto error; + if (ann_dict == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7451,7 +7813,11 @@ PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7494,7 +7860,11 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (err < 0) goto pop_1_error; + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7538,7 +7908,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; + if (err) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7593,10 +7967,10 @@ _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -7630,10 +8004,10 @@ PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -7696,10 +8070,10 @@ // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -7776,7 +8150,11 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7797,7 +8175,9 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7810,7 +8190,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7857,7 +8241,11 @@ } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); - if (err) goto pop_4_error; + if (err) { + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -7902,7 +8290,11 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; + if (err) { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -7929,10 +8321,14 @@ PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); - if (err) goto pop_3_error; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } DISPATCH(); } @@ -7967,10 +8363,10 @@ assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); Py_DECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); @@ -7980,17 +8376,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - _PyStackRef bottom_in; - _PyStackRef top_in; - _PyStackRef top_out; - _PyStackRef bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; + _PyStackRef *bottom; + _PyStackRef *top; + top = &stack_pointer[-1]; + bottom = &stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; DISPATCH(); } @@ -8027,7 +8420,11 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; + if (err < 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; @@ -8174,7 +8571,11 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -8191,7 +8592,11 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -8224,7 +8629,11 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -8266,7 +8675,11 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -8395,7 +8808,9 @@ PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; diff --git a/Python/optimizer.c b/Python/optimizer.c index e3950843964f11..ff008d1db95976 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1046,7 +1046,6 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_error = next_spare; current_error_target = target; make_exit(&buffer[next_spare], _ERROR_POP_N, 0); - buffer[next_spare].oparg = popped; buffer[next_spare].operand0 = target; next_spare++; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index fb14c4b7c8645b..91573e82841cc9 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -512,10 +512,11 @@ dummy_func(void) { top = bottom; } - op(_SWAP, (bottom_in, unused[oparg-2], top_in -- - top_out, unused[oparg-2], bottom_out)) { - bottom_out = bottom_in; - top_out = top_in; + op(_SWAP, (bottom[1], unused[oparg-2], top[1] -- bottom[1], unused[oparg-2], top[1])) { + JitOptSymbol *temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; + assert(oparg >= 2); } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { @@ -629,10 +630,10 @@ dummy_func(void) { ctx->done = true; } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { (void)callable; - func = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + callable[0] = sym_new_not_null(ctx); + self_or_null[0] = sym_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2497754745c28b..144c7c503b2661 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1837,12 +1837,6 @@ } case _EXPAND_METHOD: { - JitOptSymbol **method; - JitOptSymbol **self; - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - method[0] = sym_new_not_null(ctx); - self[0] = sym_new_not_null(ctx); break; } @@ -1870,15 +1864,13 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *callable; - JitOptSymbol *func; - JitOptSymbol *self; - callable = stack_pointer[-2 - oparg]; + JitOptSymbol **self_or_null; + JitOptSymbol **callable; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; (void)callable; - func = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; + callable[0] = sym_new_not_null(ctx); + self_or_null[0] = sym_new_not_null(ctx); break; } @@ -2214,12 +2206,6 @@ } case _EXPAND_METHOD_KW: { - JitOptSymbol **method; - JitOptSymbol **self; - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; - method[0] = sym_new_not_null(ctx); - self[0] = sym_new_not_null(ctx); break; } @@ -2410,16 +2396,14 @@ } case _SWAP: { - JitOptSymbol *top_in; - JitOptSymbol *bottom_in; - JitOptSymbol *top_out; - JitOptSymbol *bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; + JitOptSymbol **top; + JitOptSymbol **bottom; + top = &stack_pointer[-1]; + bottom = &stack_pointer[-2 - (oparg-2)]; + JitOptSymbol *temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; + assert(oparg >= 2); break; } @@ -2639,8 +2623,6 @@ } case _ERROR_POP_N: { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index bc9c42e045a610..14d40c6d99e2e4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -5,9 +5,16 @@ import re from typing import Optional +@dataclass +class EscapingCall: + start: lexer.Token + call: lexer.Token + end: lexer.Token + kills: lexer.Token | None + @dataclass class Properties: - escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] + escaping_calls: dict[lexer.Token, EscapingCall] escapes: bool error_with_pop: bool error_without_pop: bool @@ -41,7 +48,7 @@ def dump(self, indent: str) -> None: @staticmethod def from_list(properties: list["Properties"]) -> "Properties": - escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} + escaping_calls: dict[lexer.Token, EscapingCall] = {} for p in properties: escaping_calls.update(p.escaping_calls) return Properties( @@ -330,6 +337,16 @@ def convert_stack_item( cond = replace_op_arg_1 return StackItem(item.name, item.type, cond, item.size) +def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]): + "Unused items cannot be on the stack above used, non-peek items" + seen_unused = False + for item in reversed(stack): + if item.name == "unused": + seen_unused = True + elif seen_unused and not item.peek: + raise analysis_error(f"Cannot have used input '{item.name}' below an unused value on the stack", input_names[item.name]) + + def analyze_stack( op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None @@ -374,6 +391,7 @@ def analyze_stack( for output in outputs: if variable_used(op, output.name): output.used = True + check_unused(inputs, input_names) return StackEffect(inputs, outputs) @@ -550,7 +568,6 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyStackRef_AsPyObjectNew", "PyStackRef_AsPyObjectSteal", "PyStackRef_CLEAR", - "PyStackRef_CLOSE", "PyStackRef_CLOSE_SPECIALIZED", "PyStackRef_DUP", "PyStackRef_False", @@ -666,8 +683,8 @@ def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: if tkn.kind == "SEMI": return node.block.tokens[idx+1] -def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple[lexer.Token, lexer.Token]]) -> None: - calls = {escapes[t][0] for t in escapes} +def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, EscapingCall]) -> None: + calls = {escapes[t].call for t in escapes} in_if = 0 tkn_iter = iter(instr.block.tokens) for tkn in tkn_iter: @@ -685,8 +702,8 @@ def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple elif tkn in calls and in_if: raise analysis_error(f"Escaping call '{tkn.text} in condition", tkn) -def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[lexer.Token, lexer.Token]]: - result: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} +def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, EscapingCall]: + result: dict[lexer.Token, EscapingCall] = {} tokens = instr.block.tokens for idx, tkn in enumerate(tokens): try: @@ -721,9 +738,17 @@ def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[le continue elif tkn.kind != "RBRACKET": continue + if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): + if len(tokens) <= idx+2: + raise analysis_error("'(' at end", next_tkn) + kills = tokens[idx+2] + if kills.kind != "IDENTIFIER": + raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) + else: + kills = None start = find_stmt_start(instr, idx) end = find_stmt_end(instr, idx) - result[start] = tkn, end + result[start] = EscapingCall(start, tkn, end, kills) check_escaping_calls(instr, result) return result @@ -822,9 +847,6 @@ def compute_properties(op: parser.InstDef) -> Properties: variable_used(op, "Py_DECREF") or variable_used(op, "Py_XDECREF") or variable_used(op, "Py_CLEAR") or - variable_used(op, "PyStackRef_CLOSE") or - variable_used(op, "PyStackRef_XCLOSE") or - variable_used(op, "PyStackRef_CLEAR") or variable_used(op, "SETLOCAL") ) return Properties( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index f1f166ae104ba5..ab21d01244b5cd 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -120,8 +120,8 @@ def __init__(self, out: CWriter): "SYNC_SP": self.sync_sp, "SAVE_STACK": self.save_stack, "RELOAD_STACK": self.reload_stack, - "PyStackRef_CLOSE": self.stackref_close, - "PyStackRef_XCLOSE": self.stackref_close, + #"PyStackRef_CLOSE": self.stackref_close, + #"PyStackRef_XCLOSE": self.stackref_close, "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close_specialized, "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, @@ -188,30 +188,16 @@ def error_if( self.out.emit_at("if ", tkn) self.emit(lparen) emit_to(self.out, tkn_iter, "COMMA") - self.out.emit(") ") + self.out.emit(") {") label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon storage.clear_inputs("at ERROR_IF") - c_offset = storage.stack.peek_offset() - try: - offset = -int(c_offset) - except ValueError: - offset = -1 - if offset > 0: - self.out.emit(f"goto pop_{offset}_") - self.out.emit(label) - self.out.emit(";\n") - elif offset == 0: - self.out.emit("goto ") - self.out.emit(label) - self.out.emit(";\n") - else: - self.out.emit("{\n") - storage.copy().flush(self.out) - self.out.emit("goto ") - self.out.emit(label) - self.out.emit(";\n") + storage.copy().flush(self.out) + self.out.emit("goto ") + self.out.emit(label) + self.out.emit(";\n") + if not unconditional: self.out.emit("}\n") return not unconditional @@ -590,9 +576,15 @@ def _emit_block( self.out.start_line() line = tkn.line if tkn in escaping_calls: - if tkn != reload: + escape = escaping_calls[tkn] + if escape.kills is not None: + if tkn == reload: + self.emit_reload(storage) + self.stackref_kill(escape.kills, storage, True) self.emit_save(storage) - _, reload = escaping_calls[tkn] + elif tkn != reload: + self.emit_save(storage) + reload = escape.end elif tkn == reload: self.emit_reload(storage) if tkn.kind == "LBRACE": @@ -634,7 +626,6 @@ def _emit_block( raise analysis_error(ex.args[0], tkn) from None raise analysis_error("Expecting closing brace. Reached end of file", tkn) - def emit_tokens( self, uop: Uop, diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index dd16a1a7eb28b5..eed896e0956210 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -87,7 +87,11 @@ def error_if( label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - self.emit(") JUMP_TO_ERROR();\n") + self.emit(") {\n") + storage.clear_inputs("at ERROR_IF") + storage.copy().flush(self.out) + self.emit("JUMP_TO_ERROR();\n") + self.out.emit("}\n") return not always_true(first_tkn) From a67c0104df8a68aaad399a5ecbc79d9f426bc84e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 28 Jan 2025 14:49:34 +0000 Subject: [PATCH 2/8] Pacify MyPy --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 14d40c6d99e2e4..7bf3c3ed8bd4ef 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -337,7 +337,7 @@ def convert_stack_item( cond = replace_op_arg_1 return StackItem(item.name, item.type, cond, item.size) -def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]): +def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None: "Unused items cannot be on the stack above used, non-peek items" seen_unused = False for item in reversed(stack): From 88b710c2339a5a62b40194d58a55aba5297f738b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 29 Jan 2025 11:09:51 +0000 Subject: [PATCH 3/8] Remove unused code --- Tools/cases_generator/generators_common.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index ab21d01244b5cd..92b7fdf9cbafbf 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -120,8 +120,6 @@ def __init__(self, out: CWriter): "SYNC_SP": self.sync_sp, "SAVE_STACK": self.save_stack, "RELOAD_STACK": self.reload_stack, - #"PyStackRef_CLOSE": self.stackref_close, - #"PyStackRef_XCLOSE": self.stackref_close, "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close_specialized, "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, @@ -306,26 +304,6 @@ def stackref_kill( live = var.name return True - def stackref_close( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit(tkn) - tkn = next(tkn_iter) - assert tkn.kind == "LPAREN" - self.out.emit(tkn) - name = next(tkn_iter) - self.out.emit(name) - if name.kind == "IDENTIFIER": - return self.stackref_kill(name, storage, True) - rparen = emit_to(self.out, tkn_iter, "RPAREN") - self.emit(rparen) - return True - def stackref_close_specialized( self, tkn: Token, From 7869107233d8785ef71ce9b0b6355787cb1de203 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 29 Jan 2025 11:55:40 +0000 Subject: [PATCH 4/8] Restore more efficient jump-to-error handling for tier 1 --- Python/bytecodes.c | 12 +- Python/executor_cases.c.h | 172 ++++++--------- Python/generated_cases.c.h | 232 ++++++--------------- Tools/cases_generator/generators_common.py | 23 +- Tools/cases_generator/tier2_generator.py | 44 +--- 5 files changed, 149 insertions(+), 334 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e6ae7026ee4dcb..a7c312951f2e33 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5152,18 +5152,18 @@ dummy_func( } label(pop_4_error) { - STACK_SHRINK(1); - goto pop_3_error; + STACK_SHRINK(4); + goto error; } label(pop_3_error) { - STACK_SHRINK(1); - goto pop_2_error; + STACK_SHRINK(3); + goto error; } label(pop_2_error) { - STACK_SHRINK(1); - goto pop_1_error; + STACK_SHRINK(2); + goto error; } label(pop_1_error) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a8533578860934..707ba3558aa9bd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -85,9 +85,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -1591,11 +1589,9 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); @@ -1617,9 +1613,7 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -1785,9 +1779,7 @@ _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -1809,11 +1801,9 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2081,9 +2071,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -2265,9 +2253,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } SETLOCAL(oparg, PyStackRef_NULL); break; @@ -2352,9 +2338,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -2402,11 +2386,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - if (true) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -2483,11 +2465,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -2529,11 +2509,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (true) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -2552,9 +2530,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -2573,11 +2549,9 @@ for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (true) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -2608,9 +2582,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2666,11 +2638,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -2697,11 +2667,9 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3594,11 +3562,9 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (true) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3649,11 +3615,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - if (true) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -4131,9 +4095,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - if (true) { - JUMP_TO_ERROR(); - } + JUMP_TO_ERROR(); } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -4540,11 +4502,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -5063,11 +5023,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -5185,11 +5143,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -5255,11 +5211,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -5545,11 +5499,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -5683,11 +5635,9 @@ for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (true) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -5908,11 +5858,9 @@ PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - if (true) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e84e71ae451777..b676cb52e3c218 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -58,9 +58,7 @@ PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -100,9 +98,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -141,9 +137,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -182,9 +176,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -291,9 +283,7 @@ *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -337,9 +327,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -378,9 +366,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -420,9 +406,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -461,9 +445,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -568,9 +550,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -607,9 +587,7 @@ PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); if (rc <= 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); @@ -1960,9 +1938,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -1987,9 +1963,7 @@ PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -3343,9 +3317,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3356,15 +3328,11 @@ PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } if (!Py_IsNone(match_o)) { stack_pointer += -2; @@ -3399,9 +3367,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3491,9 +3457,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } if (oparg & 16) { stack_pointer += -2; @@ -3676,9 +3640,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -3709,9 +3671,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3742,9 +3702,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3826,9 +3784,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3940,9 +3896,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3970,9 +3924,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -4005,9 +3957,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -4196,9 +4146,7 @@ PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4483,18 +4431,14 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { @@ -4546,9 +4490,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4568,9 +4510,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4684,9 +4624,7 @@ PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -5519,9 +5457,7 @@ int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5556,9 +5492,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -5624,9 +5558,7 @@ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } self_or_null[0] = PyStackRef_NULL; } @@ -5638,9 +5570,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } } attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6504,9 +6434,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -6856,9 +6784,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_3_error; } } // we make no attempt to optimize here; specializations should @@ -6891,9 +6817,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_3_error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -6946,9 +6870,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (attr == NULL) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_3_error; } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; @@ -7076,9 +6998,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7114,9 +7034,7 @@ } else { if (_PyErr_Occurred(tstate)) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_3_error; } // Error! attrs = PyStackRef_None; // Failure! @@ -7687,9 +7605,7 @@ } else { PyStackRef_CLOSE(v); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } } stack_pointer += -1; @@ -7814,9 +7730,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -7861,9 +7775,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -7909,9 +7821,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } } stack_pointer += -2; @@ -8151,9 +8061,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8175,9 +8083,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8191,9 +8097,7 @@ } PyStackRef_CLOSE(v); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8242,9 +8146,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); if (err) { - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_4_error; } } stack_pointer += -4; @@ -8291,9 +8193,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_3_error; } } stack_pointer += -3; @@ -8421,9 +8321,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } res = err ? PyStackRef_True : PyStackRef_False; } @@ -8572,9 +8470,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -8593,9 +8489,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -8630,9 +8524,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -8676,9 +8568,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_1_error; } } stack_pointer += -1 + oparg; @@ -8889,20 +8779,20 @@ pop_4_error: { - STACK_SHRINK(1); - goto pop_3_error; + STACK_SHRINK(4); + goto error; } pop_3_error: { - STACK_SHRINK(1); - goto pop_2_error; + STACK_SHRINK(3); + goto error; } pop_2_error: { - STACK_SHRINK(1); - goto pop_1_error; + STACK_SHRINK(2); + goto error; } pop_1_error: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 92b7fdf9cbafbf..71c521dfa05880 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -164,6 +164,13 @@ def deopt_if( exit_if = deopt_if + def goto_error(self, offset: int, label: str, storage: Storage) -> None: + if offset > 0: + return f"goto pop_{offset}_{label};" + if offset < 0: + storage.copy().flush(self.out) + return f"goto {label};" + def error_if( self, tkn: Token, @@ -186,15 +193,19 @@ def error_if( self.out.emit_at("if ", tkn) self.emit(lparen) emit_to(self.out, tkn_iter, "COMMA") - self.out.emit(") {") + self.out.emit(") {\n") label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon storage.clear_inputs("at ERROR_IF") - storage.copy().flush(self.out) - self.out.emit("goto ") - self.out.emit(label) - self.out.emit(";\n") + + c_offset = storage.stack.peek_offset() + try: + offset = -int(c_offset) + except ValueError: + offset = -1 + self.out.emit(self.goto_error(offset, label, storage)) + self.out.emit("\n") if not unconditional: self.out.emit("}\n") return not unconditional @@ -210,7 +221,7 @@ def error_no_pop( next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - self.out.emit_at("goto error;", tkn) + self.out.emit_at(self.goto_error(0, "error", storage), tkn) return False def decref_inputs( diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index eed896e0956210..ecb3ef274e40d3 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -69,45 +69,11 @@ def __init__(self, out: CWriter): super().__init__(out) self._replacers["oparg"] = self.oparg - def error_if( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit_at("if ", tkn) - lparen = next(tkn_iter) - self.emit(lparen) - assert lparen.kind == "LPAREN" - first_tkn = next(tkn_iter) - self.out.emit(first_tkn) - emit_to(self.out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - self.emit(") {\n") - storage.clear_inputs("at ERROR_IF") - storage.copy().flush(self.out) - self.emit("JUMP_TO_ERROR();\n") - self.out.emit("}\n") - return not always_true(first_tkn) - - - def error_no_pop( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop, - storage: Storage, - inst: Instruction | None, - ) -> bool: - next(tkn_iter) # LPAREN - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - self.out.emit_at("JUMP_TO_ERROR();", tkn) - return False + def goto_error(self, offset: int, label: str, storage: Storage) -> None: + # To do: Add jump targets for popping values. + if offset != 0: + storage.copy().flush(self.out) + return f"JUMP_TO_ERROR();" def deopt_if( self, From 620dde2163af1c95976a81fb382d43694eccb805 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 29 Jan 2025 12:18:10 +0000 Subject: [PATCH 5/8] Make close in LOAD_ATTR_INSTANCE_VALUE explicit --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 4 +++- Python/generated_cases.c.h | 4 +++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bad5e515a99565..56af5688032495 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2118,7 +2118,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 82a4d26da0424e..865a894d94ff01 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -154,7 +154,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CHECK_ATTR_MODULE_PUSH_KEYS] = HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE_FROM_KEYS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_EXIT_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a7c312951f2e33..d6c78c4b26705e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2234,7 +2234,7 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); } macro(LOAD_ATTR_INSTANCE_VALUE) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 707ba3558aa9bd..c1c2c8fda20a7a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2932,8 +2932,10 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b676cb52e3c218..bdc69b69da45a0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5738,14 +5738,16 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); From 4e4946d14d233b1fccff4e875b88bda1f200d853 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 Jan 2025 10:45:50 +0000 Subject: [PATCH 6/8] Fix test cases --- Lib/test/test_generated_cases.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 0932c41a0a3265..35600ce5486642 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -587,9 +587,7 @@ def test_error_if_pop(self): left = stack_pointer[-2]; SPAM(left, right); if (cond) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto label; + goto pop_2_label; } res = 0; stack_pointer[-2] = res; @@ -620,10 +618,7 @@ def test_error_if_pop_with_result(self): left = stack_pointer[-2]; res = SPAM(left, right); if (cond) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - goto label; + goto pop_2_label; } stack_pointer[-2] = res; stack_pointer += -1; @@ -1406,9 +1401,7 @@ def test_pop_on_error_peeks(self): { // Mark j and k as used if (cond) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + goto pop_2_error; } } stack_pointer += -2; From 3369849aa3a3c1e4656550feffbc550c8cb91797 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 Jan 2025 10:53:11 +0000 Subject: [PATCH 7/8] Fix type declaration errors --- Tools/cases_generator/generators_common.py | 2 +- Tools/cases_generator/tier2_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 71c521dfa05880..d3eb948f3563e7 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -164,7 +164,7 @@ def deopt_if( exit_if = deopt_if - def goto_error(self, offset: int, label: str, storage: Storage) -> None: + def goto_error(self, offset: int, label: str, storage: Storage) -> str: if offset > 0: return f"goto pop_{offset}_{label};" if offset < 0: diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index ecb3ef274e40d3..4540eb252634ba 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -69,7 +69,7 @@ def __init__(self, out: CWriter): super().__init__(out) self._replacers["oparg"] = self.oparg - def goto_error(self, offset: int, label: str, storage: Storage) -> None: + def goto_error(self, offset: int, label: str, storage: Storage) -> str: # To do: Add jump targets for popping values. if offset != 0: storage.copy().flush(self.out) From 0e0c0953dcc8248d38d85acba03f791406d79fb6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Jan 2025 11:01:11 +0000 Subject: [PATCH 8/8] Address review comments --- Tools/cases_generator/analyzer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 8340c6169a60f1..12afbc170e2fb7 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -343,11 +343,12 @@ def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> for item in reversed(stack): if item.name == "unused": seen_unused = True - elif seen_unused and not item.peek: + elif item.peek: + break + elif seen_unused: raise analysis_error(f"Cannot have used input '{item.name}' below an unused value on the stack", input_names[item.name]) - def analyze_stack( op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None ) -> StackEffect: @@ -685,7 +686,7 @@ def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: return node.block.tokens[idx+1] def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, EscapingCall]) -> None: - calls = {escapes[t].call for t in escapes} + calls = {e.call for e in escapes.values()} in_if = 0 tkn_iter = iter(instr.block.tokens) for tkn in tkn_iter: @@ -741,7 +742,7 @@ def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, Escaping continue if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): if len(tokens) <= idx+2: - raise analysis_error("'(' at end", next_tkn) + raise analysis_error("Unexpected end of file", next_tkn) kills = tokens[idx+2] if kills.kind != "IDENTIFIER": raise analysis_error(f"Expected identifier, got '{kills.text}'", kills)