diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 111 | ||||
| -rw-r--r-- | Python/ceval.c | 9 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 28 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 1099 | ||||
| -rw-r--r-- | Python/gc.c | 1 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 1570 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 18 |
7 files changed, 2003 insertions, 833 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 703d7ec61eb..b18b0075315 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -361,7 +361,7 @@ dummy_func( } pure inst(POP_TOP, (value --)) { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); } pure inst(PUSH_NULL, (-- res)) { @@ -388,7 +388,7 @@ dummy_func( ERROR_NO_POP(); } } - DECREF_INPUTS(); + PyStackRef_CLOSE(value); } tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { @@ -400,7 +400,7 @@ dummy_func( (void)receiver; val = value; DEAD(value); - DECREF_INPUTS(); + PyStackRef_CLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) { @@ -418,7 +418,7 @@ dummy_func( inst(UNARY_NEGATIVE, (value -- res)) { PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -453,7 +453,7 @@ dummy_func( op(_TO_BOOL, (value -- res)) { int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(err < 0, error); res = err ? PyStackRef_True : PyStackRef_False; } @@ -475,7 +475,7 @@ dummy_func( res = PyStackRef_False; } else { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } } @@ -507,13 +507,13 @@ dummy_func( } else { assert(Py_SIZE(value_o)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } } op(_REPLACE_WITH_TRUE, (value -- res)) { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } @@ -524,7 +524,7 @@ dummy_func( inst(UNARY_INVERT, (value -- res)) { PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -721,7 +721,7 @@ dummy_func( // At the end we just skip over the STORE_FAST. op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); @@ -752,8 +752,7 @@ dummy_func( PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - DEAD(right); + Py_DECREF(right_o); ERROR_IF(PyStackRef_IsNull(*target_local), error); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -967,7 +966,7 @@ dummy_func( inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1048,7 +1047,7 @@ dummy_func( inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1135,12 +1134,12 @@ dummy_func( "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - DECREF_INPUTS(); + PyStackRef_CLOSE(obj); ERROR_IF(true, error); } iter_o = (*getter)(obj_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(obj); ERROR_IF(iter_o == NULL, error); if (Py_TYPE(iter_o)->tp_as_async == NULL || @@ -1166,7 +1165,7 @@ dummy_func( inst(GET_AWAITABLE, (iterable -- iter)) { PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); } @@ -1228,7 +1227,7 @@ dummy_func( JUMPBY(oparg); } else { - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(true, error); } } @@ -1357,7 +1356,7 @@ dummy_func( } } - tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #ifndef Py_TAIL_CALL_INTERP assert(throwflag); @@ -1366,9 +1365,9 @@ dummy_func( int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { - none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -1410,7 +1409,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) { @@ -1419,7 +1418,7 @@ dummy_func( else { err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); } - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1462,10 +1461,10 @@ dummy_func( (void)counter; } - op(_UNPACK_SEQUENCE, (seq -- output[oparg])) { - _PyStackRef *top = output + oparg; - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - DECREF_INPUTS(); + op(_UNPACK_SEQUENCE, (seq -- output[oparg], top[0])) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); ERROR_IF(res == 0, error); } @@ -1479,7 +1478,7 @@ dummy_func( STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - DECREF_INPUTS(); + PyStackRef_CLOSE(seq); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { @@ -1511,10 +1510,10 @@ dummy_func( DECREF_INPUTS(); } - inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8])) { - _PyStackRef *top = right + (oparg >> 8); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - DECREF_INPUTS(); + inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8], top[0])) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); ERROR_IF(res == 0, error); } @@ -1550,14 +1549,14 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1589,7 +1588,7 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0, error); if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -1885,17 +1884,17 @@ dummy_func( "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable_st); ERROR_IF(true, error); } assert(Py_IsNone(none_val)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable_st); } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(err < 0, error); } @@ -1970,10 +1969,10 @@ dummy_func( "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); ERROR_IF(true, error); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { @@ -1984,10 +1983,10 @@ dummy_func( int err = _PyDict_MergeEx(dict_o, update_o, 2); if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(update); ERROR_IF(true, error); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); } inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { @@ -2172,7 +2171,7 @@ dummy_func( CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL, error); self_or_null[0] = PyStackRef_NULL; } @@ -2180,13 +2179,12 @@ dummy_func( else { /* Classic, pushes one value. */ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL, error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } - macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + unused/8 + @@ -2736,12 +2734,11 @@ dummy_func( assert(PyExceptionInstance_Check(left_o)); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); if (err < 0) { - DECREF_INPUTS(); - ERROR_IF(true, error); + ERROR_NO_POP(); } int res = PyErr_GivenExceptionMatches(left_o, right_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(right); b = res ? PyStackRef_True : PyStackRef_False; } @@ -2968,7 +2965,7 @@ dummy_func( inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); } @@ -3410,7 +3407,7 @@ dummy_func( assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); } @@ -3426,7 +3423,7 @@ dummy_func( assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); } @@ -4797,9 +4794,11 @@ dummy_func( assert(_PyEval_BinaryOps[oparg]); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - DECREF_INPUTS(); - ERROR_IF(res_o == NULL, error); + if (res_o == NULL) { + ERROR_NO_POP(); + } res = PyStackRef_FromPyObjectSteal(res_o); + DECREF_INPUTS(); } macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; @@ -5267,10 +5266,12 @@ dummy_func( goto exit_unwind; } next_instr = frame->instr_ptr; - - LLTRACE_RESUME_FRAME(); - #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ diff --git a/Python/ceval.c b/Python/ceval.c index 1684fd7ecbd..931915c8cf7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -135,6 +135,7 @@ static void dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); @@ -165,6 +166,7 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) printf("]\n"); fflush(stdout); PyErr_SetRaisedException(exc); + _PyFrame_GetStackPointer(frame); } static void @@ -799,6 +801,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); #endif _PyInterpreterFrame entry_frame; @@ -858,8 +861,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, we need to update instrumentation */ _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); monitor_throw(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef Py_TAIL_CALL_INTERP return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0); #else @@ -2012,7 +2015,7 @@ _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *frame, PyObject* exc_value, */ int -_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, +_PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp) { int i = 0, j = 0; @@ -2020,8 +2023,6 @@ _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, PyObject *it; /* iter(v) */ PyObject *w; PyObject *l = NULL; /* variable list */ - - PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref); assert(v != NULL); it = PyObject_GetIter(v); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0a4f65feb3b..50e940c1973 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -118,7 +118,9 @@ #ifdef Py_DEBUG #define LLTRACE_RESUME_FRAME() \ do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (lltrace < 0) { \ JUMP_TO_LABEL(exit_unwind); \ } \ @@ -409,17 +411,21 @@ do { \ } while (0) #endif -#define GOTO_TIER_ONE(TARGET) \ -do { \ - next_instr = (TARGET); \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - Py_CLEAR(tstate->previous_executor); \ - if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ - goto error; \ - } \ - DISPATCH(); \ -} while (0) +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + next_instr = (TARGET); \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + Py_CLEAR(tstate->previous_executor); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) \ + { \ + next_instr = frame->instr_ptr; \ + goto error; \ + } \ + DISPATCH(); \ + } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND0() (next_uop[-1].operand0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 96b7386bd24..5962e0ebd4f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -438,9 +438,11 @@ case _POP_TOP: { _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -477,10 +479,12 @@ receiver = stack_pointer[-2]; (void)receiver; val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -491,14 +495,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -520,14 +528,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = err ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -557,8 +569,14 @@ res = PyStackRef_False; } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; break; @@ -575,7 +593,12 @@ } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = res; break; } @@ -611,8 +634,14 @@ } else { assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; break; @@ -622,9 +651,15 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -635,14 +670,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -697,7 +736,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -723,7 +764,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -749,7 +792,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -935,7 +980,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); int next_oparg; @@ -965,12 +1010,16 @@ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } #if TIER_ONE @@ -979,8 +1028,6 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); #endif - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1019,12 +1066,20 @@ STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1099,15 +1154,23 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-1] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1276,17 +1339,25 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1372,14 +1443,14 @@ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1393,17 +1464,24 @@ /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (err) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1495,16 +1573,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1517,14 +1599,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1540,17 +1626,23 @@ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1598,24 +1690,26 @@ "__aiter__ method, got %.100s", type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " @@ -1626,7 +1720,9 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1655,14 +1751,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1810,9 +1910,11 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) { @@ -1825,14 +1927,14 @@ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1865,21 +1967,21 @@ case _UNPACK_SEQUENCE: { _PyStackRef seq; - _PyStackRef *output; + _PyStackRef *top; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1903,11 +2005,13 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -1931,8 +2035,12 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1965,29 +2073,33 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_EX: { _PyStackRef seq; - _PyStackRef *right; + _PyStackRef *top; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2002,16 +2114,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2023,14 +2139,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2042,14 +2158,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2401,26 +2517,36 @@ pieces = &stack_pointer[-oparg]; STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + stack_pointer[0] = str; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2482,15 +2608,19 @@ Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2504,14 +2634,14 @@ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2524,9 +2654,14 @@ PyObject *set_o = PySet_New(NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -2539,20 +2674,25 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (err != 0) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[0] = set; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2564,9 +2704,14 @@ values = &stack_pointer[-oparg*2]; STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -2578,17 +2723,22 @@ oparg); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + stack_pointer[0] = map; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2655,14 +2805,18 @@ Py_TYPE(update_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2684,14 +2838,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2749,18 +2907,27 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + stack_pointer[0] = attr_st; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2810,13 +2977,26 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(global_super_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2850,13 +3030,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } } else { @@ -2864,12 +3048,16 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; @@ -3095,10 +3283,15 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); - stack_pointer[-2] = attr; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; break; } @@ -3124,7 +3317,12 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } @@ -3154,7 +3352,12 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } @@ -3345,17 +3548,21 @@ assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); @@ -3367,8 +3574,6 @@ } else { res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } stack_pointer[0] = res; stack_pointer += 1; @@ -3467,11 +3672,21 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3487,17 +3702,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3519,17 +3740,23 @@ // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3550,17 +3777,23 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3578,8 +3811,16 @@ int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -3589,33 +3830,35 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3632,17 +3875,20 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(right); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3658,17 +3904,23 @@ PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3705,7 +3957,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } stack_pointer[-1] = b; break; @@ -3750,25 +4008,34 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); // Success! attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } // Error! attrs = PyStackRef_None; // Failure! } - stack_pointer[-3] = attrs; - stack_pointer += -2; + stack_pointer[0] = attrs; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3826,14 +4093,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3870,7 +4141,13 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = iter; } } stack_pointer[-1] = iter; @@ -4276,9 +4553,15 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4292,9 +4575,15 @@ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4517,11 +4806,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -4534,19 +4832,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4980,7 +5287,9 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } init_frame = temp; @@ -5040,11 +5349,20 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5053,19 +5371,28 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5160,11 +5487,20 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5177,19 +5513,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5228,11 +5573,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5242,19 +5596,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5350,11 +5713,19 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = callable[0]; + callable[0] = res; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -5463,19 +5834,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5516,11 +5896,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5532,19 +5921,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5652,11 +6050,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5668,19 +6075,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5876,12 +6292,23 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5901,19 +6328,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6051,17 +6487,22 @@ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[0] = slice; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6132,17 +6573,23 @@ value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6173,17 +6620,22 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(rhs); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; break; } @@ -6480,7 +6932,12 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } diff --git a/Python/gc.c b/Python/gc.c index 0fb2f03b040..68879b9efa1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1996,6 +1996,7 @@ Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { GCState *gcstate = &tstate->interp->gc; + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); int expected = 0; if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f02e13f5e3f..094ca1f3c52 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -61,16 +61,23 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(rhs); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -167,7 +174,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -279,13 +288,21 @@ STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -324,7 +341,7 @@ // _BINARY_OP_INPLACE_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); int next_oparg; @@ -355,11 +372,17 @@ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -368,8 +391,6 @@ SKIP_OVER(1); #endif } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -466,7 +487,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -514,15 +537,25 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -895,7 +928,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -1004,9 +1039,14 @@ values = &stack_pointer[-oparg*2]; STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1018,17 +1058,22 @@ oparg); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + stack_pointer[0] = map; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1048,9 +1093,14 @@ PyObject *set_o = PySet_New(NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1063,20 +1113,25 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (err != 0) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[0] = set; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1096,17 +1151,22 @@ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[0] = slice; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1124,26 +1184,36 @@ pieces = &stack_pointer[-oparg]; STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + stack_pointer[0] = str; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1279,11 +1349,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1319,14 +1398,23 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1336,8 +1424,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1345,12 +1433,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1454,7 +1542,9 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } init_frame = temp; @@ -1787,11 +1877,20 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1800,14 +1899,23 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1817,8 +1925,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1826,12 +1934,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1881,11 +1989,20 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1898,14 +2015,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1915,8 +2041,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1924,12 +2050,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1983,11 +2109,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1997,14 +2132,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2014,8 +2158,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -2023,12 +2167,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2336,12 +2480,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2363,15 +2513,23 @@ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2426,11 +2584,19 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = callable[0]; + callable[0] = res; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2548,12 +2714,24 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -2589,21 +2767,32 @@ } } } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2793,12 +2982,23 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -2818,14 +3018,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2835,8 +3044,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -2844,12 +3053,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3144,11 +3353,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3160,14 +3378,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3177,8 +3404,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3186,12 +3413,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3248,11 +3475,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3264,14 +3500,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3281,8 +3526,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3290,12 +3535,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3474,14 +3719,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3491,8 +3745,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3500,12 +3754,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3560,11 +3814,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3577,14 +3840,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3594,8 +3866,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3603,12 +3875,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4030,38 +4302,54 @@ int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - JUMP_TO_LABEL(pop_2_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4085,15 +4373,20 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(right); - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4107,14 +4400,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; + _PyStackRef sub_iter; + _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; _PyStackRef value; exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #ifndef Py_TAIL_CALL_INTERP assert(throwflag); @@ -4124,23 +4417,34 @@ int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4184,15 +4488,21 @@ assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); @@ -4204,8 +4514,6 @@ } else { res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[0] = res; @@ -4421,16 +4729,24 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4462,15 +4778,23 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4503,15 +4827,23 @@ // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4603,12 +4935,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4737,14 +5071,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - JUMP_TO_LABEL(pop_2_error); - } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -4772,12 +5112,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4809,12 +5155,18 @@ Py_TYPE(update_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4838,20 +5190,26 @@ int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } else { Py_INCREF(exc); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4892,10 +5250,12 @@ receiver = stack_pointer[-2]; (void)receiver; val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5032,15 +5392,23 @@ value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5376,20 +5744,26 @@ "__aiter__ method, got %.100s", type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " @@ -5400,7 +5774,9 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5442,12 +5818,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5466,12 +5848,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5544,7 +5932,13 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = iter; } } stack_pointer[-1] = iter; @@ -5594,15 +5988,23 @@ PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5714,11 +6116,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -5754,14 +6165,23 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -5771,8 +6191,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -5780,12 +6200,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6102,12 +6522,24 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -6143,21 +6575,32 @@ } } } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6185,9 +6628,11 @@ JUMP_TO_LABEL(error); } } - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -6414,10 +6859,23 @@ frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } } // we make no attempt to optimize here; specializations should @@ -6448,15 +6906,26 @@ } } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (super == NULL) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); @@ -6845,11 +7314,21 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7095,13 +7574,19 @@ Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -7164,11 +7649,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } } else { @@ -7176,10 +7667,16 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -7228,13 +7725,18 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -7290,13 +7792,18 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -7722,10 +8229,16 @@ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); } - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7784,10 +8297,16 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); } - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7935,14 +8454,19 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -8039,14 +8563,19 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -8349,16 +8878,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), @@ -8379,8 +8910,6 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8405,11 +8934,11 @@ } } } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8775,10 +9304,23 @@ frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } } // we make no attempt to optimize here; specializations should @@ -8809,15 +9351,26 @@ } } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (super == NULL) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); @@ -8875,16 +9428,27 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + stack_pointer[0] = attr_st; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8946,13 +9510,26 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(global_super_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9068,23 +9645,34 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); // Success! attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } // Error! attrs = PyStackRef_None; // Failure! } - stack_pointer[-3] = attrs; - stack_pointer += -2; + stack_pointer[0] = attrs; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9208,9 +9796,11 @@ INSTRUCTION_STATS(POP_ITER); _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -9258,7 +9848,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } } // _POP_JUMP_IF_TRUE @@ -9296,7 +9892,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } } // _POP_JUMP_IF_FALSE @@ -9344,9 +9946,11 @@ INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -9419,8 +10023,6 @@ assert(oparg == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -9464,8 +10066,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } @@ -9750,8 +10350,12 @@ JUMPBY(oparg); } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -9900,12 +10504,14 @@ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9953,12 +10559,14 @@ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10001,15 +10609,21 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10347,12 +10961,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10374,8 +10990,12 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10387,12 +11007,14 @@ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10439,14 +11061,24 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-1] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_4_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10488,16 +11120,25 @@ /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10663,13 +11304,19 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = err ? PyStackRef_True : PyStackRef_False; } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10703,10 +11350,16 @@ // _REPLACE_WITH_TRUE { value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10762,8 +11415,14 @@ res = PyStackRef_False; } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; DISPATCH(); @@ -10793,7 +11452,12 @@ } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = res; DISPATCH(); } @@ -10855,8 +11519,14 @@ } else { assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; DISPATCH(); @@ -10876,12 +11546,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10899,12 +11575,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10935,18 +11617,20 @@ next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; - _PyStackRef *right; + _PyStackRef *top; seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10963,7 +11647,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; - _PyStackRef *output; + _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE { seq = stack_pointer[-1]; @@ -10985,17 +11669,19 @@ } // _UNPACK_SEQUENCE { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11041,8 +11727,12 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11079,8 +11769,12 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11116,11 +11810,13 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -11310,8 +12006,6 @@ JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } @@ -11403,8 +12097,12 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; - LLTRACE_RESUME_FRAME(); #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fd8486785ed..51d0fa63e64 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -273,14 +273,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -306,14 +308,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -339,14 +343,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: |