diff options
Diffstat (limited to 'Python/generated_cases.c.h')
| -rw-r--r-- | Python/generated_cases.c.h | 1960 |
1 files changed, 1209 insertions, 751 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9de7554d4df..a9c127784ed 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -20,15 +20,17 @@ _PyStackRef rhs; _PyStackRef res; // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -42,7 +44,9 @@ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) goto pop_2_error; @@ -63,9 +67,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -99,9 +103,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -134,9 +138,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); @@ -168,9 +172,9 @@ _PyStackRef left; _PyStackRef right; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); @@ -230,9 +234,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -266,9 +270,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -301,9 +305,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -337,9 +341,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -353,7 +357,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -379,12 +383,14 @@ #endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SLICE - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -392,8 +398,14 @@ res_o = NULL; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(container); if (res_o == NULL) goto pop_3_error; @@ -416,15 +428,18 @@ _PyStackRef sub; _PyStackRef res; // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + assert(frame->stackpointer == NULL); #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_SUBSCR); @@ -435,7 +450,9 @@ { PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) goto pop_2_error; @@ -463,9 +480,13 @@ DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); @@ -492,8 +513,8 @@ DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC - container = stack_pointer[-2]; { + container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -508,14 +529,12 @@ STAT_INC(BINARY_SUBSCR, hit); } // _BINARY_SUBSCR_INIT_CALL - sub = stack_pointer[-1]; { + sub = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -525,10 +544,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -671,10 +693,12 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( values_o, 2, values_o+1, 2, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -698,7 +722,9 @@ _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -712,7 +738,9 @@ int err = 0; for (int i = 0; i < oparg; i++) { if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(values[i]); } @@ -832,22 +860,24 @@ PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *maybe_self; _PyStackRef res; // _SPECIALIZE_CALL - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -856,27 +886,26 @@ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD - args = &stack_pointer[-oparg]; { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } // _DO_CALL - self_or_null = maybe_self; - callable = func; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -890,12 +919,15 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + tstate, callable[0], locals, args, total_args, NULL, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -907,7 +939,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -917,30 +949,36 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -954,15 +992,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -976,11 +1016,11 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; + _PyStackRef *callable; + _PyStackRef *null; _PyStackRef *args; - _PyStackRef self; - _PyStackRef init; + _PyStackRef *init; + _PyStackRef *self; _PyInterpreterFrame *init_frame; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -989,13 +1029,15 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_AND_ALLOCATE_OBJECT - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != type_version, CALL); @@ -1005,25 +1047,33 @@ PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); - if (PyStackRef_IsNull(self)) { + PyObject *self_o = _PyType_NewManagedObject(tp); + if (self_o == NULL) { goto error; } - PyStackRef_CLOSE(callable); - init = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-1 - oparg] = init; + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); } // _CREATE_INIT_FRAME { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self); - args[-1] = self; + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); init_frame = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer += -2 - oparg; + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); if (init_frame == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -1036,15 +1086,18 @@ tstate->py_recursion_remaining--; } // _PUSH_FRAME - new_frame = init_frame; { + new_frame = init_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1058,9 +1111,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1071,36 +1124,37 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + func = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); } // flush // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -1108,18 +1162,18 @@ } // _CHECK_STACK_SPACE { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; { + args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -1140,12 +1194,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1159,9 +1214,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; - _PyStackRef method; + _PyStackRef *method; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1172,11 +1227,11 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_METHOD_VERSION - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL); @@ -1185,23 +1240,24 @@ } // _EXPAND_METHOD { + method = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); } // flush // _PY_FRAME_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -1211,17 +1267,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -1237,10 +1295,11 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1254,18 +1313,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_CLASS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1277,7 +1336,7 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1288,13 +1347,15 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1305,15 +1366,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1327,19 +1390,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1352,7 +1415,7 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1363,17 +1426,19 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable_o), args_o, total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1384,15 +1449,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1406,19 +1473,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST_WITH_KEYWORDS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1428,12 +1495,14 @@ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1444,14 +1513,16 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1462,15 +1533,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1484,19 +1557,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_O - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1511,11 +1584,13 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1526,15 +1601,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1550,56 +1627,89 @@ PREDICTED(CALL_FUNCTION_EX); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; _PyStackRef func_st; _PyStackRef callargs_st; _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; - // __DO_CALL_FUNCTION_EX - if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } - callargs_st = stack_pointer[-1 - (oparg & 1)]; - func_st = stack_pointer[-3 - (oparg & 1)]; + // _MAKE_CALLARGS_A_TUPLE { - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { goto error; } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { goto error; } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); } + kwargs_out = kwargs_in; + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) goto error; - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLEAR(result); + Py_CLEAR(result_o); } } } @@ -1612,11 +1722,16 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; } @@ -1624,30 +1739,38 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(func_st); - PyStackRef_CLOSE(callargs_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - if (PyStackRef_IsNull(result)) { + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { stack_pointer += -3 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); goto error; } + result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3 - (oparg & 1)] = result; @@ -1664,7 +1787,9 @@ _PyStackRef res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -1684,7 +1809,9 @@ assert(oparg <= MAX_INTRINSIC_2); PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) goto pop_2_error; @@ -1700,7 +1827,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; @@ -1708,9 +1835,9 @@ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1722,7 +1849,9 @@ STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { goto error; } @@ -1730,7 +1859,7 @@ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(inst_stackref); PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1744,21 +1873,27 @@ PREDICTED(CALL_KW); _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; _PyStackRef res; // _SPECIALIZE_CALL_KW - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -1766,11 +1901,30 @@ #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } // _DO_CALL_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -1778,17 +1932,6 @@ args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -1797,13 +1940,17 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -1816,7 +1963,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1828,23 +1975,30 @@ goto error; } } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } @@ -1852,7 +2006,7 @@ } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -1874,10 +2028,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; _PyStackRef kwnames; - _PyStackRef method; + _PyStackRef *method; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1888,11 +2042,11 @@ DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_METHOD_VERSION_KW - null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL_KW); @@ -1900,26 +2054,26 @@ DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } // _EXPAND_METHOD_KW - kwnames = stack_pointer[-1]; { + method = &stack_pointer[-3 - oparg]; self = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-3 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); } // flush // _PY_FRAME_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -1931,14 +2085,17 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -1958,10 +2115,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1975,7 +2135,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef kwnames; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1983,21 +2143,21 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW - callable = stack_pointer[-3 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } // _CALL_KW_NON_PY - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2006,7 +2166,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2020,17 +2180,19 @@ } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2041,15 +2203,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3 - oparg] = res; @@ -2063,7 +2227,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef kwnames; _PyStackRef *args; @@ -2074,20 +2238,20 @@ DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_FUNCTION_VERSION_KW - callable = stack_pointer[-3 - oparg]; { + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL_KW); } // _PY_FRAME_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -2099,14 +2263,17 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -2126,10 +2293,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2143,7 +2313,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; @@ -2151,9 +2321,9 @@ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2165,7 +2335,9 @@ STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { goto error; } @@ -2174,7 +2346,7 @@ if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -2224,18 +2396,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2249,12 +2421,10 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2265,14 +2435,18 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2283,15 +2457,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2305,18 +2481,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2331,11 +2507,9 @@ DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2346,14 +2520,18 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2364,15 +2542,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2386,19 +2566,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_NOARGS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2417,11 +2597,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2432,15 +2614,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2454,18 +2638,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_O - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2485,14 +2669,16 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyStackRef_AsPyObjectBorrow(self_stackref), PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2503,15 +2689,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2525,27 +2713,27 @@ next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(PyFunction_Check(callable_o), CALL); DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } // _CALL_NON_PY_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2554,7 +2742,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2565,13 +2753,15 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -2585,15 +2775,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2607,7 +2799,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2617,18 +2809,18 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -2636,18 +2828,18 @@ } // _CHECK_STACK_SPACE { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; { + args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -2668,12 +2860,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2687,7 +2880,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2697,19 +2890,19 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _PY_FRAME_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -2719,17 +2912,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2745,10 +2940,11 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2769,27 +2965,36 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_STR_1 - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) goto pop_2_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3] = res; @@ -2810,27 +3015,36 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_TUPLE_1 - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) goto pop_2_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3] = res; @@ -2879,7 +3093,9 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); @@ -2887,15 +3103,23 @@ } PyObject *match_o = NULL; PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) goto pop_2_error; assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) goto pop_2_error; if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -2916,12 +3140,16 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); if (true) goto pop_1_error; } + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(right); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; @@ -2944,21 +3172,25 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - stack_pointer[-2] = value; PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); PyStackRef_CLOSE(exc_value_st); - none = PyStackRef_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer[-3] = none; + stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2975,15 +3207,17 @@ _PyStackRef right; _PyStackRef res; // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -2995,22 +3229,30 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) goto pop_2_error; if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); - if (res_bool < 0) goto pop_2_error; + if (res_bool < 0) goto error; res = res_bool ? PyStackRef_True : PyStackRef_False; } else { res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3024,9 +3266,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); @@ -3062,9 +3304,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); @@ -3104,9 +3346,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); @@ -3145,14 +3387,16 @@ _PyStackRef right; _PyStackRef b; // _SPECIALIZE_CONTAINS_OP - right = stack_pointer[-1]; { + right = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -3160,11 +3404,13 @@ #endif /* ENABLE_SPECIALIZATION */ } // _CONTAINS_OP - left = stack_pointer[-2]; { + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3191,7 +3437,9 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3218,7 +3466,9 @@ DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3239,7 +3489,9 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (result_o == NULL) goto pop_1_error; result = PyStackRef_FromPyObjectSteal(result_o); @@ -3287,7 +3539,9 @@ _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) goto pop_1_error; stack_pointer += -1; @@ -3304,7 +3558,9 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } Py_DECREF(oldobj); @@ -3317,10 +3573,12 @@ INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) goto error; } SETLOCAL(oparg, PyStackRef_NULL); @@ -3332,14 +3590,18 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { goto error; } if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } DISPATCH(); @@ -3353,16 +3615,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } DISPATCH(); @@ -3377,8 +3645,10 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) goto pop_2_error; @@ -3400,9 +3670,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); if (true) goto pop_1_error; } @@ -3422,13 +3696,19 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); if (true) goto pop_1_error; @@ -3450,14 +3730,19 @@ awaitable_st = stack_pointer[-2]; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { PyStackRef_CLOSE(awaitable_st); PyStackRef_CLOSE(exc_st); } else { Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer += -2; @@ -3483,11 +3768,13 @@ INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; (void)receiver; + val = value; PyStackRef_CLOSE(receiver); - stack_pointer[-2] = value; + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3534,9 +3821,11 @@ should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } stack_pointer += -1; @@ -3566,7 +3855,9 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (PyStackRef_IsNull(res)) goto pop_1_error; } @@ -3586,7 +3877,9 @@ _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) goto pop_2_error; @@ -3607,14 +3900,16 @@ _PyStackRef iter; _PyStackRef next; // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3625,16 +3920,21 @@ { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o == NULL) { - next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -3668,8 +3968,8 @@ DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); } // _FOR_ITER_GEN_FRAME - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3684,15 +3984,16 @@ frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } // _PUSH_FRAME - new_frame = gen_frame; { + new_frame = gen_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -3710,8 +4011,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST @@ -3745,8 +4046,8 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - stack_pointer[0] = next; } + stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3761,8 +4062,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } @@ -3806,8 +4107,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE @@ -3838,8 +4139,8 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - stack_pointer[0] = next; } + stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3860,24 +4161,32 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (true) goto pop_1_error; } + _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) goto pop_1_error; if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); - if (true) goto pop_1_error; + if (true) goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -3891,7 +4200,9 @@ _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { goto error; } @@ -3909,7 +4220,9 @@ _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) goto pop_1_error; iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -3925,7 +4238,9 @@ _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ + _PyFrame_SetStackPointer(frame, stack_pointer); iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (PyStackRef_IsNull(iter)) goto pop_1_error; stack_pointer[-1] = iter; @@ -3940,7 +4255,9 @@ _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) goto error; PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; @@ -3965,23 +4282,29 @@ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } iter = iterable; } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } else { - /* `iterable` is not a generator. */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); - if (PyStackRef_IsNull(iter)) { - goto error; + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(iter)) { + goto error; + } + PyStackRef_CLOSE(iterable); } - PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -3995,7 +4318,9 @@ _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -4014,9 +4339,11 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) goto pop_2_error; @@ -4032,57 +4359,62 @@ (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *maybe_self; _PyStackRef res; /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } // _MONITOR_CALL { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } else { - arg0 = &_PyInstrumentation_MISSING; + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } } + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _DO_CALL - self_or_null = maybe_self; - callable = func; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -4096,12 +4428,15 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, + tstate, callable[0], locals, args, total_args, NULL, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -4113,7 +4448,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -4123,30 +4458,36 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -4160,15 +4501,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -4198,9 +4541,11 @@ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); @@ -4218,7 +4563,9 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -4236,17 +4583,21 @@ INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } } + val = value; PyStackRef_CLOSE(receiver); - stack_pointer[-2] = value; + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4261,19 +4612,25 @@ _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); target = next_instr; } else { if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -4292,8 +4649,10 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -4313,9 +4672,11 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4380,6 +4741,8 @@ // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { @@ -4473,7 +4836,9 @@ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -4488,7 +4853,9 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4520,22 +4887,29 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; } // _RETURN_VALUE_EVENT - val = value; { + val = value; + stack_pointer[0] = val; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _RETURN_VALUE - retval = val; { + retval = val; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -4543,9 +4917,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -4563,19 +4937,22 @@ _PyStackRef retval; _PyStackRef res; // _RETURN_VALUE_EVENT - val = stack_pointer[-1]; { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _RETURN_VALUE - retval = val; { + retval = val; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4585,9 +4962,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -4605,22 +4982,24 @@ _PyStackRef retval; _PyStackRef value; // _YIELD_VALUE_EVENT - val = stack_pointer[-1]; { - SAVE_SP(); + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - LOAD_SP(); - if (err) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; DISPATCH(); } } // _YIELD_VALUE - retval = val; { + retval = val; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -4632,6 +5011,7 @@ assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4645,15 +5025,15 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = value; @@ -4675,6 +5055,8 @@ assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } TARGET(IS_OP) { @@ -4710,9 +5092,11 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4733,7 +5117,9 @@ start--; } _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized < 0) goto error; if (optimized) { assert(tstate->previous_executor == NULL); @@ -4782,8 +5168,9 @@ _PyStackRef v; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4799,16 +5186,22 @@ list_st = stack_pointer[-2 - (oparg-1)]; PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); if (true) goto pop_1_error; @@ -4831,15 +5224,17 @@ _PyStackRef attr; _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -4854,7 +5249,9 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. @@ -4877,9 +5274,13 @@ } else { /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -4900,8 +5301,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); @@ -4915,10 +5316,10 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -4935,8 +5336,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); @@ -4956,10 +5357,10 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -5012,8 +5413,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5057,8 +5458,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5081,9 +5482,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5100,8 +5501,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5117,9 +5518,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5136,8 +5537,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5165,9 +5566,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5184,8 +5585,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR); @@ -5226,8 +5627,8 @@ _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5243,8 +5644,8 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; } + stack_pointer[-1] = attr; DISPATCH(); } @@ -5257,8 +5658,8 @@ _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5285,8 +5686,8 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; } + stack_pointer[-1] = attr; DISPATCH(); } @@ -5303,8 +5704,8 @@ DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); } // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5340,12 +5741,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -5364,8 +5766,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5381,10 +5783,10 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - stack_pointer[-1] = attr; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -5401,8 +5803,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5449,10 +5851,15 @@ INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -5468,16 +5875,16 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; // Keep in sync with _common_constants in opcode.py - switch(oparg) { - case CONSTANT_ASSERTIONERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); - break; - case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); - break; - default: - Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; } + value = PyStackRef_FromPyObjectImmortal(val); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5504,7 +5911,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } value = PyStackRef_FromPyObjectSteal(value_o); @@ -5548,10 +5957,12 @@ _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) goto error; } value = PyStackRef_DUP(value_s); @@ -5591,7 +6002,9 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { goto error; } @@ -5599,7 +6012,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } } @@ -5618,23 +6033,30 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - if (err < 0) { - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) goto pop_1_error; if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); } goto error; } @@ -5642,20 +6064,31 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (v_o == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) goto error; } } } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(mod_or_class_dict); v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[-1] = v; DISPATCH(); @@ -5678,7 +6111,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -5692,7 +6127,9 @@ { res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; null = PyStackRef_NULL; } @@ -5788,8 +6225,10 @@ _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } locals = PyStackRef_FromPyObjectNew(l); @@ -5805,7 +6244,9 @@ INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) goto error; v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -5826,19 +6267,35 @@ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } } - if (PyStackRef_IsNull(attr)) goto pop_1_error; + if (PyStackRef_IsNull(attr)) { + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5856,16 +6313,18 @@ _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -5873,33 +6332,46 @@ #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR - self_st = stack_pointer[-1]; { + self_st = stack_pointer[-1]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - if (err) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (true) goto pop_3_error; + } } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(super); } @@ -5910,14 +6382,18 @@ PyStackRef_CLOSE(self_st); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); - if (PyStackRef_IsNull(attr)) goto pop_3_error; + if (PyStackRef_IsNull(attr)) goto error; null = PyStackRef_NULL; } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + (oparg & 1); + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5943,7 +6419,9 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); @@ -5979,8 +6457,10 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); if (attr_o == NULL) { @@ -6024,12 +6504,12 @@ _PyStackRef func; codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) { - goto error; - } + if (func_obj == NULL) goto pop_1_error; _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); @@ -6051,11 +6531,13 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -6076,10 +6558,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(subject); PyStackRef_CLOSE(type); PyStackRef_CLOSE(names); @@ -6108,8 +6592,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) goto error; values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -6162,9 +6648,11 @@ _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XSETREF(exc_info->exc_value, PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6199,8 +6687,8 @@ _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { + value = stack_pointer[-1]; if (PyStackRef_Is(value, PyStackRef_None)) { b = PyStackRef_True; } @@ -6210,8 +6698,8 @@ } } // _POP_JUMP_IF_TRUE - cond = b; { + cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION @@ -6234,8 +6722,8 @@ _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { + value = stack_pointer[-1]; if (PyStackRef_Is(value, PyStackRef_None)) { b = PyStackRef_True; } @@ -6245,8 +6733,8 @@ } } // _POP_JUMP_IF_FALSE - cond = b; { + cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION @@ -6294,9 +6782,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef new_exc; + _PyStackRef exc; _PyStackRef prev_exc; - new_exc = stack_pointer[-1]; + _PyStackRef new_exc; + exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); @@ -6304,8 +6793,9 @@ else { prev_exc = PyStackRef_None; } - assert(PyStackRef_ExceptionInstanceCheck(new_exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -6332,31 +6822,22 @@ INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = PyStackRef_AsPyObjectSteal(args[1]); - _Py_FALLTHROUGH; - case 1: - exc = PyStackRef_AsPyObjectSteal(args[0]); - _Py_FALLTHROUGH; - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; - } - if (true) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; } + if (true) goto error; } TARGET(RERAISE) { @@ -6368,7 +6849,7 @@ _PyStackRef exc_st; exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); @@ -6377,14 +6858,22 @@ assert(!_PyErr_Occurred(tstate)); } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(exc); goto error; } } assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } @@ -6410,7 +6899,9 @@ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -6433,7 +6924,9 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -6467,14 +6960,14 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; } // _RETURN_VALUE - retval = value; { + retval = value; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -6482,9 +6975,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -6500,10 +6993,10 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) goto error; assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -6513,12 +7006,12 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -6536,6 +7029,7 @@ #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6545,9 +7039,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -6566,14 +7060,16 @@ _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; { + receiver = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -6581,8 +7077,8 @@ #endif /* ENABLE_SPECIALIZATION */ } // _SEND - v = stack_pointer[-1]; { + v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; assert(frame != &entry_frame); @@ -6604,25 +7100,36 @@ DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); retval_o = PyObject_CallMethodOneArg(receiver_o, &_Py_ID(send), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); } + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); } else { - goto error; + PyStackRef_CLOSE(v); + if (true) goto pop_1_error; } } PyStackRef_CLOSE(v); @@ -6647,9 +7154,9 @@ DEOPT_IF(tstate->interp->eval_frame, SEND); } // _SEND_GEN_FRAME - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); @@ -6664,17 +7171,18 @@ gen_frame->previous = frame; } // _PUSH_FRAME - new_frame = gen_frame; { + new_frame = gen_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -6687,20 +7195,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; PyObject *ann_dict; if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); if (err) goto error; } @@ -6718,8 +7234,10 @@ _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -6732,41 +7250,20 @@ next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; - _PyStackRef func_st; - func_st = stack_pointer[-1]; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - case MAKE_FUNCTION_ANNOTATE: - assert(PyCallable_Check(attr)); - assert(func_obj->func_annotate == NULL); - func_obj->func_annotate = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func_st; + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6780,8 +7277,10 @@ _PyStackRef iterable; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; @@ -6799,15 +7298,17 @@ _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -6816,11 +7317,13 @@ } /* Skip 3 cache entries */ // _STORE_ATTR - v = stack_pointer[-2]; { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) goto pop_2_error; @@ -6839,8 +7342,8 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -6855,8 +7358,8 @@ DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); @@ -6888,16 +7391,16 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; @@ -6921,16 +7424,16 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t hint = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -6949,7 +7452,9 @@ } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); ep->me_value = PyStackRef_AsPyObjectSteal(value); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. @@ -6969,7 +7474,9 @@ _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7026,7 +7533,9 @@ _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -7044,15 +7553,23 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (true) goto pop_1_error; } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -7076,20 +7593,28 @@ #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SLICE - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); int err; if (slice == NULL) { err = 1; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); @@ -7111,15 +7636,17 @@ _PyStackRef sub; _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -7127,10 +7654,12 @@ #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR - v = stack_pointer[-3]; { + v = stack_pointer[-3]; /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -7156,9 +7685,11 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; @@ -7203,13 +7734,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; DISPATCH(); } @@ -7223,14 +7758,16 @@ _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; { + value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -7240,7 +7777,9 @@ /* Skip 2 cache entries */ // _TO_BOOL { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) goto pop_1_error; res = err ? PyStackRef_True : PyStackRef_False; @@ -7259,16 +7798,16 @@ _PyStackRef res; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE - value = owner; { + value = owner; PyStackRef_CLOSE(value); res = PyStackRef_True; } @@ -7385,7 +7924,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -7400,7 +7941,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -7431,7 +7974,9 @@ seq = stack_pointer[-1]; right = &stack_pointer[(oparg & 0xFF)]; _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; stack_pointer += (oparg & 0xFF) + (oparg >> 8); @@ -7449,14 +7994,16 @@ _PyStackRef seq; _PyStackRef *output; // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; { + seq = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -7469,7 +8016,9 @@ { output = &stack_pointer[-1]; _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; } @@ -7542,10 +8091,10 @@ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - stack_pointer[0] = val0; val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7589,8 +8138,10 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(res)) goto error; stack_pointer[0] = res; stack_pointer += 1; @@ -7616,6 +8167,7 @@ assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7629,113 +8181,19 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - - TARGET(_DO_CALL_FUNCTION_EX) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(_DO_CALL_FUNCTION_EX); - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef result; - if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } - callargs_st = stack_pointer[-1 - (oparg & 1)]; - func_st = stack_pointer[-3 - (oparg & 1)]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); - if (err < 0) { - goto error; - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - goto error; - } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) goto error; - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - PyStackRef_CLEAR(result); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - goto error; - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - } - PyStackRef_CLOSE(func_st); - PyStackRef_CLOSE(callargs_st); - PyStackRef_XCLOSE(kwargs_st); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - if (PyStackRef_IsNull(result)) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } #undef TIER_ONE |