aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python/executor_cases.c.h
diff options
context:
space:
mode:
Diffstat (limited to 'Python/executor_cases.c.h')
-rw-r--r--Python/executor_cases.c.h789
1 files changed, 562 insertions, 227 deletions
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index cc521bd7923..e152865e4ec 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -534,7 +534,65 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(value);
+ PyStackRef_XCLOSE(value);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_TOP_NOP: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) ||
+ _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value))));
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_INT: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_FLOAT: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_UNICODE: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TWO: {
+ _PyStackRef tos;
+ _PyStackRef nos;
+ tos = stack_pointer[-1];
+ nos = stack_pointer[-2];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(tos);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(nos);
stack_pointer = _PyFrame_GetStackPointer(frame);
break;
}
@@ -559,6 +617,20 @@
break;
}
+ case _POP_ITER: {
+ _PyStackRef index_or_null;
+ _PyStackRef iter;
+ index_or_null = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ (void)index_or_null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iter);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _END_SEND: {
_PyStackRef value;
_PyStackRef receiver;
@@ -820,7 +892,7 @@
_PyStackRef left;
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!_PyLong_CheckExactAndCompact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -831,7 +903,31 @@
_PyStackRef value;
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!_PyLong_CheckExactAndCompact(value_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_NOS_OVERFLOWED: {
+ _PyStackRef left;
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ assert(Py_TYPE(left_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_TOS_OVERFLOWED: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ assert(Py_TYPE(value_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -848,18 +944,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -876,18 +969,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -904,18 +994,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -1025,6 +1112,87 @@
break;
}
+ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval *
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval +
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _BINARY_OP_ADD_UNICODE: {
_PyStackRef right;
_PyStackRef left;
@@ -1519,15 +1687,16 @@
_PyStackRef getitem;
_PyStackRef sub;
_PyStackRef container;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
getitem = stack_pointer[-1];
sub = stack_pointer[-2];
container = stack_pointer[-3];
- new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
- new_frame->localsplus[0] = container;
- new_frame->localsplus[1] = sub;
+ _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
+ pushed_frame->localsplus[0] = container;
+ pushed_frame->localsplus[1] = sub;
frame->return_offset = 6 ;
- stack_pointer[-3].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-3] = new_frame;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -1875,7 +2044,7 @@
case _SEND_GEN_FRAME: {
_PyStackRef v;
_PyStackRef receiver;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
receiver = stack_pointer[-2];
@@ -1889,15 +2058,16 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(SEND, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v));
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert( 2 + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)( 2 + oparg);
- gen_frame->previous = frame;
- stack_pointer[-1].bits = (uintptr_t)gen_frame;
+ pushed_frame->previous = frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = gen_frame;
break;
}
@@ -3131,20 +3301,20 @@
case _LOAD_ATTR: {
_PyStackRef owner;
- _PyStackRef attr;
+ _PyStackRef *attr;
_PyStackRef *self_or_null;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
+ attr = &stack_pointer[-1];
self_or_null = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
- PyObject *attr_o;
if (oparg & 1) {
- attr_o = NULL;
+ *attr = PyStackRef_NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
- int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
+ int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
- assert(attr_o != NULL);
+ assert(!PyStackRef_IsNull(*attr));
self_or_null[0] = owner;
}
else {
@@ -3153,7 +3323,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(owner);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (attr_o == NULL) {
+ if (PyStackRef_IsNull(*attr)) {
JUMP_TO_ERROR();
}
self_or_null[0] = PyStackRef_NULL;
@@ -3162,7 +3332,7 @@
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
- attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -3172,10 +3342,9 @@
if (attr_o == NULL) {
JUMP_TO_ERROR();
}
+ *attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer += 1;
}
- attr = PyStackRef_FromPyObjectSteal(attr_o);
- stack_pointer[-1] = attr;
stack_pointer += (oparg&1);
assert(WITHIN_STACK_BOUNDS());
break;
@@ -3439,7 +3608,7 @@
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *fget = (PyObject *)CURRENT_OPERAND0();
@@ -3464,9 +3633,10 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
- new_frame->localsplus[0] = owner;
- stack_pointer[-1].bits = (uintptr_t)new_frame;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
+ pushed_frame->localsplus[0] = owner;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = new_frame;
break;
}
@@ -3534,15 +3704,6 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- #ifdef Py_GIL_DISABLED
- if (dict != _PyObject_GetManagedDict(owner_o)) {
- UNLOCK_OBJECT(dict);
- if (true) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
- #endif
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
@@ -3685,14 +3846,8 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
+ assert(_PyLong_IsCompact((PyLongObject *)left_o));
+ assert(_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
@@ -4154,25 +4309,37 @@
case _GET_ITER: {
_PyStackRef iterable;
_PyStackRef iter;
+ _PyStackRef index_or_null;
iterable = stack_pointer[-1];
#ifdef Py_STATS
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_GatherStats_GetIter(iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
#endif
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(iterable);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (iter_o == NULL) {
- JUMP_TO_ERROR();
+
+ PyTypeObject *tp = PyStackRef_TYPE(iterable);
+ if (tp == &PyTuple_Type || tp == &PyList_Type) {
+ iter = iterable;
+ index_or_null = PyStackRef_TagInt(0);
}
- iter = PyStackRef_FromPyObjectSteal(iter_o);
- stack_pointer[0] = iter;
+ else {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iterable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (iter_o == NULL) {
+ JUMP_TO_ERROR();
+ }
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
+ index_or_null = PyStackRef_NULL;
+ stack_pointer += 1;
+ }
+ stack_pointer[-1] = iter;
+ stack_pointer[0] = index_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4219,32 +4386,25 @@
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_ERROR();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ next = item;
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4254,21 +4414,18 @@
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyListIter_Type) {
+ if (Py_TYPE(iter_o) != &PyList_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ assert(PyStackRef_IsTaggedInt(null_or_index));
#ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
- !_PyObject_GC_IS_SHARED(it->it_seq)) {
+ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4279,24 +4436,17 @@
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
#ifndef Py_GIL_DISABLED
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- if (seq == NULL) {
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(list_o) == &PyList_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
#endif
break;
}
@@ -4304,38 +4454,30 @@
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */
case _ITER_NEXT_LIST_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- assert(seq);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(PyList_CheckExact(list_o));
#ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
- _PyObject_GC_IS_SHARED(seq));
+ assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) ||
+ _PyObject_GC_IS_SHARED(list_o));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
- int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
+ int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (result < 0) {
+ if (result <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (result == 0) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
- it->it_index++;
#else
- assert(it->it_index < PyList_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index)));
#endif
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4343,39 +4485,29 @@
}
case _ITER_CHECK_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
+ if (Py_TYPE(iter_o) != &PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- #ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- #endif
+ assert(PyStackRef_IsTaggedInt(null_or_index));
break;
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- PyTupleObject *seq = it->it_seq;
- if (seq == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (it->it_index >= PyTuple_GET_SIZE(seq)) {
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4383,19 +4515,18 @@
}
case _ITER_NEXT_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- PyTupleObject *seq = it->it_seq;
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- assert(seq);
- assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ uintptr_t i = PyStackRef_UntagInt(null_or_index);
+ assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i));
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4404,7 +4535,7 @@
case _ITER_CHECK_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4423,7 +4554,7 @@
case _GUARD_NOT_EXHAUSTED_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
@@ -4436,7 +4567,7 @@
case _ITER_NEXT_RANGE: {
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
@@ -4459,9 +4590,9 @@
case _FOR_ITER_GEN_FRAME: {
_PyStackRef iter;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4479,14 +4610,15 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(FOR_ITER, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_None);
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
- gen_frame->previous = frame;
+ pushed_frame->previous = frame;
frame->return_offset = (uint16_t)( 2 + oparg);
- stack_pointer[0].bits = (uintptr_t)gen_frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[0] = gen_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4767,7 +4899,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
@@ -4792,8 +4924,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5059,7 +5191,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 0;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5067,13 +5199,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5083,7 +5216,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 1;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5091,13 +5224,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5107,7 +5241,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 2;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5115,13 +5249,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5131,7 +5266,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 3;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5139,13 +5274,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5155,7 +5291,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 4;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5163,13 +5299,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5179,34 +5316,35 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _PUSH_FRAME: {
- _PyInterpreterFrame *new_frame;
- new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits;
+ _PyStackRef new_frame;
+ new_frame = stack_pointer[-1];
assert(tstate->interp->eval_frame == NULL);
- _PyInterpreterFrame *temp = new_frame;
+ _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- assert(new_frame->previous == frame || new_frame->previous->previous == frame);
+ assert(temp->previous == frame || temp->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = temp;
tstate->py_recursion_remaining--;
@@ -5421,7 +5559,7 @@
_PyStackRef *args;
_PyStackRef self;
_PyStackRef init;
- _PyInterpreterFrame *init_frame;
+ _PyStackRef init_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self = stack_pointer[-1 - oparg];
@@ -5445,10 +5583,10 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_ERROR();
}
- init_frame = temp;
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
tstate->py_recursion_remaining--;
- stack_pointer[0].bits = (uintptr_t)init_frame;
+ init_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = init_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -6301,7 +6439,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
@@ -6335,8 +6473,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -6739,12 +6877,44 @@
break;
}
+ case _COPY_1: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-1];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _COPY_2: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-2];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _COPY_3: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-3];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _COPY: {
_PyStackRef bottom;
_PyStackRef top;
oparg = CURRENT_OPARG();
bottom = stack_pointer[-1 - (oparg-1)];
- assert(oparg > 0);
top = PyStackRef_DUP(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
@@ -6784,6 +6954,32 @@
break;
}
+ case _SWAP_2: {
+ _PyStackRef top;
+ _PyStackRef bottom;
+ top = stack_pointer[-1];
+ bottom = stack_pointer[-2];
+ _PyStackRef temp = bottom;
+ bottom = top;
+ top = temp;
+ stack_pointer[-2] = bottom;
+ stack_pointer[-1] = top;
+ break;
+ }
+
+ case _SWAP_3: {
+ _PyStackRef top;
+ _PyStackRef bottom;
+ top = stack_pointer[-1];
+ bottom = stack_pointer[-3];
+ _PyStackRef temp = bottom;
+ bottom = top;
+ top = temp;
+ stack_pointer[-3] = bottom;
+ stack_pointer[-1] = top;
+ break;
+ }
+
case _SWAP: {
_PyStackRef top;
_PyStackRef bottom;
@@ -6793,7 +6989,6 @@
_PyStackRef temp = bottom;
bottom = top;
top = temp;
- assert(oparg >= 2);
stack_pointer[-2 - (oparg-2)] = bottom;
stack_pointer[-1] = top;
break;
@@ -7012,6 +7207,69 @@
break;
}
+ case _POP_CALL: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_ONE: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_TWO: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
_PyStackRef pop;
_PyStackRef value;
@@ -7053,6 +7311,53 @@
break;
}
+ case _POP_CALL_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: {
_PyStackRef pop2;
_PyStackRef pop1;
@@ -7087,6 +7392,36 @@
break;
}
+ case _LOAD_CONST_UNDER_INLINE: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE_BORROW: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
assert(PyStackRef_FunctionCheck(frame->f_funcobj));