aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python/executor_cases.c.h
diff options
context:
space:
mode:
Diffstat (limited to 'Python/executor_cases.c.h')
-rw-r--r--Python/executor_cases.c.h639
1 files changed, 397 insertions, 242 deletions
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 41c9bd5ba70..4f772f916d1 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -319,25 +319,11 @@
break;
}
- /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
-
- case _LOAD_CONST_MORTAL: {
- _PyStackRef value;
- oparg = CURRENT_OPARG();
- PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- value = PyStackRef_FromPyObjectNewMortal(obj);
- stack_pointer[0] = value;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_CONST_IMMORTAL: {
+ case _LOAD_CONST: {
_PyStackRef value;
oparg = CURRENT_OPARG();
PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- assert(_Py_IsImmortal(obj));
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -350,7 +336,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -363,7 +349,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -376,7 +362,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -389,7 +375,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -401,7 +387,7 @@
oparg = CURRENT_OPARG();
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -413,10 +399,6 @@
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -432,10 +414,6 @@
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -451,10 +429,6 @@
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -470,10 +444,6 @@
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -489,10 +459,6 @@
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -508,10 +474,6 @@
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -527,10 +489,6 @@
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -546,10 +504,6 @@
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -564,10 +518,6 @@
_PyStackRef value;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -584,7 +534,25 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(value);
+ PyStackRef_XCLOSE(value);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_TWO: {
+ _PyStackRef tos;
+ _PyStackRef nos;
+ tos = stack_pointer[-1];
+ nos = stack_pointer[-2];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(tos);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(nos);
stack_pointer = _PyFrame_GetStackPointer(frame);
break;
}
@@ -609,6 +577,20 @@
break;
}
+ case _POP_ITER: {
+ _PyStackRef index_or_null;
+ _PyStackRef iter;
+ index_or_null = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ (void)index_or_null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iter);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _END_SEND: {
_PyStackRef value;
_PyStackRef receiver;
@@ -1403,7 +1385,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(str_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
- res = PyStackRef_FromPyObjectImmortal(res_o);
+ res = PyStackRef_FromPyObjectBorrow(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -1569,15 +1551,16 @@
_PyStackRef getitem;
_PyStackRef sub;
_PyStackRef container;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
getitem = stack_pointer[-1];
sub = stack_pointer[-2];
container = stack_pointer[-3];
- new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
- new_frame->localsplus[0] = container;
- new_frame->localsplus[1] = sub;
+ _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
+ pushed_frame->localsplus[0] = container;
+ pushed_frame->localsplus[1] = sub;
frame->return_offset = 6 ;
- stack_pointer[-3].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-3] = new_frame;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -1925,7 +1908,7 @@
case _SEND_GEN_FRAME: {
_PyStackRef v;
_PyStackRef receiver;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
receiver = stack_pointer[-2];
@@ -1939,15 +1922,16 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(SEND, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v));
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert( 2 + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)( 2 + oparg);
- gen_frame->previous = frame;
- stack_pointer[-1].bits = (uintptr_t)gen_frame;
+ pushed_frame->previous = frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = gen_frame;
break;
}
@@ -3489,7 +3473,7 @@
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *fget = (PyObject *)CURRENT_OPERAND0();
@@ -3514,9 +3498,10 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
- new_frame->localsplus[0] = owner;
- stack_pointer[-1].bits = (uintptr_t)new_frame;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
+ pushed_frame->localsplus[0] = owner;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = new_frame;
break;
}
@@ -4204,25 +4189,37 @@
case _GET_ITER: {
_PyStackRef iterable;
_PyStackRef iter;
+ _PyStackRef index_or_null;
iterable = stack_pointer[-1];
#ifdef Py_STATS
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_GatherStats_GetIter(iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
#endif
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(iterable);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (iter_o == NULL) {
- JUMP_TO_ERROR();
+
+ PyTypeObject *tp = PyStackRef_TYPE(iterable);
+ if (tp == &PyTuple_Type || tp == &PyList_Type) {
+ iter = iterable;
+ index_or_null = PyStackRef_TagInt(0);
}
- iter = PyStackRef_FromPyObjectSteal(iter_o);
- stack_pointer[0] = iter;
+ else {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iterable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (iter_o == NULL) {
+ JUMP_TO_ERROR();
+ }
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
+ index_or_null = PyStackRef_NULL;
+ stack_pointer += 1;
+ }
+ stack_pointer[-1] = iter;
+ stack_pointer[0] = index_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4269,32 +4266,25 @@
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_ERROR();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ next = item;
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4304,21 +4294,18 @@
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyListIter_Type) {
+ if (Py_TYPE(iter_o) != &PyList_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ assert(PyStackRef_IsTaggedInt(null_or_index));
#ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
- !_PyObject_GC_IS_SHARED(it->it_seq)) {
+ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4329,24 +4316,17 @@
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
#ifndef Py_GIL_DISABLED
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- if (seq == NULL) {
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(list_o) == &PyList_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
#endif
break;
}
@@ -4354,38 +4334,30 @@
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */
case _ITER_NEXT_LIST_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- assert(seq);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(PyList_CheckExact(list_o));
#ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
- _PyObject_GC_IS_SHARED(seq));
+ assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) ||
+ _PyObject_GC_IS_SHARED(list_o));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
- int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
+ int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (result < 0) {
+ if (result <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (result == 0) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
- it->it_index++;
#else
- assert(it->it_index < PyList_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index)));
#endif
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4393,39 +4365,29 @@
}
case _ITER_CHECK_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- #ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
+ if (Py_TYPE(iter_o) != &PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- #endif
+ assert(PyStackRef_IsTaggedInt(null_or_index));
break;
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- PyTupleObject *seq = it->it_seq;
- if (seq == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (it->it_index >= PyTuple_GET_SIZE(seq)) {
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4433,19 +4395,18 @@
}
case _ITER_NEXT_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- PyTupleObject *seq = it->it_seq;
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- assert(seq);
- assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ uintptr_t i = PyStackRef_UntagInt(null_or_index);
+ assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i));
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4454,7 +4415,7 @@
case _ITER_CHECK_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4473,7 +4434,7 @@
case _GUARD_NOT_EXHAUSTED_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
@@ -4486,7 +4447,7 @@
case _ITER_NEXT_RANGE: {
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
@@ -4509,9 +4470,9 @@
case _FOR_ITER_GEN_FRAME: {
_PyStackRef iter;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4529,14 +4490,15 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(FOR_ITER, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_None);
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
- gen_frame->previous = frame;
+ pushed_frame->previous = frame;
frame->return_offset = (uint16_t)( 2 + oparg);
- stack_pointer[0].bits = (uintptr_t)gen_frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[0] = gen_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4817,7 +4779,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
@@ -4842,8 +4804,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5109,7 +5071,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 0;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5117,13 +5079,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5133,7 +5096,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 1;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5141,13 +5104,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5157,7 +5121,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 2;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5165,13 +5129,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5181,7 +5146,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 3;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5189,13 +5154,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5205,7 +5171,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 4;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5213,13 +5179,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5229,34 +5196,35 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _PUSH_FRAME: {
- _PyInterpreterFrame *new_frame;
- new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits;
+ _PyStackRef new_frame;
+ new_frame = stack_pointer[-1];
assert(tstate->interp->eval_frame == NULL);
- _PyInterpreterFrame *temp = new_frame;
+ _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- assert(new_frame->previous == frame || new_frame->previous->previous == frame);
+ assert(temp->previous == frame || temp->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = temp;
tstate->py_recursion_remaining--;
@@ -5276,6 +5244,17 @@
break;
}
+ case _GUARD_NOS_NOT_NULL: {
+ _PyStackRef nos;
+ nos = stack_pointer[-2];
+ PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
+ if (o == NULL) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
case _GUARD_THIRD_NULL: {
_PyStackRef null;
null = stack_pointer[-3];
@@ -5460,7 +5439,7 @@
_PyStackRef *args;
_PyStackRef self;
_PyStackRef init;
- _PyInterpreterFrame *init_frame;
+ _PyStackRef init_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self = stack_pointer[-1 - oparg];
@@ -5484,10 +5463,10 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_ERROR();
}
- init_frame = temp;
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
tstate->py_recursion_remaining--;
- stack_pointer[0].bits = (uintptr_t)init_frame;
+ init_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = init_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5920,6 +5899,18 @@
break;
}
+ case _GUARD_CALLABLE_LIST_APPEND: {
+ _PyStackRef callable;
+ callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyInterpreterState *interp = tstate->interp;
+ if (callable_o != interp->callable_cache.list_append) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
case _CALL_LIST_APPEND: {
_PyStackRef arg;
_PyStackRef self;
@@ -5929,18 +5920,8 @@
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
- PyInterpreterState *interp = tstate->interp;
- if (callable_o != interp->callable_cache.list_append) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (self_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!PyList_Check(self_o)) {
+ if (!PyList_CheckExact(self_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -6338,7 +6319,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
@@ -6372,8 +6353,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -7042,13 +7023,76 @@
case _LOAD_CONST_INLINE_BORROW: {
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_ONE: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_TWO: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
_PyStackRef pop;
_PyStackRef value;
@@ -7059,7 +7103,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -7083,13 +7127,124 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop1);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE_BORROW: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
assert(PyStackRef_FunctionCheck(frame->f_funcobj));