aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python/executor_cases.c.h
diff options
context:
space:
mode:
Diffstat (limited to 'Python/executor_cases.c.h')
-rw-r--r--Python/executor_cases.c.h1046
1 files changed, 697 insertions, 349 deletions
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 7f3c3141ad0..276c320c5f4 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -319,25 +319,11 @@
break;
}
- /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
-
- case _LOAD_CONST_MORTAL: {
- _PyStackRef value;
- oparg = CURRENT_OPARG();
- PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- value = PyStackRef_FromPyObjectNewMortal(obj);
- stack_pointer[0] = value;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_CONST_IMMORTAL: {
+ case _LOAD_CONST: {
_PyStackRef value;
oparg = CURRENT_OPARG();
PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- assert(_Py_IsImmortal(obj));
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -350,7 +336,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -363,7 +349,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -376,7 +362,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -389,7 +375,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -401,7 +387,7 @@
oparg = CURRENT_OPARG();
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -413,10 +399,6 @@
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -432,10 +414,6 @@
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -451,10 +429,6 @@
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -470,10 +444,6 @@
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -489,10 +459,6 @@
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -508,10 +474,6 @@
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -527,10 +489,6 @@
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -546,10 +504,6 @@
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -564,10 +518,6 @@
_PyStackRef value;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -584,7 +534,65 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(value);
+ PyStackRef_XCLOSE(value);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_TOP_NOP: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) ||
+ _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value))));
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_INT: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_FLOAT: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TOP_UNICODE: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value)));
+ PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_TWO: {
+ _PyStackRef tos;
+ _PyStackRef nos;
+ tos = stack_pointer[-1];
+ nos = stack_pointer[-2];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(tos);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(nos);
stack_pointer = _PyFrame_GetStackPointer(frame);
break;
}
@@ -609,6 +617,20 @@
break;
}
+ case _POP_ITER: {
+ _PyStackRef index_or_null;
+ _PyStackRef iter;
+ index_or_null = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ (void)index_or_null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iter);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _END_SEND: {
_PyStackRef value;
_PyStackRef receiver;
@@ -870,7 +892,7 @@
_PyStackRef left;
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!_PyLong_CheckExactAndCompact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -881,7 +903,31 @@
_PyStackRef value;
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!_PyLong_CheckExactAndCompact(value_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_NOS_OVERFLOWED: {
+ _PyStackRef left;
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ assert(Py_TYPE(left_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_TOS_OVERFLOWED: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ assert(Py_TYPE(value_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -898,18 +944,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -926,18 +969,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -954,18 +994,15 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -1075,6 +1112,87 @@
break;
}
+ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval *
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval +
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: {
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres));
+ if (PyStackRef_IsNull(res)) {
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ JUMP_TO_ERROR();
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _BINARY_OP_ADD_UNICODE: {
_PyStackRef right;
_PyStackRef left;
@@ -1403,7 +1521,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(str_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
- res = PyStackRef_FromPyObjectImmortal(res_o);
+ res = PyStackRef_FromPyObjectBorrow(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -1569,15 +1687,16 @@
_PyStackRef getitem;
_PyStackRef sub;
_PyStackRef container;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
getitem = stack_pointer[-1];
sub = stack_pointer[-2];
container = stack_pointer[-3];
- new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
- new_frame->localsplus[0] = container;
- new_frame->localsplus[1] = sub;
+ _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
+ pushed_frame->localsplus[0] = container;
+ pushed_frame->localsplus[1] = sub;
frame->return_offset = 6 ;
- stack_pointer[-3].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-3] = new_frame;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -1925,7 +2044,7 @@
case _SEND_GEN_FRAME: {
_PyStackRef v;
_PyStackRef receiver;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
receiver = stack_pointer[-2];
@@ -1939,15 +2058,16 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(SEND, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v));
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert( 2 + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)( 2 + oparg);
- gen_frame->previous = frame;
- stack_pointer[-1].bits = (uintptr_t)gen_frame;
+ pushed_frame->previous = frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = gen_frame;
break;
}
@@ -3181,20 +3301,20 @@
case _LOAD_ATTR: {
_PyStackRef owner;
- _PyStackRef attr;
+ _PyStackRef *attr;
_PyStackRef *self_or_null;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
+ attr = &stack_pointer[-1];
self_or_null = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
- PyObject *attr_o;
if (oparg & 1) {
- attr_o = NULL;
+ *attr = PyStackRef_NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
- int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
+ int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
- assert(attr_o != NULL);
+ assert(!PyStackRef_IsNull(*attr));
self_or_null[0] = owner;
}
else {
@@ -3203,7 +3323,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(owner);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (attr_o == NULL) {
+ if (PyStackRef_IsNull(*attr)) {
JUMP_TO_ERROR();
}
self_or_null[0] = PyStackRef_NULL;
@@ -3212,7 +3332,7 @@
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
- attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -3222,10 +3342,9 @@
if (attr_o == NULL) {
JUMP_TO_ERROR();
}
+ *attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer += 1;
}
- attr = PyStackRef_FromPyObjectSteal(attr_o);
- stack_pointer[-1] = attr;
stack_pointer += (oparg&1);
assert(WITHIN_STACK_BOUNDS());
break;
@@ -3489,7 +3608,7 @@
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *fget = (PyObject *)CURRENT_OPERAND0();
@@ -3514,9 +3633,10 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
- new_frame->localsplus[0] = owner;
- stack_pointer[-1].bits = (uintptr_t)new_frame;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
+ pushed_frame->localsplus[0] = owner;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-1] = new_frame;
break;
}
@@ -3735,14 +3855,8 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
+ assert(_PyLong_IsCompact((PyLongObject *)left_o));
+ assert(_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
@@ -4204,25 +4318,37 @@
case _GET_ITER: {
_PyStackRef iterable;
_PyStackRef iter;
+ _PyStackRef index_or_null;
iterable = stack_pointer[-1];
#ifdef Py_STATS
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_GatherStats_GetIter(iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
#endif
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(iterable);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (iter_o == NULL) {
- JUMP_TO_ERROR();
+
+ PyTypeObject *tp = PyStackRef_TYPE(iterable);
+ if (tp == &PyTuple_Type || tp == &PyList_Type) {
+ iter = iterable;
+ index_or_null = PyStackRef_TagInt(0);
}
- iter = PyStackRef_FromPyObjectSteal(iter_o);
- stack_pointer[0] = iter;
+ else {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iterable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (iter_o == NULL) {
+ JUMP_TO_ERROR();
+ }
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
+ index_or_null = PyStackRef_NULL;
+ stack_pointer += 1;
+ }
+ stack_pointer[-1] = iter;
+ stack_pointer[0] = index_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4269,32 +4395,25 @@
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_ERROR();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ next = item;
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4304,21 +4423,18 @@
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyListIter_Type) {
+ if (Py_TYPE(iter_o) != &PyList_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ assert(PyStackRef_IsTaggedInt(null_or_index));
#ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
- !_PyObject_GC_IS_SHARED(it->it_seq)) {
+ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4329,24 +4445,17 @@
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
#ifndef Py_GIL_DISABLED
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- if (seq == NULL) {
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(list_o) == &PyList_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
#endif
break;
}
@@ -4354,38 +4463,30 @@
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */
case _ITER_NEXT_LIST_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- assert(seq);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(PyList_CheckExact(list_o));
#ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
- _PyObject_GC_IS_SHARED(seq));
+ assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) ||
+ _PyObject_GC_IS_SHARED(list_o));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
- int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
+ int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (result < 0) {
+ if (result <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (result == 0) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
- it->it_index++;
#else
- assert(it->it_index < PyList_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index)));
#endif
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4393,39 +4494,29 @@
}
case _ITER_CHECK_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- #ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
+ if (Py_TYPE(iter_o) != &PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- #endif
+ assert(PyStackRef_IsTaggedInt(null_or_index));
break;
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- PyTupleObject *seq = it->it_seq;
- if (seq == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (it->it_index >= PyTuple_GET_SIZE(seq)) {
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4433,19 +4524,18 @@
}
case _ITER_NEXT_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- PyTupleObject *seq = it->it_seq;
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- assert(seq);
- assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ uintptr_t i = PyStackRef_UntagInt(null_or_index);
+ assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i));
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4454,7 +4544,7 @@
case _ITER_CHECK_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4473,7 +4563,7 @@
case _GUARD_NOT_EXHAUSTED_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
@@ -4486,7 +4576,7 @@
case _ITER_NEXT_RANGE: {
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
@@ -4509,9 +4599,9 @@
case _FOR_ITER_GEN_FRAME: {
_PyStackRef iter;
- _PyInterpreterFrame *gen_frame;
+ _PyStackRef gen_frame;
oparg = CURRENT_OPARG();
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4529,14 +4619,15 @@
JUMP_TO_JUMP_TARGET();
}
STAT_INC(FOR_ITER, hit);
- gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, PyStackRef_None);
+ _PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
+ _PyFrame_StackPush(pushed_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
- gen_frame->previous = frame;
+ pushed_frame->previous = frame;
frame->return_offset = (uint16_t)( 2 + oparg);
- stack_pointer[0].bits = (uintptr_t)gen_frame;
+ gen_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[0] = gen_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4817,7 +4908,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
@@ -4842,8 +4933,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5109,7 +5200,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 0;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5117,13 +5208,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5133,7 +5225,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 1;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5141,13 +5233,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5157,7 +5250,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 2;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5165,13 +5258,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5181,7 +5275,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 3;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5189,13 +5283,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5205,7 +5300,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = 4;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
@@ -5213,13 +5308,14 @@
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5229,34 +5325,35 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
- _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
- new_frame->localsplus[0] = self_or_null;
+ _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame);
+ _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self;
+ pushed_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(pushed_frame);
+ stack_pointer[-2 - oparg] = new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _PUSH_FRAME: {
- _PyInterpreterFrame *new_frame;
- new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits;
+ _PyStackRef new_frame;
+ new_frame = stack_pointer[-1];
assert(tstate->interp->eval_frame == NULL);
- _PyInterpreterFrame *temp = new_frame;
+ _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- assert(new_frame->previous == frame || new_frame->previous->previous == frame);
+ assert(temp->previous == frame || temp->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = temp;
tstate->py_recursion_remaining--;
@@ -5276,6 +5373,27 @@
break;
}
+ case _GUARD_NOS_NOT_NULL: {
+ _PyStackRef nos;
+ nos = stack_pointer[-2];
+ PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
+ if (o == NULL) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_THIRD_NULL: {
+ _PyStackRef null;
+ null = stack_pointer[-3];
+ if (!PyStackRef_IsNull(null)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
case _GUARD_CALLABLE_TYPE_1: {
_PyStackRef callable;
callable = stack_pointer[-3];
@@ -5450,7 +5568,7 @@
_PyStackRef *args;
_PyStackRef self;
_PyStackRef init;
- _PyInterpreterFrame *init_frame;
+ _PyStackRef init_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self = stack_pointer[-1 - oparg];
@@ -5474,10 +5592,10 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_ERROR();
}
- init_frame = temp;
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
tstate->py_recursion_remaining--;
- stack_pointer[0].bits = (uintptr_t)init_frame;
+ init_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = init_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -5804,35 +5922,31 @@
break;
}
- case _CALL_LEN: {
- _PyStackRef *args;
- _PyStackRef self_or_null;
+ case _GUARD_CALLABLE_LEN: {
_PyStackRef callable;
- _PyStackRef res;
- oparg = CURRENT_OPARG();
- args = &stack_pointer[-oparg];
- self_or_null = stack_pointer[-1 - oparg];
- callable = stack_pointer[-2 - oparg];
+ callable = stack_pointer[-3];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
- int total_args = oparg;
- if (!PyStackRef_IsNull(self_or_null)) {
- args--;
- total_args++;
- }
- if (total_args != 1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
PyInterpreterState *interp = tstate->interp;
if (callable_o != interp->callable_cache.len) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ break;
+ }
+
+ case _CALL_LEN: {
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
+ arg = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ (void)null;
STAT_INC(CALL, hit);
- _PyStackRef arg_stackref = args[0];
- PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
_PyFrame_SetStackPointer(frame, stack_pointer);
- Py_ssize_t len_i = PyObject_Length(arg);
+ Py_ssize_t len_i = PyObject_Length(arg_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (len_i < 0) {
JUMP_TO_ERROR();
@@ -5842,10 +5956,12 @@
if (res_o == NULL) {
JUMP_TO_ERROR();
}
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(arg_stackref);
+ PyStackRef_CLOSE(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -2 - oparg;
+ stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callable);
@@ -5857,59 +5973,70 @@
break;
}
- case _CALL_ISINSTANCE: {
- _PyStackRef *args;
- _PyStackRef self_or_null;
+ case _GUARD_CALLABLE_ISINSTANCE: {
_PyStackRef callable;
- _PyStackRef res;
- oparg = CURRENT_OPARG();
- args = &stack_pointer[-oparg];
- self_or_null = stack_pointer[-1 - oparg];
- callable = stack_pointer[-2 - oparg];
+ callable = stack_pointer[-4];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
- int total_args = oparg;
- _PyStackRef *arguments = args;
- if (!PyStackRef_IsNull(self_or_null)) {
- arguments--;
- total_args++;
- }
- if (total_args != 2) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
PyInterpreterState *interp = tstate->interp;
if (callable_o != interp->callable_cache.isinstance) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ break;
+ }
+
+ case _CALL_ISINSTANCE: {
+ _PyStackRef cls;
+ _PyStackRef instance;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
+ cls = stack_pointer[-1];
+ instance = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
STAT_INC(CALL, hit);
- _PyStackRef cls_stackref = arguments[1];
- _PyStackRef inst_stackref = arguments[0];
+ PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance);
+ PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls);
_PyFrame_SetStackPointer(frame, stack_pointer);
- int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
+ int retval = PyObject_IsInstance(inst_o, cls_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (retval < 0) {
JUMP_TO_ERROR();
}
- res = retval ? PyStackRef_True : PyStackRef_False;
- assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
+ (void)null;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- _PyStackRef tmp = callable;
- callable = res;
- stack_pointer[-2 - oparg] = callable;
- PyStackRef_CLOSE(tmp);
- for (int _i = oparg; --_i >= 0;) {
- tmp = args[_i];
- args[_i] = PyStackRef_NULL;
- PyStackRef_CLOSE(tmp);
- }
- tmp = self_or_null;
- self_or_null = PyStackRef_NULL;
- stack_pointer[-1 - oparg] = self_or_null;
- PyStackRef_XCLOSE(tmp);
+ PyStackRef_CLOSE(cls);
stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -1 - oparg;
+ stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(instance);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ res = retval ? PyStackRef_True : PyStackRef_False;
+ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
+ stack_pointer[0] = res;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _GUARD_CALLABLE_LIST_APPEND: {
+ _PyStackRef callable;
+ callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyInterpreterState *interp = tstate->interp;
+ if (callable_o != interp->callable_cache.list_append) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
break;
}
@@ -5922,18 +6049,8 @@
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
- PyInterpreterState *interp = tstate->interp;
- if (callable_o != interp->callable_cache.list_append) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (self_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!PyList_Check(self_o)) {
+ if (!PyList_CheckExact(self_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -6331,7 +6448,7 @@
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
- _PyInterpreterFrame *new_frame;
+ _PyStackRef new_frame;
oparg = CURRENT_OPARG();
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
@@ -6365,8 +6482,8 @@
if (temp == NULL) {
JUMP_TO_ERROR();
}
- new_frame = temp;
- stack_pointer[0].bits = (uintptr_t)new_frame;
+ new_frame = PyStackRef_Wrap(temp);
+ stack_pointer[0] = new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -6769,12 +6886,44 @@
break;
}
+ case _COPY_1: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-1];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _COPY_2: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-2];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _COPY_3: {
+ _PyStackRef bottom;
+ _PyStackRef top;
+ bottom = stack_pointer[-3];
+ top = PyStackRef_DUP(bottom);
+ stack_pointer[0] = top;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _COPY: {
_PyStackRef bottom;
_PyStackRef top;
oparg = CURRENT_OPARG();
bottom = stack_pointer[-1 - (oparg-1)];
- assert(oparg > 0);
top = PyStackRef_DUP(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
@@ -6814,6 +6963,32 @@
break;
}
+ case _SWAP_2: {
+ _PyStackRef top;
+ _PyStackRef bottom;
+ top = stack_pointer[-1];
+ bottom = stack_pointer[-2];
+ _PyStackRef temp = bottom;
+ bottom = top;
+ top = temp;
+ stack_pointer[-2] = bottom;
+ stack_pointer[-1] = top;
+ break;
+ }
+
+ case _SWAP_3: {
+ _PyStackRef top;
+ _PyStackRef bottom;
+ top = stack_pointer[-1];
+ bottom = stack_pointer[-3];
+ _PyStackRef temp = bottom;
+ bottom = top;
+ top = temp;
+ stack_pointer[-3] = bottom;
+ stack_pointer[-1] = top;
+ break;
+ }
+
case _SWAP: {
_PyStackRef top;
_PyStackRef bottom;
@@ -6823,7 +6998,6 @@
_PyStackRef temp = bottom;
bottom = top;
top = temp;
- assert(oparg >= 2);
stack_pointer[-2 - (oparg-2)] = bottom;
stack_pointer[-1] = top;
break;
@@ -7035,13 +7209,76 @@
case _LOAD_CONST_INLINE_BORROW: {
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_ONE: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_TWO: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
_PyStackRef pop;
_PyStackRef value;
@@ -7052,7 +7289,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -7076,13 +7313,124 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop1);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE_BORROW: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-02 00:47:04 +0000