aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python/executor_cases.c.h
diff options
context:
space:
mode:
Diffstat (limited to 'Python/executor_cases.c.h')
-rw-r--r--Python/executor_cases.c.h505
1 files changed, 325 insertions, 180 deletions
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 41c9bd5ba70..d19605169d5 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -319,25 +319,11 @@
break;
}
- /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
-
- case _LOAD_CONST_MORTAL: {
- _PyStackRef value;
- oparg = CURRENT_OPARG();
- PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- value = PyStackRef_FromPyObjectNewMortal(obj);
- stack_pointer[0] = value;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_CONST_IMMORTAL: {
+ case _LOAD_CONST: {
_PyStackRef value;
oparg = CURRENT_OPARG();
PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
- assert(_Py_IsImmortal(obj));
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -350,7 +336,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -363,7 +349,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -376,7 +362,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -389,7 +375,7 @@
assert(oparg == CURRENT_OPARG());
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -401,7 +387,7 @@
oparg = CURRENT_OPARG();
assert(oparg < _PY_NSMALLPOSINTS);
PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
- value = PyStackRef_FromPyObjectImmortal(obj);
+ value = PyStackRef_FromPyObjectBorrow(obj);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -413,10 +399,6 @@
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -432,10 +414,6 @@
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -451,10 +429,6 @@
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -470,10 +444,6 @@
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -489,10 +459,6 @@
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -508,10 +474,6 @@
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -527,10 +489,6 @@
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -546,10 +504,6 @@
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -564,10 +518,6 @@
_PyStackRef value;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
- assert(
- ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
- PyStackRef_IsHeapSafe(value)
- );
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = value;
stack_pointer += -1;
@@ -584,7 +534,25 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(value);
+ PyStackRef_XCLOSE(value);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_TWO: {
+ _PyStackRef tos;
+ _PyStackRef nos;
+ tos = stack_pointer[-1];
+ nos = stack_pointer[-2];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(tos);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(nos);
stack_pointer = _PyFrame_GetStackPointer(frame);
break;
}
@@ -609,6 +577,20 @@
break;
}
+ case _POP_ITER: {
+ _PyStackRef index_or_null;
+ _PyStackRef iter;
+ index_or_null = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ (void)index_or_null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iter);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _END_SEND: {
_PyStackRef value;
_PyStackRef receiver;
@@ -1403,7 +1385,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(str_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
- res = PyStackRef_FromPyObjectImmortal(res_o);
+ res = PyStackRef_FromPyObjectBorrow(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4204,25 +4186,37 @@
case _GET_ITER: {
_PyStackRef iterable;
_PyStackRef iter;
+ _PyStackRef index_or_null;
iterable = stack_pointer[-1];
#ifdef Py_STATS
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_GatherStats_GetIter(iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
#endif
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(iterable);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (iter_o == NULL) {
- JUMP_TO_ERROR();
+
+ PyTypeObject *tp = PyStackRef_TYPE(iterable);
+ if (tp == &PyTuple_Type || tp == &PyList_Type) {
+ iter = iterable;
+ index_or_null = PyStackRef_TagInt(0);
}
- iter = PyStackRef_FromPyObjectSteal(iter_o);
- stack_pointer[0] = iter;
+ else {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(iterable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (iter_o == NULL) {
+ JUMP_TO_ERROR();
+ }
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
+ index_or_null = PyStackRef_NULL;
+ stack_pointer += 1;
+ }
+ stack_pointer[-1] = iter;
+ stack_pointer[0] = index_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@@ -4269,32 +4263,25 @@
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_ERROR();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ next = item;
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4304,21 +4291,18 @@
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyListIter_Type) {
+ if (Py_TYPE(iter_o) != &PyList_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ assert(PyStackRef_IsTaggedInt(null_or_index));
#ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
- !_PyObject_GC_IS_SHARED(it->it_seq)) {
+ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4329,24 +4313,17 @@
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
#ifndef Py_GIL_DISABLED
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- if (seq == NULL) {
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(list_o) == &PyList_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
#endif
break;
}
@@ -4354,38 +4331,30 @@
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */
case _ITER_NEXT_LIST_TIER_TWO: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyListIterObject *it = (_PyListIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyListIter_Type);
- PyListObject *seq = it->it_seq;
- assert(seq);
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(PyList_CheckExact(list_o));
#ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
- _PyObject_GC_IS_SHARED(seq));
+ assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) ||
+ _PyObject_GC_IS_SHARED(list_o));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
- int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
+ int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (result < 0) {
+ if (result <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (result == 0) {
- it->it_index = -1;
- if (1) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- }
- it->it_index++;
#else
- assert(it->it_index < PyList_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index)));
#endif
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4393,39 +4362,29 @@
}
case _ITER_CHECK_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- #ifdef Py_GIL_DISABLED
- if (!_PyObject_IsUniquelyReferenced(iter_o)) {
+ if (Py_TYPE(iter_o) != &PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- #endif
+ assert(PyStackRef_IsTaggedInt(null_or_index));
break;
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- PyTupleObject *seq = it->it_seq;
- if (seq == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (it->it_index >= PyTuple_GET_SIZE(seq)) {
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4433,19 +4392,18 @@
}
case _ITER_NEXT_TUPLE: {
+ _PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
- assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
- PyTupleObject *seq = it->it_seq;
- #ifdef Py_GIL_DISABLED
- assert(_PyObject_IsUniquelyReferenced(iter_o));
- #endif
- assert(seq);
- assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
+ null_or_index = stack_pointer[-1];
+ iter = stack_pointer[-2];
+ PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter);
+ assert(Py_TYPE(tuple_o) == &PyTuple_Type);
+ uintptr_t i = PyStackRef_UntagInt(null_or_index);
+ assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i));
+ null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -4454,7 +4412,7 @@
case _ITER_CHECK_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -4473,7 +4431,7 @@
case _GUARD_NOT_EXHAUSTED_RANGE: {
_PyStackRef iter;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
@@ -4486,7 +4444,7 @@
case _ITER_NEXT_RANGE: {
_PyStackRef iter;
_PyStackRef next;
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
@@ -4511,7 +4469,7 @@
_PyStackRef iter;
_PyInterpreterFrame *gen_frame;
oparg = CURRENT_OPARG();
- iter = stack_pointer[-1];
+ iter = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
@@ -5276,6 +5234,17 @@
break;
}
+ case _GUARD_NOS_NOT_NULL: {
+ _PyStackRef nos;
+ nos = stack_pointer[-2];
+ PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
+ if (o == NULL) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
case _GUARD_THIRD_NULL: {
_PyStackRef null;
null = stack_pointer[-3];
@@ -5920,6 +5889,18 @@
break;
}
+ case _GUARD_CALLABLE_LIST_APPEND: {
+ _PyStackRef callable;
+ callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyInterpreterState *interp = tstate->interp;
+ if (callable_o != interp->callable_cache.list_append) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
case _CALL_LIST_APPEND: {
_PyStackRef arg;
_PyStackRef self;
@@ -5929,18 +5910,8 @@
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
- PyInterpreterState *interp = tstate->interp;
- if (callable_o != interp->callable_cache.list_append) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (self_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!PyList_Check(self_o)) {
+ if (!PyList_CheckExact(self_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -7042,13 +7013,76 @@
case _LOAD_CONST_INLINE_BORROW: {
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_ONE: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
+ case _POP_CALL_TWO: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ break;
+ }
+
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
_PyStackRef pop;
_PyStackRef value;
@@ -7059,7 +7093,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -7083,13 +7117,124 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop1);
stack_pointer = _PyFrame_GetStackPointer(frame);
- value = PyStackRef_FromPyObjectImmortal(ptr);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ null = stack_pointer[-1];
+ callable = stack_pointer[-2];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
+ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop = stack_pointer[-1];
+ null = stack_pointer[-2];
+ callable = stack_pointer[-3];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: {
+ _PyStackRef pop2;
+ _PyStackRef pop1;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef value;
+ pop2 = stack_pointer[-1];
+ pop1 = stack_pointer[-2];
+ null = stack_pointer[-3];
+ callable = stack_pointer[-4];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop2);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ (void)null;
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(callable);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
+ case _LOAD_CONST_UNDER_INLINE_BORROW: {
+ _PyStackRef old;
+ _PyStackRef value;
+ _PyStackRef new;
+ old = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ new = old;
+ value = PyStackRef_FromPyObjectBorrow(ptr);
+ stack_pointer[-1] = value;
+ stack_pointer[0] = new;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-11 04:00:05 +0000