diff options
Diffstat (limited to 'Python/generated_cases.c.h')
| -rw-r--r-- | Python/generated_cases.c.h | 478 |
1 files changed, 222 insertions, 256 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b3f2a2067f7..5be2671700a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -905,7 +905,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectImmortal(res_o); + res = PyStackRef_FromPyObjectBorrow(res_o); } stack_pointer[0] = res; stack_pointer += 1; @@ -3475,58 +3475,79 @@ INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; + _PyStackRef nos; _PyStackRef self; _PyStackRef arg; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (self_o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!PyList_Check(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - #if TIER_ONE + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!PyList_CheckExact(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } DISPATCH(); } @@ -5710,17 +5731,19 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; // _SPECIALIZE_FOR_ITER { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -5731,29 +5754,44 @@ // _FOR_ITER { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); } - next = PyStackRef_FromPyObjectSteal(next_o); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5785,7 +5823,7 @@ } // _FOR_ITER_GEN_FRAME { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5842,26 +5880,22 @@ INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -5870,42 +5904,30 @@ } // _ITER_JUMP_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } #endif } // _ITER_NEXT_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); @@ -5913,16 +5935,17 @@ JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5945,7 +5968,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_RANGE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -6008,63 +6031,44 @@ INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - #endif + assert(PyStackRef_IsTaggedInt(null_or_index)); } // _ITER_JUMP_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } } // _ITER_NEXT_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6195,25 +6199,37 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6978,7 +6994,7 @@ _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -7040,34 +7056,48 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + JUMPBY(oparg + 1); + DISPATCH(); + } INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o != NULL) { + next = PyStackRef_FromPyObjectSteal(next_o); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); } stack_pointer[0] = next; stack_pointer += 1; @@ -7335,9 +7365,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; - iter = stack_pointer[-1]; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); @@ -8981,63 +9014,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION_FT - #ifdef Py_GIL_DISABLED - uint8_t expected = LOAD_CONST; - if (!_Py_atomic_compare_exchange_uint8( - &this_instr->op.code, &expected, - _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { - assert(expected >= MIN_INSTRUMENTED_OPCODE); - } - #else - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } - #endif - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_CONST_IMMORTAL) { - #if Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST_IMMORTAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_CONST_MORTAL) { - #if Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST_MORTAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_MORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNewMortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -9572,7 +9551,7 @@ _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -10155,12 +10134,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -10308,7 +10290,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -11219,10 +11201,6 @@ INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -11244,10 +11222,6 @@ _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -11272,14 +11246,6 @@ _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value2) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); |