diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 70 | ||||
| -rw-r--r-- | Python/ceval.c | 58 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 15 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 114 | ||||
| -rw-r--r-- | Python/frame.c | 25 | ||||
| -rw-r--r-- | Python/gc.c | 6 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 86 | ||||
| -rw-r--r-- | Python/stackrefs.c | 9 |
8 files changed, 233 insertions, 150 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5191b5785f7..17dc0c5bfb5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -311,7 +311,7 @@ dummy_func( inst(LOAD_CONST_MORTAL, (-- value)) { PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); + value = PyStackRef_FromPyObjectNewMortal(obj); } inst(LOAD_CONST_IMMORTAL, (-- value)) { @@ -327,6 +327,10 @@ dummy_func( } replicate(8) inst(STORE_FAST, (value --)) { + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; DEAD(value); @@ -338,6 +342,10 @@ dummy_func( }; inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) { + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value1) + ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -348,6 +356,14 @@ dummy_func( } inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value1) + ); + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value2) + ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -642,10 +658,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); INPUTS_DEAD(); - ERROR_IF(res_o == NULL, error); - res = PyStackRef_FromPyObjectSteal(res_o); + ERROR_IF(PyStackRef_IsNull(res), error); } pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { @@ -658,10 +673,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); INPUTS_DEAD(); - ERROR_IF(res_o == NULL, error); - res = PyStackRef_FromPyObjectSteal(res_o); + ERROR_IF(PyStackRef_IsNull(res), error); } pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { @@ -674,10 +688,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); INPUTS_DEAD(); - ERROR_IF(res_o == NULL, error); - res = PyStackRef_FromPyObjectSteal(res_o); + ERROR_IF(PyStackRef_IsNull(res), error); } macro(BINARY_OP_MULTIPLY_FLOAT) = @@ -733,6 +746,7 @@ dummy_func( next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -856,17 +870,16 @@ dummy_func( PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); DEOPT_IF(res_o == NULL); STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); #else DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); - Py_INCREF(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - DEAD(sub_st); - PyStackRef_CLOSE(list_st); - res = PyStackRef_FromPyObjectSteal(res_o); + STAT_INC(BINARY_SUBSCR, hit); + DECREF_INPUTS(); } inst(BINARY_OP_SUBSCR_STR_INT, (unused/5, str_st, sub_st -- res)) { @@ -886,7 +899,7 @@ dummy_func( PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectImmortal(res_o); } inst(BINARY_OP_SUBSCR_TUPLE_INT, (unused/5, tuple_st, sub_st -- res)) { @@ -903,11 +916,9 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); - Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - DEAD(sub_st); - PyStackRef_CLOSE(tuple_st); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); + DECREF_INPUTS(); } inst(BINARY_OP_SUBSCR_DICT, (unused/5, dict_st, sub_st -- res)) { @@ -1094,6 +1105,7 @@ dummy_func( inst(RETURN_VALUE, (retval -- res)) { assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; + assert(PyStackRef_IsHeapSafe(temp)); DEAD(retval); SAVE_STACK(); assert(EMPTY()); @@ -1855,7 +1867,7 @@ dummy_func( ERROR_NO_POP(); } INPUTS_DEAD(); - tup = PyStackRef_FromPyObjectSteal(tup_o); + tup = PyStackRef_FromPyObjectStealMortal(tup_o); } inst(BUILD_LIST, (values[oparg] -- list)) { @@ -1864,7 +1876,7 @@ dummy_func( ERROR_NO_POP(); } INPUTS_DEAD(); - list = PyStackRef_FromPyObjectSteal(list_o); + list = PyStackRef_FromPyObjectStealMortal(list_o); } inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) { @@ -1913,7 +1925,7 @@ dummy_func( Py_DECREF(set_o); ERROR_IF(true, error); } - set = PyStackRef_FromPyObjectSteal(set_o); + set = PyStackRef_FromPyObjectStealMortal(set_o); } inst(BUILD_MAP, (values[oparg*2] -- map)) { @@ -1929,7 +1941,7 @@ dummy_func( STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); ERROR_IF(map_o == NULL, error); - map = PyStackRef_FromPyObjectSteal(map_o); + map = PyStackRef_FromPyObjectStealMortal(map_o); } inst(SETUP_ANNOTATIONS, (--)) { @@ -3789,7 +3801,7 @@ dummy_func( DEOPT_IF(callable_o != (PyObject *)&PyType_Type); DEAD(callable); STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); PyStackRef_CLOSE(arg); } @@ -4413,9 +4425,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. SYNC_SP(); - if (temp == NULL) { - ERROR_NO_POP(); - } + ERROR_IF(temp == NULL, error); new_frame = temp; } @@ -4695,7 +4705,7 @@ dummy_func( frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); RELOAD_STACK(); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); } @@ -4706,7 +4716,7 @@ dummy_func( PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); ERROR_IF(slice_o == NULL, error); - slice = PyStackRef_FromPyObjectSteal(slice_o); + slice = PyStackRef_FromPyObjectStealMortal(slice_o); } inst(CONVERT_VALUE, (value -- result)) { diff --git a/Python/ceval.c b/Python/ceval.c index 0a3b3051373..a702d296e22 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -134,35 +134,54 @@ #ifdef Py_DEBUG static void +dump_item(_PyStackRef item) +{ + if (PyStackRef_IsNull(item)) { + printf("<NULL>"); + return; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(item); + if (obj == NULL) { + printf("<nil>"); + return; + } + if ( + obj == Py_None + || PyBool_Check(obj) + || PyLong_CheckExact(obj) + || PyFloat_CheckExact(obj) + || PyUnicode_CheckExact(obj) + ) { + if (PyObject_Print(obj, stdout, 0) == 0) { + return; + } + PyErr_Clear(); + } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); +} + +static void dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame); _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); + printf(" locals=["); + for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) { + if (ptr != locals_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); printf(" stack=["); for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { if (ptr != stack_base) { printf(", "); } - PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr); - if (obj == NULL) { - printf("<nil>"); - continue; - } - if ( - obj == Py_None - || PyBool_Check(obj) - || PyLong_CheckExact(obj) - || PyFloat_CheckExact(obj) - || PyUnicode_CheckExact(obj) - ) { - if (PyObject_Print(obj, stdout, 0) == 0) { - continue; - } - PyErr_Clear(); - } - // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(obj)->tp_name, PyStackRef_AsPyObjectBorrow(*ptr)); + dump_item(*ptr); } printf("]\n"); fflush(stdout); @@ -1390,7 +1409,6 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, { PyCodeObject *co = (PyCodeObject*)func->func_code; const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; - /* Create a dictionary for keyword parameters (**kwags) */ PyObject *kwdict; Py_ssize_t i; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1bef2b845d0..f19ffd23161 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -440,28 +440,13 @@ do { \ /* How much scratch space to give stackref to PyObject* conversion. */ #define MAX_STACKREF_SCRATCH 10 -#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); -#else -#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ - PyObject **NAME = (PyObject **)ARGS; \ - assert(NAME != NULL); -#endif -#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ _PyObjectArray_Free(NAME - 1, NAME##_temp); -#else -#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ - (void)(NAME); -#endif -#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define CONVERSION_FAILED(NAME) ((NAME) == NULL) -#else -#define CONVERSION_FAILED(NAME) (0) -#endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 94f05c62089..8886564cded 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -218,7 +218,7 @@ _PyStackRef value; oparg = CURRENT_OPARG(); PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); + value = PyStackRef_FromPyObjectNewMortal(obj); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -306,6 +306,10 @@ oparg = 0; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -321,6 +325,10 @@ oparg = 1; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -336,6 +344,10 @@ oparg = 2; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -351,6 +363,10 @@ oparg = 3; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -366,6 +382,10 @@ oparg = 4; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -381,6 +401,10 @@ oparg = 5; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -396,6 +420,10 @@ oparg = 6; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -411,6 +439,10 @@ oparg = 7; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -425,6 +457,10 @@ _PyStackRef value; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -863,13 +899,13 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - stack_pointer += -2; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -890,13 +926,13 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - stack_pointer += -2; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -917,13 +953,13 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - stack_pointer += -2; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -991,6 +1027,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1205,6 +1242,7 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); #else if (index >= PyList_GET_SIZE(list)) { UOP_STAT_INC(uopcode, miss); @@ -1213,18 +1251,21 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); - Py_INCREF(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(BINARY_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(sub_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; break; } @@ -1267,7 +1308,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectImmortal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1303,17 +1344,17 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); - Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tuple_st); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -1654,6 +1695,7 @@ retval = stack_pointer[-1]; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; + assert(PyStackRef_IsHeapSafe(temp)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2534,7 +2576,7 @@ if (tup_o == NULL) { JUMP_TO_ERROR(); } - tup = PyStackRef_FromPyObjectSteal(tup_o); + tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2552,7 +2594,7 @@ if (list_o == NULL) { JUMP_TO_ERROR(); } - list = PyStackRef_FromPyObjectSteal(list_o); + list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2666,7 +2708,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - set = PyStackRef_FromPyObjectSteal(set_o); + set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[0] = set; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2712,7 +2754,7 @@ if (map_o == NULL) { JUMP_TO_ERROR(); } - map = PyStackRef_FromPyObjectSteal(map_o); + map = PyStackRef_FromPyObjectStealMortal(map_o); stack_pointer[0] = map; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5047,7 +5089,7 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -6394,7 +6436,7 @@ frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -6424,7 +6466,7 @@ if (slice_o == NULL) { JUMP_TO_ERROR(); } - slice = PyStackRef_FromPyObjectSteal(slice_o); + slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/frame.c b/Python/frame.c index 68ac2acbaee..afb3e768491 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -50,24 +50,23 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); - Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; - memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); - frame = (_PyInterpreterFrame *)f->_f_frame_data; - frame->stackpointer = (_PyStackRef *)(((char *)frame) + size); - frame->f_executable = PyStackRef_DUP(frame->f_executable); - f->f_frame = frame; - frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; - if (_PyFrame_IsIncomplete(frame)) { + _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)f->_f_frame_data; + _PyFrame_Copy(frame, new_frame); + // _PyFrame_Copy takes the reference to the executable, + // so we need to restore it. + frame->f_executable = PyStackRef_DUP(new_frame->f_executable); + f->f_frame = new_frame; + new_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; + if (_PyFrame_IsIncomplete(new_frame)) { // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: - PyCodeObject *code = _PyFrame_GetCode(frame); - frame->instr_ptr = - _PyFrame_GetBytecode(frame) + code->_co_firsttraceable + 1; + PyCodeObject *code = _PyFrame_GetCode(new_frame); + new_frame->instr_ptr = + _PyFrame_GetBytecode(new_frame) + code->_co_firsttraceable + 1; } - assert(!_PyFrame_IsIncomplete(frame)); + assert(!_PyFrame_IsIncomplete(new_frame)); assert(f->f_back == NULL); _PyInterpreterFrame *prev = _PyFrame_GetFirstComplete(frame->previous); - frame->previous = NULL; if (prev) { assert(prev->owner < FRAME_OWNED_BY_INTERPRETER); /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */ diff --git a/Python/gc.c b/Python/gc.c index 7faf368f358..f2a88657e8d 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1488,11 +1488,11 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b objects_marked += move_to_reachable(func, &reachable, visited_space); while (sp > locals) { sp--; - if (PyStackRef_IsNull(*sp)) { + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (op == NULL || _Py_IsImmortal(op)) { continue; } - PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); - if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op) && gc_old_space(gc) != visited_space) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c9371f77e1d..ad0e4118817 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -123,11 +123,10 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { JUMP_TO_LABEL(pop_2_error); } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -352,6 +351,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -436,11 +436,10 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { JUMP_TO_LABEL(pop_2_error); } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -691,6 +690,7 @@ JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); #else if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); @@ -700,18 +700,21 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); - Py_INCREF(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(BINARY_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(sub_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; DISPATCH(); } @@ -770,7 +773,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectImmortal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -821,17 +824,17 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); - Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tuple_st); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; DISPATCH(); } @@ -877,11 +880,10 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { JUMP_TO_LABEL(pop_2_error); } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1021,7 +1023,7 @@ if (list_o == NULL) { JUMP_TO_LABEL(error); } - list = PyStackRef_FromPyObjectSteal(list_o); + list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1073,7 +1075,7 @@ if (map_o == NULL) { JUMP_TO_LABEL(error); } - map = PyStackRef_FromPyObjectSteal(map_o); + map = PyStackRef_FromPyObjectStealMortal(map_o); stack_pointer[0] = map; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1131,7 +1133,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } - set = PyStackRef_FromPyObjectSteal(set_o); + set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[0] = set; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1166,7 +1168,7 @@ if (slice_o == NULL) { JUMP_TO_LABEL(error); } - slice = PyStackRef_FromPyObjectSteal(slice_o); + slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1235,7 +1237,7 @@ if (tup_o == NULL) { JUMP_TO_LABEL(error); } - tup = PyStackRef_FromPyObjectSteal(tup_o); + tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4271,7 +4273,7 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7237,6 +7239,7 @@ retval = val; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; + assert(PyStackRef_IsHeapSafe(temp)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8741,7 +8744,7 @@ static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); + value = PyStackRef_FromPyObjectNewMortal(obj); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -10254,7 +10257,7 @@ frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -10275,6 +10278,7 @@ retval = stack_pointer[-1]; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; + assert(PyStackRef_IsHeapSafe(temp)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10910,6 +10914,10 @@ INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; @@ -10931,6 +10939,10 @@ _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value1) + ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -10955,6 +10967,14 @@ _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value1) + ); + assert( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value2) + ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 2e889b19bf3..a7693ba75c5 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -1,6 +1,7 @@ #include "Python.h" +#include "pycore_object.h" #include "pycore_stackref.h" #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) @@ -175,8 +176,16 @@ _Py_stackref_report_leaks(PyInterpreterState *interp) int leak = 0; _Py_hashtable_foreach(interp->open_stackrefs_table, report_leak, &leak); if (leak) { + fflush(stdout); Py_FatalError("Stackrefs leaked."); } } +void +PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) +{ + PyObject *obj = _Py_stackref_close(ref); + _Py_DECREF_SPECIALIZED(obj, destruct); +} + #endif |