diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 183 |
1 files changed, 122 insertions, 61 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index d36e15e55f5..2412256ace8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -39,6 +39,7 @@ #include "opcode.h" #include "pydtrace.h" #include "setobject.h" +#include "pycore_stackref.h" #include <stdbool.h> // bool @@ -104,33 +105,34 @@ #ifdef LLTRACE static void -dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - PyObject **stack_base = _PyFrame_Stackbase(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); - for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) { + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { if (ptr != stack_base) { printf(", "); } - if (*ptr == NULL) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr); + if (obj == NULL) { printf("<nil>"); continue; } if ( - *ptr == Py_None - || PyBool_Check(*ptr) - || PyLong_CheckExact(*ptr) - || PyFloat_CheckExact(*ptr) - || PyUnicode_CheckExact(*ptr) + obj == Py_None + || PyBool_Check(obj) + || PyLong_CheckExact(obj) + || PyFloat_CheckExact(obj) + || PyUnicode_CheckExact(obj) ) { - if (PyObject_Print(*ptr, stdout, 0) == 0) { + if (PyObject_Print(obj, stdout, 0) == 0) { continue; } PyErr_Clear(); } // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits)); } printf("]\n"); fflush(stdout); @@ -139,7 +141,7 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) static void lltrace_instruction(_PyInterpreterFrame *frame, - PyObject **stack_pointer, + _PyStackRef *stack_pointer, _Py_CODEUNIT *next_instr, int opcode, int oparg) @@ -695,6 +697,35 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif +PyObject ** +_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) +{ + PyObject **result; + if (nargs > MAX_STACKREF_SCRATCH) { + // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set. + result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); + if (result == NULL) { + return NULL; + } + result++; + } + else { + result = scratch; + } + for (int i = 0; i < nargs; i++) { + result[i] = PyStackRef_AsPyObjectBorrow(input[i]); + } + return result; +} + +void +_PyObjectArray_Free(PyObject **array, PyObject **scratch) +{ + if (array != scratch) { + PyMem_Free(array); + } +} + /* _PyEval_EvalFrameDefault() is a *big* function, * so consume 3 units of C stack */ #define PY_EVAL_C_STACK_UNITS 2 @@ -773,7 +804,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Local "register" variables. * These are cached values from the frame and code object. */ _Py_CODEUNIT *next_instr; - PyObject **stack_pointer; + _PyStackRef *stack_pointer; #if defined(_Py_TIER2) && !defined(_Py_JIT) /* Tier 2 interpreter state */ @@ -916,10 +947,9 @@ exception_unwind: assert(_PyErr_Occurred(tstate)); /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); + PyStackRef_XCLOSE(POP()); } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -928,10 +958,9 @@ exception_unwind: } assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); + PyStackRef_XCLOSE(POP()); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -939,7 +968,7 @@ exception_unwind: if (lasti == NULL) { goto exception_unwind; } - PUSH(lasti); + PUSH(PyStackRef_FromPyObjectSteal(lasti)); } /* Make the raw exception data @@ -947,7 +976,7 @@ exception_unwind: so a program can emulate the Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(exc); + PUSH(PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { @@ -1217,7 +1246,7 @@ format_missing(PyThreadState *tstate, const char *kind, static void missing_arguments(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { Py_ssize_t i, j = 0; Py_ssize_t start, end; @@ -1238,7 +1267,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, end = start + co->co_kwonlyargcount; } for (i = start; i < end; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *name = PyObject_Repr(raw); if (name == NULL) { @@ -1256,7 +1285,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, static void too_many_positional(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t given, PyObject *defaults, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { int plural; Py_ssize_t kwonly_given = 0; @@ -1267,7 +1296,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, assert((co->co_flags & CO_VARARGS) == 0); /* Count missing keyword-only args. */ for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { - if (localsplus[i] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) { kwonly_given++; } } @@ -1445,7 +1474,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i static int initialize_locals(PyThreadState *tstate, PyFunctionObject *func, - PyObject **localsplus, PyObject *const *args, + _PyStackRef *localsplus, _PyStackRef const *args, Py_ssize_t argcount, PyObject *kwnames) { PyCodeObject *co = (PyCodeObject*)func->func_code; @@ -1463,8 +1492,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { i++; } - assert(localsplus[i] == NULL); - localsplus[i] = kwdict; + assert(PyStackRef_IsNull(localsplus[i])); + localsplus[i] = PyStackRef_FromPyObjectSteal(kwdict); } else { kwdict = NULL; @@ -1479,9 +1508,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, n = argcount; } for (j = 0; j < n; j++) { - PyObject *x = args[j]; - assert(localsplus[j] == NULL); - localsplus[j] = x; + assert(PyStackRef_IsNull(localsplus[j])); + localsplus[j] = args[j]; } /* Pack other positional arguments into the *args argument */ @@ -1492,18 +1520,23 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, } else { assert(args != NULL); - u = _PyTuple_FromArraySteal(args + n, argcount - n); + STACKREFS_TO_PYOBJECTS((_PyStackRef *)args, argcount, args_o); + if (args_o == NULL) { + goto fail_pre_positional; + } + u = _PyTuple_FromArraySteal((args_o + n), argcount - n); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); } if (u == NULL) { goto fail_post_positional; } - assert(localsplus[total_args] == NULL); - localsplus[total_args] = u; + assert(PyStackRef_AsPyObjectBorrow(localsplus[total_args]) == NULL); + localsplus[total_args] = PyStackRef_FromPyObjectSteal(u); } else if (argcount > n) { /* Too many positional args. Error is reported later */ for (j = n; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } @@ -1513,7 +1546,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, for (i = 0; i < kwcount; i++) { PyObject **co_varnames; PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); - PyObject *value = args[i+argcount]; + _PyStackRef value_stackref = args[i+argcount]; Py_ssize_t j; if (keyword == NULL || !PyUnicode_Check(keyword)) { @@ -1586,27 +1619,26 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - if (PyDict_SetItem(kwdict, keyword, value) == -1) { + if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectSteal(value_stackref)) == -1) { goto kw_fail; } - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); continue; kw_fail: for (;i < kwcount; i++) { - PyObject *value = args[i+argcount]; - Py_DECREF(value); + PyStackRef_CLOSE(args[i+argcount]); } goto fail_post_args; kw_found: - if (localsplus[j] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[j]) != NULL) { _PyErr_Format(tstate, PyExc_TypeError, "%U() got multiple values for argument '%S'", func->func_qualname, keyword); goto kw_fail; } - localsplus[j] = value; + localsplus[j] = value_stackref; } } @@ -1623,7 +1655,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, Py_ssize_t m = co->co_argcount - defcount; Py_ssize_t missing = 0; for (i = argcount; i < m; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { missing++; } } @@ -1639,9 +1671,9 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (defcount) { PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); for (; i < defcount; i++) { - if (localsplus[m+i] == NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[m+i]) == NULL) { PyObject *def = defs[i]; - localsplus[m+i] = Py_NewRef(def); + localsplus[m+i] = PyStackRef_FromPyObjectNew(def); } } } @@ -1651,7 +1683,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_kwonlyargcount > 0) { Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { - if (localsplus[i] != NULL) + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) continue; PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (func->func_kwdefaults != NULL) { @@ -1660,7 +1692,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto fail_post_args; } if (def) { - localsplus[i] = def; + localsplus[i] = PyStackRef_FromPyObjectSteal(def); continue; } } @@ -1676,14 +1708,14 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, fail_pre_positional: for (j = 0; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } /* fall through */ fail_post_positional: if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (j = argcount; j < argcount+kwcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } /* fall through */ @@ -1738,7 +1770,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) /* Consumes references to func, locals and all the args */ _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, + PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames) { PyCodeObject * code = (PyCodeObject *)func->func_code; @@ -1759,18 +1791,45 @@ fail: Py_DECREF(func); Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); + PyStackRef_CLOSE(args[i+argcount]); } } PyErr_NoMemory(); return NULL; } +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject *const* args, + size_t argcount, PyObject *kwnames) +{ +#if defined(Py_GIL_DISABLED) + size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + size_t total_argcount = argcount + kw_count; + _PyStackRef *tagged_args_buffer = PyMem_Malloc(sizeof(_PyStackRef) * total_argcount); + if (tagged_args_buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (size_t i = 0; i < argcount; i++) { + tagged_args_buffer[i] = PyStackRef_FromPyObjectSteal(args[i]); + } + for (size_t i = 0; i < kw_count; i++) { + tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); + } + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + PyMem_Free(tagged_args_buffer); + return res; +#else + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); +#endif +} + /* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. Steals references to func, callargs and kwargs. */ @@ -1795,7 +1854,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(PyTuple_GET_ITEM(callargs, i)); } } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, newargs, nargs, kwnames ); @@ -1833,7 +1892,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(args[i+argcount]); } } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( tstate, func, locals, args, argcount, kwnames); if (frame == NULL) { return NULL; @@ -2085,8 +2144,8 @@ _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, */ int -_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, + int argcnt, int argcntafter, _PyStackRef *sp) { int i = 0, j = 0; Py_ssize_t ll = 0; @@ -2094,6 +2153,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, PyObject *w; PyObject *l = NULL; /* variable list */ + PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref); assert(v != NULL); it = PyObject_GetIter(v); @@ -2128,7 +2188,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, } goto Error; } - *--sp = w; + *--sp = PyStackRef_FromPyObjectSteal(w); } if (argcntafter == -1) { @@ -2150,7 +2210,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, l = PySequence_List(it); if (l == NULL) goto Error; - *--sp = l; + *--sp = PyStackRef_FromPyObjectSteal(l); i++; ll = PyList_GET_SIZE(l); @@ -2163,7 +2223,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, /* Pop the "after-variable" args off the list. */ for (j = argcntafter; j > 0; j--, i++) { - *--sp = PyList_GET_ITEM(l, ll - j); + *--sp = PyStackRef_FromPyObjectSteal(PyList_GET_ITEM(l, ll - j)); } /* Resize the list. */ Py_SET_SIZE(l, ll - argcntafter); @@ -2171,8 +2231,9 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, return 1; Error: - for (; i > 0; i--, sp++) - Py_DECREF(*sp); + for (; i > 0; i--, sp++) { + PyStackRef_CLOSE(*sp); + } Py_XDECREF(it); return 0; } |