diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 149 |
1 files changed, 120 insertions, 29 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index e39ec67614b..6620c00d0f9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -98,6 +98,12 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs); static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); +static InterpreterFrame * +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames, int steal_args); +static int +_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame); #define NAME_ERROR_MSG \ "name '%.200s' is not defined" @@ -1516,6 +1522,12 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame) return 0; } +static PyObject * +make_coro(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames); + static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); @@ -1543,10 +1555,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *retval = NULL; /* Return value */ _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; - if (_Py_EnterRecursiveCall(tstate, "")) { - return NULL; - } - CFrame cframe; /* WARNING: Because the CFrame lives on the C stack, @@ -1558,9 +1566,18 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr cframe.previous = prev_cframe; tstate->cframe = &cframe; + assert(frame->depth == 0); /* push frame */ tstate->frame = frame; +start_frame: + if (_Py_EnterRecursiveCall(tstate, "")) { + tstate->recursion_depth++; + goto exit_eval_frame; + } + + assert(frame == tstate->frame); + if (cframe.use_tracing) { if (trace_function_entry(tstate, frame)) { goto exit_eval_frame; @@ -1582,7 +1599,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - +resume_frame: + co = frame->f_code; PyObject *names = co->co_names; PyObject *consts = co->co_consts; _Py_CODEUNIT *first_instr = co->co_firstinstr; @@ -1594,12 +1612,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in - direct succession without updating frame->f_lasti. A successful - prediction effectively links the two codes together as if they - were a single new opcode; accordingly,frame->f_lasti will point to - the first code in the pair (for instance, GET_ITER followed by - FOR_ITER is effectively a single opcode and frame->f_lasti will point - to the beginning of the combined pair.) + direct succession. A successful prediction effectively links the two + codes together as if they were a single new opcode, but the value + of frame->f_lasti is correctly updated so potential inlined calls + or lookups of frame->f_lasti are aways correct when the macros are used. */ assert(frame->f_lasti >= -1); _Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1; @@ -1625,6 +1641,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr #endif if (throwflag) { /* support for generator.throw() */ + throwflag = 0; goto error; } @@ -4591,10 +4608,44 @@ check_eval_breaker: TARGET(CALL_FUNCTION) { PREDICTED(CALL_FUNCTION); - PyObject **sp, *res; - sp = stack_pointer; - res = call_function(tstate, &sp, oparg, NULL, cframe.use_tracing); - stack_pointer = sp; + PyObject *res; + + // Check if the call can be inlined or not + PyObject *function = PEEK(oparg + 1); + if (Py_TYPE(function) == &PyFunction_Type) { + PyCodeObject *code = (PyCodeObject*)PyFunction_GET_CODE(function); + PyObject *locals = code->co_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function); + if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == 0) { + InterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, PyFunction_AS_FRAME_CONSTRUCTOR(function), locals, stack_pointer-oparg, oparg, NULL, 1); + if (new_frame == NULL) { + // When we exit here, we own all variables in the stack (the frame creation has not stolen + // any variable) so we need to clean the whole stack (done in the "error" label). + goto error; + } + STACK_SHRINK(oparg + 1); + assert(tstate->interp->eval_frame != NULL); + // The frame has stolen all the arguments from the stack, so there is no need to clean them up.``` + Py_DECREF(function); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->depth = frame->depth + 1; + tstate->frame = frame = new_frame; + goto start_frame; + } + else { + /* Callable is a generator or coroutine function: create coroutine or generator. */ + res = make_coro(tstate, PyFunction_AS_FRAME_CONSTRUCTOR(function), locals, stack_pointer-oparg, oparg, NULL); + STACK_SHRINK(oparg + 1); + for (int i = 0; i < oparg + 1; i++) { + Py_DECREF(stack_pointer[i]); + } + } + } + else { + PyObject **sp = stack_pointer; + res = call_function(tstate, &sp, oparg, NULL, cframe.use_tracing); + stack_pointer = sp; + } PUSH(res); if (res == NULL) { goto error; @@ -5018,14 +5069,28 @@ exiting: /* pop frame */ exit_eval_frame: - /* Restore previous cframe */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(frame); _Py_LeaveRecursiveCall(tstate); + + if (frame->depth) { + _PyFrame_StackPush(frame->previous, retval); + if (_PyEvalFrameClearAndPop(tstate, frame)) { + retval = NULL; + } + frame = tstate->frame; + if (retval == NULL) { + assert(_PyErr_Occurred(tstate)); + throwflag = 1; + } + retval = NULL; + goto resume_frame; + } tstate->frame = frame->previous; + + /* Restore previous cframe */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); } @@ -5336,7 +5401,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i static int initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, PyObject **localsplus, PyObject *const *args, - Py_ssize_t argcount, PyObject *kwnames) + Py_ssize_t argcount, PyObject *kwnames, int steal_args) { PyCodeObject *co = (PyCodeObject*)con->fc_code; const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; @@ -5346,8 +5411,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, Py_ssize_t i; if (co->co_flags & CO_VARKEYWORDS) { kwdict = PyDict_New(); - if (kwdict == NULL) + if (kwdict == NULL) { goto fail; + } i = total_args; if (co->co_flags & CO_VARARGS) { i++; @@ -5369,14 +5435,21 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, } for (j = 0; j < n; j++) { PyObject *x = args[j]; - Py_INCREF(x); + if (!steal_args) { + Py_INCREF(x); + } assert(localsplus[j] == NULL); localsplus[j] = x; } /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { - PyObject *u = _PyTuple_FromArray(args + n, argcount - n); + PyObject *u = NULL; + if (steal_args) { + u = _PyTuple_FromArraySteal(args + n, argcount - n); + } else { + u = _PyTuple_FromArray(args + n, argcount - n); + } if (u == NULL) { goto fail; } @@ -5442,6 +5515,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, if (PyDict_SetItem(kwdict, keyword, value) == -1) { goto fail; } + if (steal_args) { + Py_DECREF(value); + } continue; kw_found: @@ -5451,7 +5527,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, con->fc_qualname, keyword); goto fail; } - Py_INCREF(value); + if (!steal_args) { + Py_INCREF(value); + } localsplus[j] = value; } } @@ -5555,7 +5633,7 @@ make_coro_frame(PyThreadState *tstate, } _PyFrame_InitializeSpecials(frame, con, locals, code->co_nlocalsplus); assert(frame->frame_obj == NULL); - if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames)) { + if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames, 0)) { _PyFrame_Clear(frame, 1); return NULL; } @@ -5581,17 +5659,30 @@ make_coro(PyThreadState *tstate, PyFrameConstructor *con, return gen; } +// If *steal_args* is set, the function will steal the references to all the arguments. +// In case of error, the function returns null and if *steal_args* is set, the caller +// will still own all the arguments. static InterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, int steal_args) { InterpreterFrame * frame = _PyThreadState_PushFrame(tstate, con, locals); if (frame == NULL) { return NULL; } PyObject **localsarray = _PyFrame_GetLocalsArray(frame); - if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames)) { + if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames, steal_args)) { + if (steal_args) { + // If we failed to initialize locals, make sure the caller still own all the + // arguments. Notice that we only need to increase the reference count of the + // *valid* arguments (i.e. the ones that fit into the frame). + PyCodeObject *co = (PyCodeObject*)con->fc_code; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + for (Py_ssize_t i = 0; i < Py_MIN(argcount, total_args); i++) { + Py_XINCREF(frame->localsplus[i]); + } + } _PyFrame_Clear(frame, 0); return NULL; } @@ -5606,9 +5697,9 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame) ++tstate->recursion_depth; assert(frame->frame_obj == NULL || frame->frame_obj->f_own_locals_memory == 0); if (_PyFrame_Clear(frame, 0)) { + --tstate->recursion_depth; return -1; } - assert(frame->frame_obj == NULL); --tstate->recursion_depth; tstate->frame = frame->previous; _PyThreadState_PopFrame(tstate, frame); @@ -5628,7 +5719,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con, return make_coro(tstate, con, locals, args, argcount, kwnames); } InterpreterFrame *frame = _PyEvalFramePushAndInit( - tstate, con, locals, args, argcount, kwnames); + tstate, con, locals, args, argcount, kwnames, 0); if (frame == NULL) { return NULL; } |