diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 53 | ||||
| -rw-r--r-- | Python/ceval.c | 195 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 14 | ||||
| -rw-r--r-- | Python/executor.c | 125 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 52 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 54 |
6 files changed, 249 insertions, 244 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b9434390ea4..3a191aca730 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -77,7 +77,6 @@ dummy_func( PyObject **stack_pointer, PyObject *kwnames, int throwflag, - binaryfunc binary_ops[], PyObject *args[] ) { @@ -893,7 +892,7 @@ dummy_func( iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { - format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } DECREF_INPUTS(); @@ -1120,9 +1119,9 @@ dummy_func( err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); goto error; } } @@ -1145,7 +1144,7 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; - int res = unpack_iterable(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1185,7 +1184,7 @@ dummy_func( inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; - int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1235,8 +1234,8 @@ dummy_func( // Can't use ERROR_IF here. if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } goto error; } @@ -1274,7 +1273,7 @@ dummy_func( goto error; } if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); goto error; @@ -1315,8 +1314,8 @@ dummy_func( if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } ERROR_IF(true, error); } @@ -1331,7 +1330,7 @@ dummy_func( /* namespace 2: builtins */ ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); ERROR_IF(true, error); @@ -1413,7 +1412,7 @@ dummy_func( // Can't use ERROR_IF here. // Fortunately we don't need its superpower. if (oldobj == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } PyCell_SET(cell, NULL); @@ -1434,7 +1433,7 @@ dummy_func( PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } Py_INCREF(value); @@ -1445,7 +1444,7 @@ dummy_func( PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } Py_INCREF(value); @@ -1612,7 +1611,7 @@ dummy_func( PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); DECREF_INPUTS(); ERROR_IF(true, error); } @@ -2126,15 +2125,15 @@ dummy_func( } inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) { - if (check_except_star_type_valid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } match = NULL; rest = NULL; - int res = exception_group_match(exc_value, match_type, - &match, &rest); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); DECREF_INPUTS(); ERROR_IF(res < 0, error); @@ -2148,7 +2147,7 @@ dummy_func( inst(CHECK_EXC_MATCH, (left, right -- left, b)) { assert(PyExceptionInstance_Check(left)); - if (check_except_type_valid(tstate, right) < 0) { + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } @@ -2275,7 +2274,7 @@ dummy_func( // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); - attrs = match_class(tstate, subject, type, oparg, names); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); DECREF_INPUTS(); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! @@ -2298,7 +2297,7 @@ dummy_func( inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = match_keys(tstate, subject, keys); + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); ERROR_IF(values_or_none == NULL, error); } @@ -3617,10 +3616,10 @@ dummy_func( STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - res = binary_ops[oparg](lhs, rhs); + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } diff --git a/Python/ceval.c b/Python/ceval.c index b56ddfb4bd2..9f2855f964a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -96,13 +96,6 @@ } while (0) #endif -// GH-89279: Similar to above, force inlining by using a macro. -#if defined(_MSC_VER) && SIZEOF_INT == 4 -#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value))) -#else -#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) -#endif - #ifdef LLTRACE static void @@ -206,13 +199,7 @@ static void monitor_throw(PyThreadState *tstate, static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); -static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *); -static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); -static int check_except_type_valid(PyThreadState *tstate, PyObject* right); -static int check_except_star_type_valid(PyThreadState *tstate, PyObject* right); -static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs); -static void format_awaitable_error(PyThreadState *, PyTypeObject *, int); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, @@ -224,12 +211,6 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -#define UNBOUNDLOCAL_ERROR_MSG \ - "cannot access local variable '%s' where it is not associated with a value" -#define UNBOUNDFREE_ERROR_MSG \ - "cannot access free variable '%s' where it is not associated with a" \ - " value in enclosing scope" - #ifdef HAVE_ERRNO_H #include <errno.h> #endif @@ -286,7 +267,7 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) } -static const binaryfunc binary_ops[] = { +const binaryfunc _PyEval_BinaryOps[] = { [NB_ADD] = PyNumber_Add, [NB_AND] = PyNumber_And, [NB_FLOOR_DIVIDE] = PyNumber_FloorDivide, @@ -321,8 +302,8 @@ static const binaryfunc binary_ops[] = { // Return a tuple of values corresponding to keys, with error checks for // duplicate/missing keys. -static PyObject* -match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys) +PyObject * +_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) { assert(PyTuple_CheckExact(keys)); Py_ssize_t nkeys = PyTuple_GET_SIZE(keys); @@ -403,7 +384,7 @@ fail: // Extract a named attribute from the subject, with additional bookkeeping to // raise TypeErrors for repeated lookups. On failure, return NULL (with no // error set). Use _PyErr_Occurred(tstate) to disambiguate. -static PyObject* +static PyObject * match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type, PyObject *name, PyObject *seen) { @@ -425,9 +406,9 @@ match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type, // On success (match), return a tuple of extracted attributes. On failure (no // match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate. -static PyObject* -match_class(PyThreadState *tstate, PyObject *subject, PyObject *type, - Py_ssize_t nargs, PyObject *kwargs) +PyObject* +_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, + Py_ssize_t nargs, PyObject *kwargs) { if (!PyType_Check(type)) { const char *e = "called match pattern must be a class"; @@ -533,11 +514,6 @@ fail: static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause); -static int exception_group_match( - PyObject* exc_value, PyObject *match_type, - PyObject **match, PyObject **rest); - -static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **); PyObject * PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) @@ -827,7 +803,7 @@ resume_frame: unbound_local_error: { - format_exc_check_arg(tstate, PyExc_UnboundLocalError, + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); @@ -1777,9 +1753,9 @@ raise_error: complicated for inlining). */ -static int -exception_group_match(PyObject* exc_value, PyObject *match_type, - PyObject **match, PyObject **rest) +int +_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, + PyObject **match, PyObject **rest) { if (Py_IsNone(exc_value)) { *match = Py_NewRef(Py_None); @@ -1840,9 +1816,9 @@ exception_group_match(PyObject* exc_value, PyObject *match_type, with a variable target. */ -static int -unpack_iterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +int +_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, + int argcnt, int argcntafter, PyObject **sp) { int i = 0, j = 0; Py_ssize_t ll = 0; @@ -2487,8 +2463,8 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name) #define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\ "is not allowed. Use except instead." -static int -check_except_type_valid(PyThreadState *tstate, PyObject* right) +int +_PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right) { if (PyTuple_Check(right)) { Py_ssize_t i, length; @@ -2512,10 +2488,10 @@ check_except_type_valid(PyThreadState *tstate, PyObject* right) return 0; } -static int -check_except_star_type_valid(PyThreadState *tstate, PyObject* right) +int +_PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right) { - if (check_except_type_valid(tstate, right) < 0) { + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { return -1; } @@ -2569,8 +2545,8 @@ check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args) return 0; } -static void -format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs) +void +_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs) { /* _PyDict_MergeEx raises attribute * error (percolated from an attempt @@ -2611,9 +2587,9 @@ format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs) } } -static void -format_exc_check_arg(PyThreadState *tstate, PyObject *exc, - const char *format_str, PyObject *obj) +void +_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, + const char *format_str, PyObject *obj) { const char *obj_str; @@ -2640,8 +2616,8 @@ format_exc_check_arg(PyThreadState *tstate, PyObject *exc, } } -static void -format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg) +void +_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) { PyObject *name; /* Don't stomp existing exception */ @@ -2649,16 +2625,16 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg) return; name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); if (oparg < PyCode_GetFirstFree(co)) { - format_exc_check_arg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, name); } else { - format_exc_check_arg(tstate, PyExc_NameError, - UNBOUNDFREE_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + UNBOUNDFREE_ERROR_MSG, name); } } -static void -format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int oparg) +void +_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg) { if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { if (oparg == 1) { @@ -2703,110 +2679,3 @@ void Py_LeaveRecursiveCall(void) { _Py_LeaveRecursiveCall(); } - -///////////////////// Experimental UOp Interpreter ///////////////////// - -#undef ASSERT_KWNAMES_IS_NULL -#define ASSERT_KWNAMES_IS_NULL() (void)0 - -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - goto deoptimize; \ - } - -_PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ -#ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } -#define DPRINTF(level, ...) \ - if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } -#else -#define DPRINTF(level, ...) -#endif - - DPRINTF(3, - "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n", - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), - _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->prev_instr + 1 - - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); - - PyThreadState *tstate = _PyThreadState_GET(); - _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; - - CHECK_EVAL_BREAKER(); - - OBJECT_STAT_INC(optimization_traces_executed); - _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - int pc = 0; - int opcode; - int oparg; - uint64_t operand; - - for (;;) { - opcode = self->trace[pc].opcode; - oparg = self->trace[pc].oparg; - operand = self->trace[pc].operand; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", - pc, - opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], - oparg, - operand, - (int)(stack_pointer - _PyFrame_Stackbase(frame))); - pc++; - OBJECT_STAT_INC(optimization_uops_executed); - switch (opcode) { - -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 -#include "executor_cases.c.h" - - default: - { - fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); - Py_FatalError("Unknown uop"); - } - - } - } - -unbound_local_error: - format_exc_check_arg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from cframe.current_frame. - DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return NULL; - -deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - frame->prev_instr--; // Back up to just before destination - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return frame; -} diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c2c323317d1..8dc8b754485 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -1,4 +1,4 @@ -// Macros and other things needed by ceval.c and bytecodes.c +// Macros and other things needed by ceval.c, executor.c, and bytecodes.c /* Computed GOTOs, or the-optimization-commonly-but-improperly-known-as-"threaded code" @@ -304,6 +304,11 @@ GETITEM(PyObject *v, Py_ssize_t i) { (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); +#define UNBOUNDLOCAL_ERROR_MSG \ + "cannot access local variable '%s' where it is not associated with a value" +#define UNBOUNDFREE_ERROR_MSG \ + "cannot access free variable '%s' where it is not associated with a value" \ + " in enclosing scope" #define NAME_ERROR_MSG "name '%.200s' is not defined" #define KWNAMES_LEN() \ @@ -352,3 +357,10 @@ static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { }; #define ASSERT_KWNAMES_IS_NULL() assert(kwnames == NULL) + +// GH-89279: Force inlining by using a macro. +#if defined(_MSC_VER) && SIZEOF_INT == 4 +#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value))) +#else +#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) +#endif diff --git a/Python/executor.c b/Python/executor.c new file mode 100644 index 00000000000..17322526a7b --- /dev/null +++ b/Python/executor.c @@ -0,0 +1,125 @@ +#include "Python.h" + +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" +#include "pycore_intrinsics.h" +#include "pycore_long.h" +#include "pycore_object.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_pyerrors.h" +#include "pycore_range.h" +#include "pycore_sliceobject.h" +#include "pycore_uops.h" + +#include "ceval_macros.h" + + +#undef ASSERT_KWNAMES_IS_NULL +#define ASSERT_KWNAMES_IS_NULL() (void)0 + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + if ((COND)) { \ + goto deoptimize; \ + } + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 + + +_PyInterpreterFrame * +_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) +{ +#ifdef Py_DEBUG + char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + #define DPRINTF(level, ...) \ + if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + DPRINTF(3, + "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n", + PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), + PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), + _PyFrame_GetCode(frame)->co_firstlineno, + 2 * (long)(frame->prev_instr + 1 - + (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); + + PyThreadState *tstate = _PyThreadState_GET(); + _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; + + CHECK_EVAL_BREAKER(); + + OBJECT_STAT_INC(optimization_traces_executed); + _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + int pc = 0; + int opcode; + int oparg; + uint64_t operand; + + for (;;) { + opcode = self->trace[pc].opcode; + oparg = self->trace[pc].oparg; + operand = self->trace[pc].operand; + DPRINTF(3, + "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", + pc, + opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], + oparg, + operand, + (int)(stack_pointer - _PyFrame_Stackbase(frame))); + pc++; + OBJECT_STAT_INC(optimization_uops_executed); + switch (opcode) { + +#include "executor_cases.c.h" + + default: + { + fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); + Py_FatalError("Unknown uop"); + } + + } + } + +unbound_local_error: + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + goto error; + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + // On ERROR_IF we return NULL as the frame. + // The caller recovers the frame from cframe.current_frame. + DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(self); + return NULL; + +deoptimize: + // On DEOPT_IF we just repeat the last instruction. + // This presumes nothing was popped from the stack (nor pushed). + DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + frame->prev_instr--; // Back up to just before destination + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(self); + return frame; +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5b38c6a5fd8..064965a1a1a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -672,7 +672,7 @@ iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { - format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } Py_DECREF(iterable); @@ -758,9 +758,9 @@ err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); goto error; } break; @@ -780,7 +780,7 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; - int res = unpack_iterable(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; STACK_SHRINK(1); @@ -839,7 +839,7 @@ PyObject *seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; - int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; STACK_GROW((oparg & 0xFF) + (oparg >> 8)); @@ -897,8 +897,8 @@ // Can't use ERROR_IF here. if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } goto error; } @@ -941,7 +941,7 @@ goto error; } if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); goto error; @@ -978,8 +978,8 @@ if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -994,7 +994,7 @@ /* namespace 2: builtins */ if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); if (true) goto error; @@ -1080,7 +1080,7 @@ // Can't use ERROR_IF here. // Fortunately we don't need its superpower. if (oldobj == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } PyCell_SET(cell, NULL); @@ -1104,7 +1104,7 @@ PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } Py_INCREF(value); @@ -1118,7 +1118,7 @@ PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) goto error; } Py_INCREF(value); @@ -1348,7 +1348,7 @@ PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -1646,7 +1646,7 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - if (check_except_star_type_valid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); if (true) goto pop_2_error; @@ -1654,8 +1654,8 @@ match = NULL; rest = NULL; - int res = exception_group_match(exc_value, match_type, - &match, &rest); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; @@ -1676,7 +1676,7 @@ PyObject *left = stack_pointer[-2]; PyObject *b; assert(PyExceptionInstance_Check(left)); - if (check_except_type_valid(tstate, right) < 0) { + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); if (true) goto pop_1_error; } @@ -1723,7 +1723,7 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); - attrs = match_class(tstate, subject, type, oparg, names); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -1764,7 +1764,7 @@ PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = match_keys(tstate, subject, keys); + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; STACK_GROW(1); stack_pointer[-1] = values_or_none; @@ -2462,10 +2462,10 @@ STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - res = binary_ops[oparg](lhs, rhs); + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2d70966aa09..9d25fc60431 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1068,7 +1068,7 @@ iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { - format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } Py_DECREF(iterable); @@ -1336,9 +1336,9 @@ err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); goto error; } DISPATCH(); @@ -1359,7 +1359,7 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; - int res = unpack_iterable(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; STACK_SHRINK(1); @@ -1422,7 +1422,7 @@ PyObject *seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; - int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; STACK_GROW((oparg & 0xFF) + (oparg >> 8)); @@ -1482,8 +1482,8 @@ // Can't use ERROR_IF here. if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } goto error; } @@ -1543,7 +1543,7 @@ goto error; } if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); goto error; @@ -1581,7 +1581,7 @@ goto error; } if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); goto error; @@ -1621,8 +1621,8 @@ if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -1637,7 +1637,7 @@ /* namespace 2: builtins */ if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; if (v == NULL) { - format_exc_check_arg( + _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); if (true) goto error; @@ -1755,7 +1755,7 @@ // Can't use ERROR_IF here. // Fortunately we don't need its superpower. if (oldobj == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } PyCell_SET(cell, NULL); @@ -1779,7 +1779,7 @@ PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } Py_INCREF(value); @@ -1793,7 +1793,7 @@ PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { - format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) goto error; } Py_INCREF(value); @@ -2023,7 +2023,7 @@ PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -2679,7 +2679,7 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - if (check_except_star_type_valid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); if (true) goto pop_2_error; @@ -2687,8 +2687,8 @@ match = NULL; rest = NULL; - int res = exception_group_match(exc_value, match_type, - &match, &rest); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; @@ -2709,7 +2709,7 @@ PyObject *left = stack_pointer[-2]; PyObject *b; assert(PyExceptionInstance_Check(left)); - if (check_except_type_valid(tstate, right) < 0) { + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); if (true) goto pop_1_error; } @@ -2890,7 +2890,7 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); - attrs = match_class(tstate, subject, type, oparg, names); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -2931,7 +2931,7 @@ PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = match_keys(tstate, subject, keys); + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; STACK_GROW(1); stack_pointer[-1] = values_or_none; @@ -4464,10 +4464,10 @@ STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - res = binary_ops[oparg](lhs, rhs); + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error; |