diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 115 | ||||
| -rw-r--r-- | Python/ceval.c | 40 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 5 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 8 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 222 | ||||
| -rw-r--r-- | Python/opcode_targets.h | 6 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 8 |
7 files changed, 233 insertions, 171 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index eb8b66f8d8b..971397c955d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -945,48 +945,25 @@ dummy_func( LLTRACE_RESUME_FRAME(); } - inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + tier1 op(_RETURN_VALUE_EVENT, (val -- val)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); if (err) ERROR_NO_POP(); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; } + macro(INSTRUMENTED_RETURN_VALUE) = + _RETURN_VALUE_EVENT + + RETURN_VALUE; + macro(RETURN_CONST) = LOAD_CONST + RETURN_VALUE; - inst(INSTRUMENTED_RETURN_CONST, (--)) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) ERROR_NO_POP(); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); - LOAD_IP(frame->return_offset); - goto resume_frame; - } + macro(INSTRUMENTED_RETURN_CONST) = + LOAD_CONST + + _RETURN_VALUE_EVENT + + RETURN_VALUE; inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; @@ -1183,31 +1160,6 @@ dummy_func( _SEND_GEN_FRAME + _PUSH_FRAME; - inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) ERROR_NO_POP(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -1244,6 +1196,23 @@ dummy_func( LLTRACE_RESUME_FRAME(); } + tier1 op(_YIELD_VALUE_EVENT, (val -- val)) { + SAVE_SP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + LOAD_SP(); + if (err) ERROR_NO_POP(); + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + + macro(INSTRUMENTED_YIELD_VALUE) = + _YIELD_VALUE_EVENT + + YIELD_VALUE; + inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, @@ -4450,6 +4419,36 @@ dummy_func( assert(oparg >= 2); } + inst(INSTRUMENTED_LINE, ( -- )) { + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); diff --git a/Python/ceval.c b/Python/ceval.c index 1e911d3ba17..c0074c45b27 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -835,46 +835,6 @@ resume_frame: #include "generated_cases.c.h" - /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->instr_ptr before it is updated, - * as happens in the standard instruction prologue. - */ -#if USE_COMPUTED_GOTOS - TARGET_INSTRUMENTED_LINE: -#else - case INSTRUMENTED_LINE: -#endif - { - _Py_CODEUNIT *prev = frame->instr_ptr; - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - #if USE_COMPUTED_GOTOS _unknown_opcode: diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 595b72bfaf9..60efe3d78ff 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -402,7 +402,10 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3379f0be227..288e0f9135c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1153,10 +1153,6 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ - case _GET_AITER: { _PyStackRef obj; _PyStackRef iter; @@ -1304,8 +1300,6 @@ break; } - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ - case _YIELD_VALUE: { _PyStackRef retval; _PyStackRef value; @@ -4913,6 +4907,8 @@ break; } + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 because it is instrumented */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c9907438ddc..634053a9383 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3657,6 +3657,41 @@ DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + _Py_CODEUNIT *prev_instr = frame->instr_ptr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3785,23 +3820,44 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); - LOAD_IP(frame->return_offset); - goto resume_frame; + _PyStackRef value; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _LOAD_CONST + { + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + } + // _RETURN_VALUE_EVENT + val = value; + { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + if (err) goto error; + } + // _RETURN_VALUE + retval = val; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -3809,24 +3865,41 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; _PyStackRef retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) goto error; - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + _PyStackRef res; + // _RETURN_VALUE_EVENT + val = stack_pointer[-1]; + { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + if (err) goto error; + } + // _RETURN_VALUE + retval = val; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_YIELD_VALUE) { @@ -3834,30 +3907,65 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + _PyStackRef value; + // _YIELD_VALUE_EVENT + val = stack_pointer[-1]; + { + SAVE_SP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + LOAD_SP(); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + retval = val; + { + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INTERPRETER_EXIT) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 74544a1dff2..224aeb83493 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -238,9 +238,6 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, - &&TARGET_INSTRUMENTED_RETURN_VALUE, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_CALL, @@ -253,6 +250,9 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_RETURN_CONST, + &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, }; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 33af8552ba6..3c9e6d3043c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -621,10 +621,6 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ - case _GET_AITER: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); @@ -656,8 +652,6 @@ break; } - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ - case _YIELD_VALUE: { _Py_UopsSymbol *res; res = sym_new_unknown(ctx); @@ -2056,6 +2050,8 @@ break; } + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ |