From 7797182b78baf78f64fe16f436aa2279cf6afc23 Mon Sep 17 00:00:00 2001
From: Brandt Bucher <brandtbucher@microsoft.com>
Date: Mon, 29 Jul 2024 14:49:17 -0700
Subject: GH-118093: Improve handling of short and mid-loop traces (GH-122252)
---
Python/optimizer.c | 64 ++++++++++++++++++++++++++----------------------------
1 file changed, 31 insertions(+), 33 deletions(-)
(limited to 'Python/optimizer.c')
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 7b875af2aae..ce8a36575cd 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -503,8 +503,7 @@ add_to_trace(
if (trace_stack_depth >= TRACE_STACK_SIZE) { \
DPRINTF(2, "Trace stack overflow\n"); \
OPT_STAT_INC(trace_stack_overflow); \
- trace_length = 0; \
- goto done; \
+ return 0; \
} \
assert(func == NULL || func->func_code == (PyObject *)code); \
trace_stack[trace_stack_depth].func = func; \
@@ -550,6 +549,7 @@ translate_bytecode_to_trace(
} trace_stack[TRACE_STACK_SIZE];
int trace_stack_depth = 0;
int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions
+ bool jump_seen = false;
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
@@ -568,7 +568,6 @@ translate_bytecode_to_trace(
ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code));
uint32_t target = 0;
-top: // Jump here after _PUSH_FRAME or likely branches
for (;;) {
target = INSTR_IP(instr, code);
// Need space for _DEOPT
@@ -577,6 +576,13 @@ top: // Jump here after _PUSH_FRAME or likely branches
uint32_t opcode = instr->op.code;
uint32_t oparg = instr->op.arg;
+ if (!progress_needed && instr == initial_instr) {
+ // We have looped around to the start:
+ RESERVE(1);
+ ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);
+ goto done;
+ }
+
DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg);
if (opcode == ENTER_EXECUTOR) {
@@ -603,30 +609,21 @@ top: // Jump here after _PUSH_FRAME or likely branches
/* Special case the first instruction,
* so that we can guarantee forward progress */
if (progress_needed) {
- progress_needed = false;
- if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) {
- instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg;
- initial_instr = instr;
- if (opcode == JUMP_BACKWARD) {
- ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
- }
- continue;
- }
- else {
- if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
- opcode = _PyOpcode_Deopt[opcode];
- }
- assert(!OPCODE_HAS_EXIT(opcode));
- assert(!OPCODE_HAS_DEOPT(opcode));
+ if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
+ opcode = _PyOpcode_Deopt[opcode];
}
+ assert(!OPCODE_HAS_EXIT(opcode));
+ assert(!OPCODE_HAS_DEOPT(opcode));
}
if (OPCODE_HAS_EXIT(opcode)) {
- // Make space for exit code
+ // Make space for side exit and final _EXIT_TRACE:
+ RESERVE_RAW(2, "_EXIT_TRACE");
max_length--;
}
if (OPCODE_HAS_ERROR(opcode)) {
- // Make space for error code
+ // Make space for error stub and final _EXIT_TRACE:
+ RESERVE_RAW(2, "_ERROR_POP_N");
max_length--;
}
switch (opcode) {
@@ -672,19 +669,18 @@ top: // Jump here after _PUSH_FRAME or likely branches
}
case JUMP_BACKWARD:
+ ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target);
+ _Py_FALLTHROUGH;
case JUMP_BACKWARD_NO_INTERRUPT:
{
- _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[opcode] - (int)oparg;
- if (target == initial_instr) {
- /* We have looped round to the start */
- RESERVE(1);
- ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);
- }
- else {
+ instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg;
+ if (jump_seen) {
OPT_STAT_INC(inner_loop);
DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n");
+ goto done;
}
- goto done;
+ jump_seen = true;
+ goto top;
}
case JUMP_FORWARD:
@@ -904,6 +900,9 @@ top: // Jump here after _PUSH_FRAME or likely branches
assert(instr->op.code == POP_TOP);
instr++;
}
+ top:
+ // Jump here after _PUSH_FRAME or likely branches.
+ progress_needed = false;
} // End for (;;)
done:
@@ -911,16 +910,15 @@ done:
TRACE_STACK_POP();
}
assert(code == initial_code);
- // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE
- if (progress_needed || trace_length < 5) {
+ // Skip short traces where we can't even translate a single instruction:
+ if (progress_needed) {
OPT_STAT_INC(trace_too_short);
DPRINTF(2,
- "No trace for %s (%s:%d) at byte offset %d (%s)\n",
+ "No trace for %s (%s:%d) at byte offset %d (no progress)\n",
PyUnicode_AsUTF8(code->co_qualname),
PyUnicode_AsUTF8(code->co_filename),
code->co_firstlineno,
- 2 * INSTR_IP(initial_instr, code),
- progress_needed ? "no progress" : "too short");
+ 2 * INSTR_IP(initial_instr, code));
return 0;
}
if (trace[trace_length-1].opcode != _JUMP_TO_TOP) {
--
cgit v1.2.3