diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 86 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 95 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 633 | ||||
| -rw-r--r-- | Python/optimizer_bytecodes.c | 61 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 94 |
5 files changed, 504 insertions, 465 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 83847e31ef4..c50df896cc9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -524,6 +524,12 @@ dummy_func( res = PyStackRef_False; } + op(_GUARD_NOS_UNICODE, (nos, tos -- nos, tos)) { + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyUnicode_CheckExact(o)); + } + op(_GUARD_TOS_UNICODE, (value -- value)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); EXIT_IF(!PyUnicode_CheckExact(value_o)); @@ -581,14 +587,8 @@ dummy_func( BINARY_OP_EXTEND, }; - op(_GUARD_BOTH_INT, (left, right -- left, right)) { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - EXIT_IF(!PyLong_CheckExact(left_o)); - EXIT_IF(!PyLong_CheckExact(right_o)); - } - - op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + op(_GUARD_NOS_INT, (left, right -- left, right)) { + (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); EXIT_IF(!PyLong_CheckExact(left_o)); } @@ -644,20 +644,14 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_INT) = - _GUARD_BOTH_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; macro(BINARY_OP_ADD_INT) = - _GUARD_BOTH_INT + unused/5 + _BINARY_OP_ADD_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_BOTH_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; - op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - EXIT_IF(!PyFloat_CheckExact(left_o)); - EXIT_IF(!PyFloat_CheckExact(right_o)); - } - - op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + op(_GUARD_NOS_FLOAT, (left, right -- left, right)) { + (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); EXIT_IF(!PyFloat_CheckExact(left_o)); } @@ -713,19 +707,11 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_FLOAT) = - _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = - _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT; + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT; macro(BINARY_OP_SUBTRACT_FLOAT) = - _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT; - - op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - - EXIT_IF(!PyUnicode_CheckExact(left_o)); - EXIT_IF(!PyUnicode_CheckExact(right_o)); - } + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT; pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -743,7 +729,7 @@ dummy_func( } macro(BINARY_OP_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE; + _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE; // This is a subtle one. It's a super-instruction for // BINARY_OP_ADD_UNICODE followed by STORE_FAST @@ -822,7 +808,7 @@ dummy_func( unused/1 + _GUARD_BINARY_OP_EXTEND + rewind/-4 + _BINARY_OP_EXTEND; macro(BINARY_OP_INPLACE_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE; + _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE; specializing op(_SPECIALIZE_BINARY_SLICE, (container, start, stop -- container, start, stop)) { // Placeholder until we implement BINARY_SLICE specialization @@ -875,11 +861,14 @@ dummy_func( macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE; - inst(BINARY_OP_SUBSCR_LIST_INT, (unused/5, list_st, sub_st -- res)) { + macro(BINARY_OP_SUBSCR_LIST_INT) = + _GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_LIST_INT; + + op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub)); + assert(PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) @@ -901,12 +890,15 @@ dummy_func( DECREF_INPUTS(); } - inst(BINARY_OP_SUBSCR_STR_INT, (unused/5, str_st, sub_st -- res)) { + macro(BINARY_OP_SUBSCR_STR_INT) = + _GUARD_TOS_INT + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_SUBSCR_STR_INT; + + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub)); - DEOPT_IF(!PyUnicode_CheckExact(str)); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); @@ -921,11 +913,14 @@ dummy_func( res = PyStackRef_FromPyObjectImmortal(res_o); } - inst(BINARY_OP_SUBSCR_TUPLE_INT, (unused/5, tuple_st, sub_st -- res)) { + macro(BINARY_OP_SUBSCR_TUPLE_INT) = + _GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT; + + op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub)); + assert(PyLong_CheckExact(sub)); DEOPT_IF(!PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) @@ -1025,11 +1020,14 @@ dummy_func( macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; - inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) { + macro(STORE_SUBSCR_LIST_INT) = + _GUARD_TOS_INT + unused/1 + _STORE_SUBSCR_LIST_INT; + + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub)); + assert(PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. @@ -2584,13 +2582,13 @@ dummy_func( macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; macro(COMPARE_OP_FLOAT) = - _GUARD_BOTH_FLOAT + unused/1 + _COMPARE_OP_FLOAT; + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT; macro(COMPARE_OP_INT) = - _GUARD_BOTH_INT + unused/1 + _COMPARE_OP_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT; macro(COMPARE_OP_STR) = - _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; + _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR; op(_COMPARE_OP_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 260d52be02c..117c80c8d78 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -760,6 +760,20 @@ break; } + case _GUARD_NOS_UNICODE: { + _PyStackRef tos; + _PyStackRef nos; + tos = stack_pointer[-1]; + nos = stack_pointer[-2]; + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _GUARD_TOS_UNICODE: { _PyStackRef value; value = stack_pointer[-1]; @@ -834,27 +848,12 @@ break; } - case _GUARD_BOTH_INT: { + case _GUARD_NOS_INT: { _PyStackRef right; _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyLong_CheckExact(right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - break; - } - - case _GUARD_NOS_INT: { - _PyStackRef left; - left = stack_pointer[-2]; + (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); @@ -958,27 +957,12 @@ break; } - case _GUARD_BOTH_FLOAT: { + case _GUARD_NOS_FLOAT: { _PyStackRef right; _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyFloat_CheckExact(right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - break; - } - - case _GUARD_NOS_FLOAT: { - _PyStackRef left; - left = stack_pointer[-2]; + (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); @@ -1079,24 +1063,6 @@ break; } - case _GUARD_BOTH_UNICODE: { - _PyStackRef right; - _PyStackRef left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_CheckExact(right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - break; - } - case _BINARY_OP_ADD_UNICODE: { _PyStackRef right; _PyStackRef left; @@ -1332,10 +1298,7 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyLong_CheckExact(sub)); if (!PyList_CheckExact(list)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1390,14 +1353,8 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - if (!PyLong_CheckExact(sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_CheckExact(str)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1436,10 +1393,7 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - if (!PyLong_CheckExact(sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyLong_CheckExact(sub)); if (!PyTuple_CheckExact(tuple)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1646,10 +1600,7 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyLong_CheckExact(sub)); if (!PyList_CheckExact(list)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a4b5673c9e5..41ea054d3f5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -92,21 +92,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_FLOAT + // _GUARD_TOS_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyFloat_CheckExact(right_o)) { + } + // _GUARD_NOS_FLOAT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -145,21 +151,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_INT + // _GUARD_TOS_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyLong_CheckExact(right_o)) { + } + // _GUARD_NOS_INT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -200,21 +212,29 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_UNICODE + // _GUARD_TOS_UNICODE { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyUnicode_CheckExact(right_o)) { + } + // _GUARD_NOS_UNICODE + { + tos = value; + nos = stack_pointer[-2]; + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -223,6 +243,8 @@ /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { + right = tos; + left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyUnicode_CheckExact(left_o)); @@ -317,20 +339,28 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tos; _PyStackRef left; _PyStackRef right; - // _GUARD_BOTH_UNICODE + // _GUARD_TOS_UNICODE { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyUnicode_CheckExact(right_o)) { + } + // _GUARD_NOS_UNICODE + { + tos = value; + nos = stack_pointer[-2]; + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -339,6 +369,8 @@ /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { + right = tos; + left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); @@ -405,21 +437,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_FLOAT + // _GUARD_TOS_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyFloat_CheckExact(right_o)) { + } + // _GUARD_NOS_FLOAT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -458,21 +496,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_INT + // _GUARD_TOS_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyLong_CheckExact(right_o)) { + } + // _GUARD_NOS_INT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -655,66 +699,76 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 5 cache entries */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + // Deopt unless 0 <= sub < PyList_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -729,51 +783,71 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tos; _PyStackRef str_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 5 cache entries */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyUnicode_CheckExact(str)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + // _GUARD_NOS_UNICODE + { + tos = value; + nos = stack_pointer[-2]; + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = tos; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectImmortal(res_o); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectImmortal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -791,49 +865,59 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef tuple_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 5 cache entries */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyTuple_CheckExact(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + // Deopt unless 0 <= sub < PyTuple_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -848,21 +932,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_FLOAT + // _GUARD_TOS_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyFloat_CheckExact(right_o)) { + } + // _GUARD_NOS_FLOAT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -901,21 +991,27 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_INT + // _GUARD_TOS_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - if (!PyLong_CheckExact(right_o)) { + } + // _GUARD_NOS_INT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -4513,21 +4609,27 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_FLOAT + // _GUARD_TOS_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } - if (!PyFloat_CheckExact(right_o)) { + } + // _GUARD_NOS_FLOAT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4565,21 +4667,27 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_INT + // _GUARD_TOS_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } - if (!PyLong_CheckExact(right_o)) { + } + // _GUARD_NOS_INT + { + right = value; + left = stack_pointer[-2]; + (void)right; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4629,21 +4737,29 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_UNICODE + // _GUARD_TOS_UNICODE { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } - if (!PyUnicode_CheckExact(right_o)) { + } + // _GUARD_NOS_UNICODE + { + tos = value; + nos = stack_pointer[-2]; + (void)tos; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4652,6 +4768,8 @@ /* Skip 1 cache entry */ // _COMPARE_OP_STR { + right = tos; + left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); @@ -11346,56 +11464,65 @@ _PyStackRef value; _PyStackRef list_st; _PyStackRef sub_st; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + // Ensure nonnegative, zero-or-one-digit ints. + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 163a8bfd267..a948cccbf85 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,22 +104,18 @@ dummy_func(void) { res = sym_new_null(ctx); } - op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyLong_Type)) { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); - } + op(_GUARD_TOS_INT, (tos -- tos)) { + if (sym_matches_type(tos, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); } - else { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); - } + sym_set_type(tos, &PyLong_Type); + } + + op(_GUARD_NOS_INT, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + sym_set_type(nos, &PyLong_Type); } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { @@ -145,32 +141,18 @@ dummy_func(void) { } } - op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyFloat_Type)) { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); - } - } - else { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); - } + op(_GUARD_TOS_FLOAT, (tos -- tos)) { + if (sym_matches_type(tos, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); } - - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + sym_set_type(tos, &PyFloat_Type); } - op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - if (sym_matches_type(left, &PyUnicode_Type) && - sym_matches_type(right, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + op(_GUARD_NOS_FLOAT, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); + sym_set_type(nos, &PyFloat_Type); } op(_BINARY_OP, (left, right -- res)) { @@ -418,6 +400,13 @@ dummy_func(void) { } } + op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyUnicode_Type); + } + op(_GUARD_TOS_UNICODE, (value -- value)) { if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f1280ba1e40..6fdc81f1e5a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -217,6 +217,16 @@ break; } + case _GUARD_NOS_UNICODE: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyUnicode_Type); + break; + } + case _GUARD_TOS_UNICODE: { JitOptSymbol *value; value = stack_pointer[-1]; @@ -252,34 +262,23 @@ break; } - case _GUARD_BOTH_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); - } - } - else { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); - } - } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); - break; - } - case _GUARD_NOS_INT: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyLong_Type); break; } case _GUARD_TOS_INT: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyLong_Type); break; } @@ -376,34 +375,23 @@ break; } - case _GUARD_BOTH_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type)) { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); - } - } - else { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); - } - } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); - break; - } - case _GUARD_NOS_FLOAT: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyFloat_Type); break; } @@ -503,20 +491,6 @@ break; } - case _GUARD_BOTH_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyUnicode_Type) && - sym_matches_type(right, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); - break; - } - case _BINARY_OP_ADD_UNICODE: { JitOptSymbol *right; JitOptSymbol *left; |