diff options
Diffstat (limited to 'Python/generated_cases.c.h')
| -rw-r--r-- | Python/generated_cases.c.h | 583 |
1 files changed, 265 insertions, 318 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 914b06987ed..f02e13f5e3f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -50,7 +50,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION_FT */ assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); + assert(oparg <= NB_OPARG_LAST); } /* Skip 4 cache entries */ // _BINARY_OP @@ -480,251 +480,31 @@ DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_FLOAT) { + TARGET(BINARY_OP_SUBSCR_DICT) { #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_OP_SUBTRACT_FLOAT; + int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_INT) { - #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_OP_SUBTRACT_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(BINARY_SLICE) { - #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SLICE; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(BINARY_SUBSCR) { - #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED_BINARY_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_DICT) { - #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); _PyStackRef dict_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ sub_st = stack_pointer[-1]; dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); int rc = PyDict_GetItemRef(dict, sub, &res_o); @@ -747,70 +527,70 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR_GETITEM) { + TARGET(BINARY_OP_SUBSCR_GETITEM) { #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR_GETITEM; + int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; _PyStackRef sub; _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _CHECK_PEP_523 { if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } } - // _BINARY_SUBSCR_CHECK_FUNC + // _BINARY_OP_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); } - // _BINARY_SUBSCR_INIT_CALL + // _BINARY_OP_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; + frame->return_offset = 6 ; } // _PUSH_FRAME { @@ -832,40 +612,40 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR_LIST_INT) { + TARGET(BINARY_OP_SUBSCR_LIST_INT) { #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR_LIST_INT; + int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ sub_st = stack_pointer[-1]; list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } // Deopt unless 0 <= sub < PyList_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED @@ -873,18 +653,18 @@ PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); #else if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); @@ -902,54 +682,54 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR_STR_INT) { + TARGET(BINARY_OP_SUBSCR_STR_INT) { #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR_STR_INT; + int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef str_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ sub_st = stack_pointer[-1]; str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyUnicode_CheckExact(str)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; @@ -964,48 +744,48 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR_TUPLE_INT) { + TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { #if defined(Py_TAIL_CALL_INTERP) - int opcode = BINARY_SUBSCR_TUPLE_INT; + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tuple_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyTuple_CheckExact(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } // Deopt unless 0 <= sub < PyTuple_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_SUBSCR, hit); + STAT_INC(BINARY_OP, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); @@ -1022,6 +802,173 @@ DISPATCH(); } + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + TARGET(BUILD_LIST) { #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_LIST; |