diff options
Diffstat (limited to 'Python/optimizer_bytecodes.c')
| -rw-r--r-- | Python/optimizer_bytecodes.c | 608 |
1 files changed, 386 insertions, 222 deletions
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 567caad2255..3182e8b3b70 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -27,13 +27,16 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop #define sym_new_tuple _Py_uop_sym_new_tuple #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_is_immortal _Py_uop_symbol_is_immortal +#define sym_new_compact_int _Py_uop_sym_new_compact_int +#define sym_is_compact_int _Py_uop_sym_is_compact_int #define sym_new_truthiness _Py_uop_sym_new_truthiness extern int @@ -87,12 +90,12 @@ dummy_func(void) { } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -104,18 +107,40 @@ dummy_func(void) { res = sym_new_null(ctx); } - op(_GUARD_TOS_INT, (tos -- tos)) { - if (sym_matches_type(tos, &PyLong_Type)) { + op(_GUARD_TOS_INT, (value -- value)) { + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } } - op(_GUARD_NOS_INT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyLong_Type)) { + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } + } + + op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { + PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); + if (type) { + if (type == sym_get_const(ctx, owner)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_const(owner, type); + } + } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { @@ -141,25 +166,26 @@ dummy_func(void) { } } - op(_GUARD_TOS_FLOAT, (tos -- tos)) { - if (sym_matches_type(tos, &PyFloat_Type)) { + op(_GUARD_TOS_FLOAT, (value -- value)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); } - op(_GUARD_NOS_FLOAT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyFloat_Type)) { + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); } - op(_BINARY_OP, (left, right -- res)) { - bool lhs_int = sym_matches_type(left, &PyLong_Type); - bool rhs_int = sym_matches_type(right, &PyLong_Type); - bool lhs_float = sym_matches_type(left, &PyFloat_Type); - bool rhs_float = sym_matches_type(right, &PyFloat_Type); + op(_BINARY_OP, (lhs, rhs -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(lhs, rhs); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: res = sym_new_unknown(ctx); @@ -185,11 +211,11 @@ dummy_func(void) { // Case C: res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, right)) { + else if (!sym_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { // Case B: res = sym_new_type(ctx, &PyFloat_Type); } @@ -210,140 +236,54 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -361,12 +301,12 @@ dummy_func(void) { GETLOCAL(this_instr->operand0) = res; } - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { - new_frame = NULL; + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { + new_frame = PyJitRef_NULL; ctx->done = true; } - op(_BINARY_OP_SUBSCR_STR_INT, (left, right -- res)) { + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } @@ -398,11 +338,11 @@ dummy_func(void) { } } - op(_TO_BOOL_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res); + op(_TO_BOOL_BOOL, (value -- value)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, true); + value = sym_new_truthiness(ctx, value, true); } } @@ -451,10 +391,35 @@ dummy_func(void) { } op(_UNARY_NOT, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); } + op(_UNARY_NEGATIVE, (value -- res)) { + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + PyTypeObject *type = sym_get_type(value); + if (type == &PyLong_Type || type == &PyFloat_Type) { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } + } + + op(_UNARY_INVERT, (value -- res)) { + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_not_null(ctx); + } + } + op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); @@ -493,45 +458,34 @@ dummy_func(void) { res = sym_new_type(ctx, &PyBool_Type); } - op(_IS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_IS_OP, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_SET, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_SET, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_DICT, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_DICT, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); - int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } - - op(_LOAD_CONST_MORTAL, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); - int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } - - op(_LOAD_CONST_IMMORTAL, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { - PyObject *val = PyLong_FromLong(this_instr->oparg); - value = sym_new_const(ctx, val); + PyObject *val = PyLong_FromLong(oparg); + assert(val); + assert(_Py_IsImmortal(val)); + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { @@ -539,7 +493,7 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { @@ -547,7 +501,37 @@ dummy_func(void) { } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_TOP, (value -- )) { + PyTypeObject *typ = sym_get_type(value); + if (PyJitRef_IsBorrowed(value) || + sym_is_immortal(PyJitRef_Unwrap(value)) || + sym_is_null(value)) { + REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + } + else if (typ == &PyLong_Type) { + REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + } + else if (typ == &PyFloat_Type) { + REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + } + else if (typ == &PyUnicode_Type) { + REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + } } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -556,7 +540,7 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -570,7 +554,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -580,11 +564,17 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } + } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -603,7 +593,7 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; attr = sym_new_not_null(ctx); - if (oparg &1) { + if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } } @@ -619,31 +609,65 @@ dummy_func(void) { } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { - attr = sym_new_not_null(ctx); (void)descr; + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -661,6 +685,16 @@ dummy_func(void) { sym_set_type(callable, &PyFunction_Type); } + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); + } + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { @@ -679,7 +713,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; PyCodeObject *co = NULL; @@ -691,7 +725,7 @@ dummy_func(void) { } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -700,20 +734,19 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); - + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -722,28 +755,30 @@ dummy_func(void) { break; } - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { - new_frame = NULL; + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + new_frame = PyJitRef_NULL; ctx->done = true; } - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { - init_frame = NULL; + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { + init_frame = PyJitRef_NULL; ctx->done = true; } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + // We wrap and unwrap the value to mimic PyStackRef_MakeHeapSafe + // in bytecodes.c + JitOptRef temp = PyJitRef_Wrap(PyJitRef_Unwrap(retval)); DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; @@ -789,21 +824,34 @@ dummy_func(void) { } } - op(_YIELD_VALUE, (unused -- res)) { - res = sym_new_unknown(ctx); + op(_YIELD_VALUE, (unused -- value)) { + value = sym_new_unknown(ctx); + } + + op(_GET_ITER, (iterable -- iter, index_or_null)) { + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); + } + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); + } } - op(_FOR_ITER_GEN_FRAME, ( -- )) { + op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } - op(_SEND_GEN_FRAME, ( -- )) { + op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } - op(_CHECK_STACK_SPACE, ( --)) { + op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; } @@ -815,12 +863,12 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) { + op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -848,14 +896,16 @@ dummy_func(void) { corresponding_check_stack = NULL; } - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { + (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } } - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { + (void)top; /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { @@ -863,13 +913,23 @@ dummy_func(void) { } } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); + } + + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { next = sym_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -886,6 +946,26 @@ dummy_func(void) { } } + op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { + // the result is always a bool, but sometimes we can + // narrow it down to True or False + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + // isinstance(inst, cls) where both inst and cls have + // known types, meaning we can deduce either True or False + + // The below check is equivalent to PyObject_TypeCheck(inst, cls) + PyObject *out = Py_False; + if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { + out = Py_True; + } + sym_set_const(res, out); + REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + } + } + op(_GUARD_IS_TRUE_POP, (flag -- )) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -904,27 +984,27 @@ dummy_func(void) { sym_set_const(flag, Py_False); } - op(_GUARD_IS_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(flag, Py_None); + sym_set_const(val, Py_None); } - op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NOT_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -969,7 +1049,7 @@ dummy_func(void) { list = sym_new_type(ctx, &PyList_Type); } - op(_BUILD_SLICE, (values[oparg] -- slice)) { + op(_BUILD_SLICE, (args[oparg] -- slice)) { slice = sym_new_type(ctx, &PySlice_Type); } @@ -977,7 +1057,7 @@ dummy_func(void) { map = sym_new_type(ctx, &PyDict_Type); } - op(_BUILD_STRING, (values[oparg] -- str)) { + op(_BUILD_STRING, (pieces[oparg] -- str)) { str = sym_new_type(ctx, &PyUnicode_Type); } @@ -1063,6 +1143,20 @@ dummy_func(void) { sym_set_null(null); } + op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { + if (sym_is_not_null(nos)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_non_null(nos); + } + + op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { + if (sym_is_null(null)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_null(null); + } + op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1084,8 +1178,78 @@ dummy_func(void) { sym_set_const(callable, (PyObject *)&PyUnicode_Type); } - op(_CALL_LEN, (callable[1], self_or_null[1], args[oparg] -- res)) { + op(_CALL_LEN, (callable, null, arg -- res)) { res = sym_new_type(ctx, &PyLong_Type); + int tuple_length = sym_tuple_length(arg); + if (tuple_length >= 0) { + PyObject *temp = PyLong_FromLong(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + } + + op(_GET_LEN, (obj -- obj, len)) { + int tuple_length = sym_tuple_length(obj); + if (tuple_length == -1) { + len = sym_new_type(ctx, &PyLong_Type); + } + else { + assert(tuple_length >= 0); + PyObject *temp = PyLong_FromLong(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + } + len = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + } + + op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { + PyObject *len = _PyInterpreterState_GET()->callable_cache.len; + if (sym_get_const(ctx, callable) == len) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, len); + } + + op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { + PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; + if (sym_get_const(ctx, callable) == isinstance) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, isinstance); + } + + op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { + PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; + if (sym_get_const(ctx, callable) == list_append) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, list_append); + } + + op(_BINARY_SLICE, (container, start, stop -- res)) { + // Slicing a string/list/tuple always returns the same type. + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } } // END BYTECODES // |