diff options
Diffstat (limited to 'Python/optimizer_cases.c.h')
| -rw-r--r-- | Python/optimizer_cases.c.h | 744 |
1 files changed, 391 insertions, 353 deletions
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index adab110c5ce..91927180b35 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -26,7 +26,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); if (sym_is_null(value)) { ctx->done = true; @@ -38,7 +38,7 @@ } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -47,8 +47,8 @@ } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -56,9 +56,9 @@ } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -67,10 +67,10 @@ } case _LOAD_CONST: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -78,12 +78,12 @@ } case _LOAD_SMALL_INT: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -91,7 +91,7 @@ } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -112,7 +112,7 @@ } case _PUSH_NULL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -133,7 +133,7 @@ } case _END_SEND: { - JitOptSymbol *val; + JitOptRef val; val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; @@ -142,15 +142,22 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef res; + value = stack_pointer[-1]; + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); @@ -159,8 +166,8 @@ } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -171,7 +178,7 @@ } case _TO_BOOL_BOOL: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { @@ -183,8 +190,8 @@ } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -196,7 +203,7 @@ } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -206,7 +213,7 @@ } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -220,8 +227,8 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -232,8 +239,8 @@ } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -245,7 +252,7 @@ } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -255,7 +262,7 @@ } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -265,8 +272,8 @@ } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -277,7 +284,7 @@ } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; @@ -285,8 +292,8 @@ } case _UNARY_INVERT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); @@ -299,111 +306,72 @@ } case _GUARD_NOS_INT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } break; } case _GUARD_TOS_INT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyLong_Type)) { + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } + break; + } + + case _GUARD_NOS_OVERFLOWED: { + break; + } + + case _GUARD_TOS_OVERFLOWED: { break; } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + JitOptRef res; + res = sym_new_compact_int(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_ADD_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + JitOptRef res; + res = sym_new_compact_int(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + JitOptRef res; + res = sym_new_compact_int(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_NOS_FLOAT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -413,7 +381,7 @@ } case _GUARD_TOS_FLOAT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -423,9 +391,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -447,14 +415,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -476,14 +447,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -505,14 +479,44 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -537,11 +541,11 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -566,7 +570,7 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -575,8 +579,8 @@ } case _BINARY_SLICE: { - JitOptSymbol *container; - JitOptSymbol *res; + JitOptRef container; + JitOptRef res; container = stack_pointer[-3]; PyTypeObject *type = sym_get_type(container); if (type == &PyUnicode_Type || @@ -601,7 +605,7 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -610,7 +614,7 @@ } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -619,7 +623,7 @@ } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -628,7 +632,7 @@ } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -638,7 +642,7 @@ } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -648,9 +652,9 @@ } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; assert(sym_matches_type(tuple_st, &PyTuple_Type)); @@ -677,7 +681,7 @@ } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -687,7 +691,7 @@ } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -697,7 +701,7 @@ } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -706,7 +710,7 @@ } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; + JitOptRef getitem; getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; @@ -715,8 +719,8 @@ } case _BINARY_OP_SUBSCR_INIT_CALL: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -761,14 +765,14 @@ } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -777,10 +781,10 @@ } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -804,14 +808,14 @@ } case _GET_AITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; + JitOptRef awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -820,7 +824,7 @@ } case _GET_AWAITABLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -829,15 +833,15 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_unknown(ctx); stack_pointer[-1] = value; break; @@ -850,7 +854,7 @@ } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -859,7 +863,7 @@ } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; + JitOptRef bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -878,8 +882,8 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[-1 + oparg]; (void)top; @@ -892,9 +896,9 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); @@ -906,8 +910,8 @@ } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { @@ -919,7 +923,7 @@ } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -930,8 +934,8 @@ } case _UNPACK_EX: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; (void)top; @@ -967,7 +971,7 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; + JitOptRef locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -978,7 +982,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; + JitOptRef v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -987,7 +991,7 @@ } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); stack_pointer += 1; @@ -996,7 +1000,7 @@ } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); @@ -1015,7 +1019,7 @@ } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1024,7 +1028,7 @@ } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1045,14 +1049,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -1071,7 +1075,7 @@ } case _BUILD_STRING: { - JitOptSymbol *str; + JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -1080,7 +1084,7 @@ } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; + JitOptRef interpolation; interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); @@ -1089,7 +1093,7 @@ } case _BUILD_TEMPLATE: { - JitOptSymbol *template; + JitOptRef template; template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; @@ -1098,8 +1102,8 @@ } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; @@ -1109,7 +1113,7 @@ } case _BUILD_LIST: { - JitOptSymbol *list; + JitOptRef list; list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -1130,7 +1134,7 @@ } case _BUILD_SET: { - JitOptSymbol *set; + JitOptRef set; set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -1139,7 +1143,7 @@ } case _BUILD_MAP: { - JitOptSymbol *map; + JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -1170,7 +1174,7 @@ } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; + JitOptRef attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1179,8 +1183,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; + JitOptRef attr; + JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1191,9 +1195,9 @@ } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; @@ -1208,7 +1212,7 @@ } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); @@ -1235,7 +1239,7 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)offset; @@ -1244,14 +1248,14 @@ } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -1262,11 +1266,16 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; @@ -1274,7 +1283,7 @@ } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; @@ -1283,7 +1292,7 @@ } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; @@ -1292,7 +1301,7 @@ } case _CHECK_ATTR_CLASS: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); @@ -1308,8 +1317,8 @@ } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1323,10 +1332,10 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1357,7 +1366,7 @@ } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef res; if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -1371,7 +1380,7 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1380,9 +1389,9 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -1412,7 +1421,7 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1421,7 +1430,7 @@ } case _IS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1430,7 +1439,7 @@ } case _CONTAINS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1439,7 +1448,7 @@ } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) @@ -1450,7 +1459,7 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1459,7 +1468,7 @@ } case _CONTAINS_OP_DICT: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1468,8 +1477,8 @@ } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; + JitOptRef rest; + JitOptRef match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1478,14 +1487,14 @@ } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1494,7 +1503,7 @@ } case _IMPORT_FROM: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1507,15 +1516,15 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - JitOptSymbol *obj; - JitOptSymbol *len; + JitOptRef obj; + JitOptRef len; obj = stack_pointer[-1]; int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { @@ -1544,7 +1553,7 @@ } case _MATCH_CLASS: { - JitOptSymbol *attrs; + JitOptRef attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1553,7 +1562,7 @@ } case _MATCH_MAPPING: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1562,7 +1571,7 @@ } case _MATCH_SEQUENCE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1571,7 +1580,7 @@ } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; + JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1580,9 +1589,9 @@ } case _GET_ITER: { - JitOptSymbol *iterable; - JitOptSymbol *iter; - JitOptSymbol *index_or_null; + JitOptRef iterable; + JitOptRef iter; + JitOptRef index_or_null; iterable = stack_pointer[-1]; if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { iter = iterable; @@ -1600,7 +1609,7 @@ } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1609,7 +1618,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1632,7 +1641,7 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1641,7 +1650,7 @@ } case _ITER_CHECK_TUPLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1657,7 +1666,7 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1676,7 +1685,7 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; @@ -1685,8 +1694,8 @@ } case _FOR_ITER_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -1695,8 +1704,8 @@ } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); @@ -1707,7 +1716,7 @@ } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; method_and_self[0] = sym_new_not_null(ctx); method_and_self[1] = sym_new_unknown(ctx); @@ -1715,7 +1724,7 @@ } case _WITH_EXCEPT_START: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1724,8 +1733,8 @@ } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; + JitOptRef prev_exc; + JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1744,9 +1753,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1764,9 +1773,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1784,8 +1793,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1799,8 +1808,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1818,9 +1827,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1838,9 +1847,9 @@ } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1857,7 +1866,7 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1865,7 +1874,7 @@ ctx->done = true; break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1873,7 +1882,7 @@ } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { @@ -1890,6 +1899,16 @@ } case _CHECK_METHOD_VERSION: { + JitOptRef callable; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand0; + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); break; } @@ -1902,7 +1921,7 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1911,8 +1930,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1921,8 +1940,8 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; callable = sym_new_not_null(ctx); @@ -1940,8 +1959,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); @@ -1968,9 +1987,9 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *new_frame; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; @@ -1981,16 +2000,16 @@ ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { args--; argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -1999,12 +2018,12 @@ } case _PUSH_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); @@ -2031,7 +2050,7 @@ } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2041,7 +2060,7 @@ } case _GUARD_NOS_NOT_NULL: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2051,7 +2070,7 @@ } case _GUARD_THIRD_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2061,7 +2080,7 @@ } case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2071,8 +2090,8 @@ } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; PyObject* type = (PyObject *)sym_get_type(arg); if (type) { @@ -2090,7 +2109,7 @@ } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2100,8 +2119,8 @@ } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyUnicode_Type)) { res = arg; @@ -2116,7 +2135,7 @@ } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2126,8 +2145,8 @@ } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyTuple_Type)) { res = arg; @@ -2142,9 +2161,9 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -2159,8 +2178,8 @@ } case _CREATE_INIT_FRAME: { - JitOptSymbol *init_frame; - init_frame = NULL; + JitOptRef init_frame; + init_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2175,7 +2194,7 @@ } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2184,7 +2203,7 @@ } case _CALL_BUILTIN_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2193,7 +2212,7 @@ } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2202,7 +2221,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2211,7 +2230,7 @@ } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { @@ -2222,8 +2241,27 @@ } case _CALL_LEN: { - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; + arg = stack_pointer[-1]; res = sym_new_type(ctx, &PyLong_Type); + int tuple_length = sym_tuple_length(arg); + if (tuple_length >= 0) { + PyObject *temp = PyLong_FromLong(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); + } + res = sym_new_const(ctx, temp); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + Py_DECREF(temp); + stack_pointer += 2; + } stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2231,7 +2269,7 @@ } case _GUARD_CALLABLE_ISINSTANCE: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { @@ -2242,9 +2280,9 @@ } case _CALL_ISINSTANCE: { - JitOptSymbol *cls; - JitOptSymbol *instance; - JitOptSymbol *res; + JitOptRef cls; + JitOptRef instance; + JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; res = sym_new_type(ctx, &PyBool_Type); @@ -2265,7 +2303,7 @@ } case _GUARD_CALLABLE_LIST_APPEND: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { @@ -2282,7 +2320,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2291,7 +2329,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2300,7 +2338,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2309,7 +2347,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2326,8 +2364,8 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; @@ -2352,7 +2390,7 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2367,14 +2405,14 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; + JitOptRef func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; + JitOptRef func_out; func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; @@ -2383,7 +2421,7 @@ } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2405,7 +2443,7 @@ } case _BUILD_SLICE: { - JitOptSymbol *slice; + JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; @@ -2414,21 +2452,21 @@ } case _CONVERT_VALUE: { - JitOptSymbol *result; + JitOptRef result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2437,8 +2475,8 @@ } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2449,9 +2487,9 @@ } case _BINARY_OP: { - JitOptSymbol *rhs; - JitOptSymbol *lhs; - JitOptSymbol *res; + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; bool lhs_int = sym_matches_type(lhs, &PyLong_Type); @@ -2494,11 +2532,11 @@ } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2526,7 +2564,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2540,7 +2578,7 @@ } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2554,7 +2592,7 @@ } case _GUARD_IS_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2572,7 +2610,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2620,9 +2658,9 @@ } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2630,17 +2668,17 @@ } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2666,15 +2704,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -2683,9 +2721,9 @@ } case _POP_CALL_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2693,9 +2731,9 @@ } case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2703,9 +2741,9 @@ } case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2713,8 +2751,8 @@ } case _LOAD_CONST_UNDER_INLINE: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; @@ -2725,8 +2763,8 @@ } case _LOAD_CONST_UNDER_INLINE_BORROW: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; |