diff options
author | Mark Shannon <mark@hotpy.org> | 2025-02-28 18:00:38 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-02-28 18:00:38 +0000 |
commit | 54965f3fb25b381995a73b09d928c344bd2b86bd (patch) | |
tree | cf2f71ac2775a4cfec347a70fd123d176c353948 /Python | |
parent | ab11c097052757b79060c75dd4835c2431e752b7 (diff) | |
download | cpython-54965f3fb25b381995a73b09d928c344bd2b86bd.tar.gz cpython-54965f3fb25b381995a73b09d928c344bd2b86bd.zip |
GH-130296: Avoid stack transients in four instructions. (GH-130310)
* Combine _GUARD_GLOBALS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_MODULE_FROM_KEYS into _LOAD_GLOBAL_MODULE
* Combine _GUARD_BUILTINS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_BUILTINS_FROM_KEYS into _LOAD_GLOBAL_BUILTINS
* Combine _CHECK_ATTR_MODULE_PUSH_KEYS and _LOAD_ATTR_MODULE_FROM_KEYS into _LOAD_ATTR_MODULE
* Remove stack transient in LOAD_ATTR_WITH_HINT
Diffstat (limited to 'Python')
-rw-r--r-- | Python/bytecodes.c | 128 | ||||
-rw-r--r-- | Python/executor_cases.c.h | 233 | ||||
-rw-r--r-- | Python/generated_cases.c.h | 78 | ||||
-rw-r--r-- | Python/optimizer.c | 27 | ||||
-rw-r--r-- | Python/optimizer_analysis.c | 87 | ||||
-rw-r--r-- | Python/optimizer_bytecodes.c | 65 | ||||
-rw-r--r-- | Python/optimizer_cases.c.h | 142 |
7 files changed, 226 insertions, 534 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1b9b651fd01..24aa7bbb87c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1689,70 +1689,55 @@ dummy_func( assert(DK_IS_UNICODE(keys)); } - op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version / 1 -- globals_keys: PyDictKeysObject *)) + op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict)); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - - op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version / 1 -- builtins_keys: PyDictKeysObject *)) - { - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict)); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - - op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEAD(globals_keys); - SYNC_SP(); DEOPT_IF(res_o == NULL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); DEOPT_IF(!increfed); #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } - op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) + { + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict)); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEAD(builtins_keys); - SYNC_SP(); DEOPT_IF(res_o == NULL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); DEOPT_IF(!increfed); #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } macro(LOAD_GLOBAL_MODULE) = unused/1 + // Skip over the counter - _GUARD_GLOBALS_VERSION_PUSH_KEYS + - unused/1 + // Skip over the builtins version - _LOAD_GLOBAL_MODULE_FROM_KEYS + + NOP + // For guard insertion in the JIT optimizer + _LOAD_GLOBAL_MODULE + _PUSH_NULL_CONDITIONAL; macro(LOAD_GLOBAL_BUILTIN) = unused/1 + // Skip over the counter _GUARD_GLOBALS_VERSION + - _GUARD_BUILTINS_VERSION_PUSH_KEYS + - _LOAD_GLOBAL_BUILTINS_FROM_KEYS + + _LOAD_GLOBAL_BUILTINS + _PUSH_NULL_CONDITIONAL; inst(DELETE_FAST, (--)) { @@ -2247,23 +2232,17 @@ dummy_func( unused/5 + _PUSH_NULL_CONDITIONAL; - op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys: PyDictKeysObject *)) { + op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version); - mod_keys = keys; - } - - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr)) { - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - POP_INPUT(mod_keys); DEOPT_IF(attr_o == NULL); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -2271,8 +2250,7 @@ dummy_func( DEOPT_IF(true); } #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); PyStackRef_CLOSE(owner); @@ -2280,62 +2258,49 @@ dummy_func( macro(LOAD_ATTR_MODULE) = unused/1 + - _CHECK_ATTR_MODULE_PUSH_KEYS + - _LOAD_ATTR_MODULE_FROM_KEYS + + _LOAD_ATTR_MODULE + unused/5 + _PUSH_NULL_CONDITIONAL; - op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - EXIT_IF(dict_o == NULL); - assert(PyDict_CheckExact((PyObject *)dict_o)); - dict = dict_o; - } - - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr)) { + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL); + assert(PyDict_CheckExact((PyObject *)dict)); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - POP_INPUT(dict); DEOPT_IF(true); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - POP_INPUT(dict); DEOPT_IF(true); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - POP_INPUT(dict); DEOPT_IF(true); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - POP_INPUT(dict); DEOPT_IF(true); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - POP_INPUT(dict); DEOPT_IF(true); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - DEAD(dict); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); } macro(LOAD_ATTR_WITH_HINT) = unused/1 + _GUARD_TYPE_VERSION + - _CHECK_ATTR_WITH_HINT + _LOAD_ATTR_WITH_HINT + unused/5 + _PUSH_NULL_CONDITIONAL; @@ -5045,6 +5010,11 @@ dummy_func( value = PyStackRef_FromPyObjectNew(ptr); } + tier2 pure op (_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectNew(ptr); + } + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } @@ -5060,38 +5030,6 @@ dummy_func( DEOPT_IF(func->func_version != func_version); } - tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res)) { - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - PyObject *res_o = entries[index].me_value; - DEOPT_IF(res_o == NULL); - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - } - - tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res)) { - PyDictObject *dict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - PyObject *res_o = entries[index].me_value; - DEOPT_IF(res_o == NULL); - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - } - - tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr)) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - PyObject *attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - DECREF_INPUTS(); - } - tier2 op(_START_EXECUTOR, (executor/4 --)) { Py_CLEAR(tstate->previous_executor); #ifndef _Py_JIT diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6ae88a81724..e164f11620d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2267,9 +2267,10 @@ break; } - case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { - PyDictKeysObject *globals_keys; + case _LOAD_GLOBAL_MODULE: { + _PyStackRef res; uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -2280,44 +2281,10 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - stack_pointer[0].bits = (uintptr_t)globals_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { - PyDictKeysObject *builtins_keys; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - stack_pointer[0].bits = (uintptr_t)builtins_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_GLOBAL_MODULE_FROM_KEYS: { - PyDictKeysObject *globals_keys; - _PyStackRef res; - globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2329,8 +2296,7 @@ JUMP_TO_JUMP_TARGET(); } #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; @@ -2339,15 +2305,23 @@ break; } - case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { - PyDictKeysObject *builtins_keys; + case _LOAD_GLOBAL_BUILTINS: { _PyStackRef res; - builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2359,8 +2333,7 @@ JUMP_TO_JUMP_TARGET(); } #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; @@ -3143,11 +3116,12 @@ break; } - case _CHECK_ATTR_MODULE_PUSH_KEYS: { + case _LOAD_ATTR_MODULE: { _PyStackRef owner; - PyDictKeysObject *mod_keys; + _PyStackRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); @@ -3160,27 +3134,10 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - mod_keys = keys; - stack_pointer[0].bits = (uintptr_t)mod_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_MODULE_FROM_KEYS: { - PyDictKeysObject *mod_keys; - _PyStackRef owner; - _PyStackRef attr; - mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - owner = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3194,8 +3151,7 @@ } } #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); stack_pointer[-1] = attr; @@ -3205,37 +3161,22 @@ break; } - case _CHECK_ATTR_WITH_HINT: { + case _LOAD_ATTR_WITH_HINT: { _PyStackRef owner; - PyDictObject *dict; + _PyStackRef attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - if (dict_o == NULL) { + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict_o)); - dict = dict_o; - stack_pointer[0].bits = (uintptr_t)dict; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_WITH_HINT: { - PyDictObject *dict; - _PyStackRef owner; - _PyStackRef attr; - oparg = CURRENT_OPARG(); - dict = (PyDictObject *)stack_pointer[-1].bits; - owner = stack_pointer[-2]; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + assert(PyDict_CheckExact((PyObject *)dict)); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3243,8 +3184,6 @@ } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3253,8 +3192,6 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3263,8 +3200,6 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3273,8 +3208,6 @@ attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3283,15 +3216,10 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); + PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; break; } @@ -3395,7 +3323,7 @@ break; } - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has unused cache entries */ + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ case _GUARD_DORV_NO_DICT: { _PyStackRef owner; @@ -6833,6 +6761,23 @@ break; } + case _POP_TOP_LOAD_CONST_INLINE: { + _PyStackRef pop; + _PyStackRef value; + pop = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectNew(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _LOAD_CONST_INLINE_BORROW: { _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); @@ -6871,70 +6816,6 @@ break; } - case _LOAD_GLOBAL_MODULE: { - _PyStackRef res; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - PyObject *res_o = entries[index].me_value; - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_GLOBAL_BUILTINS: { - _PyStackRef res; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - PyObject *res_o = entries[index].me_value; - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_MODULE: { - _PyStackRef owner; - _PyStackRef attr; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - PyObject *attr_o = ep->me_value; - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; - break; - } - case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND0(); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 564e5be92ae..8c3c0e3910b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8173,14 +8173,14 @@ INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; - PyDictKeysObject *mod_keys; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS + // _LOAD_ATTR_MODULE { owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); @@ -8195,16 +8195,10 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); @@ -8220,8 +8214,7 @@ } } #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); stack_pointer[-1] = attr; @@ -8530,7 +8523,6 @@ INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; - PyDictObject *dict; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -8546,22 +8538,18 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _CHECK_ATTR_WITH_HINT + // _LOAD_ATTR_WITH_HINT { + uint16_t hint = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - if (dict_o == NULL) { + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - assert(PyDict_CheckExact((PyObject *)dict_o)); - dict = dict_o; - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); + assert(PyDict_CheckExact((PyObject *)dict)); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { if (true) { @@ -8608,13 +8596,10 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); + stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); + PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9052,7 +9037,6 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -9073,9 +9057,10 @@ } assert(DK_IS_UNICODE(keys)); } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS + // _LOAD_GLOBAL_BUILTINS { uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -9088,13 +9073,8 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -9109,8 +9089,7 @@ JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } @@ -9136,13 +9115,16 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS + // _NOP + { + } + // _LOAD_GLOBAL_MODULE { uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -9155,14 +9137,9 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -9177,8 +9154,7 @@ JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } diff --git a/Python/optimizer.c b/Python/optimizer.c index d86d58d82e3..e05523451da 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -230,16 +230,18 @@ _PyUOpPrint(const _PyUOpInstruction *uop) } switch(uop->format) { case UOP_FORMAT_TARGET: - printf(" (%d, target=%d, operand=%#" PRIx64, + printf(" (%d, target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64, uop->oparg, uop->target, - (uint64_t)uop->operand0); + (uint64_t)uop->operand0, + (uint64_t)uop->operand1); break; case UOP_FORMAT_JUMP: - printf(" (%d, jump_target=%d, operand=%#" PRIx64, + printf(" (%d, jump_target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64, uop->oparg, uop->jump_target, - (uint64_t)uop->operand0); + (uint64_t)uop->operand0, + (uint64_t)uop->operand1); break; default: printf(" (%d, Unknown format)", uop->oparg); @@ -682,7 +684,7 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { - case OPARG_FULL: + case OPARG_SIMPLE: assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); break; case OPARG_CACHE_1: @@ -716,6 +718,21 @@ translate_bytecode_to_trace( } #endif break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; default: fprintf(stderr, "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6c0aadb87e6..29a05088e62 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -93,26 +93,27 @@ type_watcher_callback(PyTypeObject* type) } static PyObject * -convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) +convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) { - assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE_FROM_KEYS); + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); assert(PyDict_CheckExact(obj)); PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - assert(inst->operand0 <= UINT16_MAX); - if ((int)inst->operand0 >= dict->ma_keys->dk_nentries) { + int64_t index = inst->operand1; + assert(index <= UINT16_MAX); + if ((int)index >= dict->ma_keys->dk_nentries) { return NULL; } - PyObject *res = entries[inst->operand0].me_value; + PyObject *res = entries[index].me_value; if (res == NULL) { return NULL; } if (_Py_IsImmortal(res)) { - inst->opcode = _LOAD_CONST_INLINE_BORROW; + inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE_BORROW; } else { - inst->opcode = _LOAD_CONST_INLINE; + inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE : _LOAD_CONST_INLINE; } if (inst->oparg & 1) { assert(inst[1].opcode == _PUSH_NULL_CONDITIONAL); @@ -198,21 +199,18 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; switch(opcode) { - case _GUARD_BUILTINS_VERSION_PUSH_KEYS: - if (incorrect_keys(inst, builtins)) { + case _GUARD_GLOBALS_VERSION: + if (incorrect_keys(inst, globals)) { OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } - if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - continue; - } - if (!check_next_uop(buffer, buffer_size, pc, - _LOAD_GLOBAL_BUILTINS_FROM_KEYS)) { + if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { continue; } - if ((builtins_watched & 1) == 0) { - PyDict_Watch(BUILTINS_WATCHER_ID, builtins); - builtins_watched |= 1; + if ((globals_watched & 1) == 0) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + globals_watched |= 1; } if (function_checked & 1) { buffer[pc].opcode = NOP; @@ -222,24 +220,29 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, buffer[pc].operand0 = function_version; function_checked |= 1; } - // We're no longer pushing the builtins keys; rewrite the - // instruction that consumed the keys to load them from the - // frame. - buffer[pc + 1].opcode = _LOAD_GLOBAL_BUILTINS; break; - case _GUARD_GLOBALS_VERSION: - case _GUARD_GLOBALS_VERSION_PUSH_KEYS: - if (incorrect_keys(inst, globals)) { + case _LOAD_GLOBAL_BUILTINS: + if (incorrect_keys(inst, builtins)) { OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } - uint64_t watched_mutations = get_mutations(globals); - if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { continue; } - if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS && - !check_next_uop(buffer, buffer_size, pc, - _LOAD_GLOBAL_MODULE_FROM_KEYS)) { + if ((builtins_watched & 1) == 0) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + builtins_watched |= 1; + } + if (function_checked & globals_watched & 1) { + convert_global_to_const(inst, builtins, false); + } + break; + case _LOAD_GLOBAL_MODULE: + if (incorrect_keys(inst, globals)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + return 0; + } + if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { continue; } if ((globals_watched & 1) == 0) { @@ -247,29 +250,13 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, _Py_BloomFilter_Add(dependencies, globals); globals_watched |= 1; } - if (function_checked & 1) { - buffer[pc].opcode = NOP; - } - else { - buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand0 = function_version; + if ((function_checked & 1) == 0 && buffer[pc-1].opcode == _NOP) { + buffer[pc-1].opcode = _CHECK_FUNCTION; + buffer[pc-1].operand0 = function_version; function_checked |= 1; } - if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) { - // We're no longer pushing the globals keys; rewrite the - // instruction that consumed the keys to load them from the - // frame. - buffer[pc + 1].opcode = _LOAD_GLOBAL_MODULE; - } - break; - case _LOAD_GLOBAL_BUILTINS: - if (function_checked & globals_watched & builtins_watched & 1) { - convert_global_to_const(inst, builtins); - } - break; - case _LOAD_GLOBAL_MODULE: - if (function_checked & globals_watched & 1) { - convert_global_to_const(inst, globals); + if (function_checked & 1) { + convert_global_to_const(inst, globals, false); } break; case _PUSH_FRAME: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 41eb59c931a..f3625a1492c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -492,6 +492,14 @@ dummy_func(void) { value = sym_new_const(ctx, ptr); } + op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + value = sym_new_const(ctx, ptr); + } + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = bottom; @@ -509,22 +517,27 @@ dummy_func(void) { (void)offset; } - op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) { + op(_LOAD_ATTR_MODULE, (dict_version/2, owner, index/1 -- attr)) { (void)dict_version; - mod_keys = sym_new_not_null(ctx); + (void)index; + attr = NULL; if (sym_is_const(owner)) { - PyObject *cnst = sym_get_const(owner); - if (PyModule_CheckExact(cnst)) { - PyModuleObject *mod = (PyModuleObject *)cnst; + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); - this_instr->opcode = _NOP; + PyObject *res = convert_global_to_const(this_instr, dict, true); + attr = sym_new_const(ctx, res); } } } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } } op (_PUSH_NULL_CONDITIONAL, ( -- null if (oparg & 1))) { @@ -541,35 +554,7 @@ dummy_func(void) { } } - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) { - (void)index; - attr = NULL; - if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. - assert(sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); - assert(PyModule_CheckExact(mod)); - PyObject *dict = mod->md_dict; - PyObject *res = convert_global_to_const(this_instr, dict); - if (res != NULL) { - this_instr[-1].opcode = _POP_TOP; - attr = sym_new_const(ctx, res); - } - else { - this_instr->opcode = _LOAD_ATTR_MODULE; - } - } - if (attr == NULL) { - /* No conversion made. We don't know what `attr` is. */ - attr = sym_new_not_null(ctx); - } - } - - op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { - dict = sym_new_not_null(ctx); - } - - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { attr = sym_new_not_null(ctx); (void)hint; } @@ -888,16 +873,6 @@ dummy_func(void) { ctx->done = true; } - op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version/1 -- globals_keys)) { - globals_keys = sym_new_unknown(ctx); - (void)version; - } - - op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version/1 -- builtins_keys)) { - builtins_keys = sym_new_unknown(ctx); - (void)version; - } - op(_REPLACE_WITH_TRUE, (value -- res)) { res = sym_new_const(ctx, Py_True); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 51d0fa63e64..0372870b94e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -932,39 +932,21 @@ break; } - case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { - JitOptSymbol *globals_keys; - uint16_t version = (uint16_t)this_instr->operand0; - globals_keys = sym_new_unknown(ctx); - (void)version; - stack_pointer[0] = globals_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { - JitOptSymbol *builtins_keys; - uint16_t version = (uint16_t)this_instr->operand0; - builtins_keys = sym_new_unknown(ctx); - (void)version; - stack_pointer[0] = builtins_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_GLOBAL_MODULE_FROM_KEYS: { + case _LOAD_GLOBAL_MODULE: { JitOptSymbol *res; res = sym_new_not_null(ctx); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { + case _LOAD_GLOBAL_BUILTINS: { JitOptSymbol *res; res = sym_new_not_null(ctx); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1168,80 +1150,34 @@ break; } - case _CHECK_ATTR_MODULE_PUSH_KEYS: { + case _LOAD_ATTR_MODULE: { JitOptSymbol *owner; - JitOptSymbol *mod_keys; + JitOptSymbol *attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; + uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; - mod_keys = sym_new_not_null(ctx); + (void)index; + attr = NULL; if (sym_is_const(owner)) { - PyObject *cnst = sym_get_const(owner); - if (PyModule_CheckExact(cnst)) { - PyModuleObject *mod = (PyModuleObject *)cnst; + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; - stack_pointer[0] = mod_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = attr; uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); - this_instr->opcode = _NOP; + PyObject *res = convert_global_to_const(this_instr, dict, true); + attr = sym_new_const(ctx, res); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[0] = mod_keys; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_MODULE_FROM_KEYS: { - JitOptSymbol *owner; - JitOptSymbol *attr; - owner = stack_pointer[-2]; - uint16_t index = (uint16_t)this_instr->operand0; - (void)index; - attr = NULL; - if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. - assert(sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); - assert(PyModule_CheckExact(mod)); - PyObject *dict = mod->md_dict; - stack_pointer[-2] = attr; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - PyObject *res = convert_global_to_const(this_instr, dict); - if (res != NULL) { - this_instr[-1].opcode = _POP_TOP; - attr = sym_new_const(ctx, res); - } - else { - this_instr->opcode = _LOAD_ATTR_MODULE; - } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } - stack_pointer[-2] = attr; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _CHECK_ATTR_WITH_HINT: { - JitOptSymbol *dict; - dict = sym_new_not_null(ctx); - stack_pointer[0] = dict; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = attr; break; } @@ -1250,9 +1186,7 @@ uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; - stack_pointer[-2] = attr; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = attr; break; } @@ -2420,6 +2354,14 @@ break; } + case _POP_TOP_LOAD_CONST_INLINE: { + JitOptSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = sym_new_const(ctx, ptr); + stack_pointer[-1] = value; + break; + } + case _LOAD_CONST_INLINE_BORROW: { JitOptSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand0; @@ -2432,7 +2374,8 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { JitOptSymbol *value; - value = sym_new_not_null(ctx); + PyObject *ptr = (PyObject *)this_instr->operand0; + value = sym_new_const(ctx, ptr); stack_pointer[-1] = value; break; } @@ -2441,31 +2384,6 @@ break; } - case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_MODULE: { - JitOptSymbol *attr; - attr = sym_new_not_null(ctx); - stack_pointer[-1] = attr; - break; - } - case _START_EXECUTOR: { break; } |