aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python
diff options
context:
space:
mode:
authorMark Shannon <mark@hotpy.org>2025-02-28 18:00:38 +0000
committerGitHub <noreply@github.com>2025-02-28 18:00:38 +0000
commit54965f3fb25b381995a73b09d928c344bd2b86bd (patch)
treecf2f71ac2775a4cfec347a70fd123d176c353948 /Python
parentab11c097052757b79060c75dd4835c2431e752b7 (diff)
downloadcpython-54965f3fb25b381995a73b09d928c344bd2b86bd.tar.gz
cpython-54965f3fb25b381995a73b09d928c344bd2b86bd.zip
GH-130296: Avoid stack transients in four instructions. (GH-130310)
* Combine _GUARD_GLOBALS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_MODULE_FROM_KEYS into _LOAD_GLOBAL_MODULE * Combine _GUARD_BUILTINS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_BUILTINS_FROM_KEYS into _LOAD_GLOBAL_BUILTINS * Combine _CHECK_ATTR_MODULE_PUSH_KEYS and _LOAD_ATTR_MODULE_FROM_KEYS into _LOAD_ATTR_MODULE * Remove stack transient in LOAD_ATTR_WITH_HINT
Diffstat (limited to 'Python')
-rw-r--r--Python/bytecodes.c128
-rw-r--r--Python/executor_cases.c.h233
-rw-r--r--Python/generated_cases.c.h78
-rw-r--r--Python/optimizer.c27
-rw-r--r--Python/optimizer_analysis.c87
-rw-r--r--Python/optimizer_bytecodes.c65
-rw-r--r--Python/optimizer_cases.c.h142
7 files changed, 226 insertions, 534 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 1b9b651fd01..24aa7bbb87c 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -1689,70 +1689,55 @@ dummy_func(
assert(DK_IS_UNICODE(keys));
}
- op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version / 1 -- globals_keys: PyDictKeysObject *))
+ op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res))
{
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict));
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version);
- globals_keys = keys;
- assert(DK_IS_UNICODE(globals_keys));
- }
-
- op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version / 1 -- builtins_keys: PyDictKeysObject *))
- {
- PyDictObject *dict = (PyDictObject *)BUILTINS();
- DEOPT_IF(!PyDict_CheckExact(dict));
- PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version);
- builtins_keys = keys;
- assert(DK_IS_UNICODE(builtins_keys));
- }
-
- op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res)) {
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
+ assert(index < DK_SIZE(keys));
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEAD(globals_keys);
- SYNC_SP();
DEOPT_IF(res_o == NULL);
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
DEOPT_IF(!increfed);
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
}
- op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res)) {
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
+ op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res))
+ {
+ PyDictObject *dict = (PyDictObject *)BUILTINS();
+ DEOPT_IF(!PyDict_CheckExact(dict));
+ PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version);
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEAD(builtins_keys);
- SYNC_SP();
DEOPT_IF(res_o == NULL);
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
DEOPT_IF(!increfed);
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
}
macro(LOAD_GLOBAL_MODULE) =
unused/1 + // Skip over the counter
- _GUARD_GLOBALS_VERSION_PUSH_KEYS +
- unused/1 + // Skip over the builtins version
- _LOAD_GLOBAL_MODULE_FROM_KEYS +
+ NOP + // For guard insertion in the JIT optimizer
+ _LOAD_GLOBAL_MODULE +
_PUSH_NULL_CONDITIONAL;
macro(LOAD_GLOBAL_BUILTIN) =
unused/1 + // Skip over the counter
_GUARD_GLOBALS_VERSION +
- _GUARD_BUILTINS_VERSION_PUSH_KEYS +
- _LOAD_GLOBAL_BUILTINS_FROM_KEYS +
+ _LOAD_GLOBAL_BUILTINS +
_PUSH_NULL_CONDITIONAL;
inst(DELETE_FAST, (--)) {
@@ -2247,23 +2232,17 @@ dummy_func(
unused/5 +
_PUSH_NULL_CONDITIONAL;
- op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys: PyDictKeysObject *)) {
+ op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version);
- mod_keys = keys;
- }
-
- op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr)) {
- assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
- assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
- PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
+ assert(keys->dk_kind == DICT_KEYS_UNICODE);
+ assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries));
+ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- // Clear mod_keys from stack in case we need to deopt
- POP_INPUT(mod_keys);
DEOPT_IF(attr_o == NULL);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
@@ -2271,8 +2250,7 @@ dummy_func(
DEOPT_IF(true);
}
#else
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
PyStackRef_CLOSE(owner);
@@ -2280,62 +2258,49 @@ dummy_func(
macro(LOAD_ATTR_MODULE) =
unused/1 +
- _CHECK_ATTR_MODULE_PUSH_KEYS +
- _LOAD_ATTR_MODULE_FROM_KEYS +
+ _LOAD_ATTR_MODULE +
unused/5 +
_PUSH_NULL_CONDITIONAL;
- op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) {
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
-
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
- EXIT_IF(dict_o == NULL);
- assert(PyDict_CheckExact((PyObject *)dict_o));
- dict = dict_o;
- }
-
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr)) {
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ DEOPT_IF(dict == NULL);
+ assert(PyDict_CheckExact((PyObject *)dict));
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
- POP_INPUT(dict);
DEOPT_IF(true);
}
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UNLOCK_OBJECT(dict);
- POP_INPUT(dict);
DEOPT_IF(true);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
- POP_INPUT(dict);
DEOPT_IF(true);
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- POP_INPUT(dict);
DEOPT_IF(true);
}
attr_o = ep->me_value;
if (attr_o == NULL) {
UNLOCK_OBJECT(dict);
- POP_INPUT(dict);
DEOPT_IF(true);
}
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
- DEAD(dict);
- DECREF_INPUTS();
+ PyStackRef_CLOSE(owner);
}
macro(LOAD_ATTR_WITH_HINT) =
unused/1 +
_GUARD_TYPE_VERSION +
- _CHECK_ATTR_WITH_HINT +
_LOAD_ATTR_WITH_HINT +
unused/5 +
_PUSH_NULL_CONDITIONAL;
@@ -5045,6 +5010,11 @@ dummy_func(
value = PyStackRef_FromPyObjectNew(ptr);
}
+ tier2 pure op (_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) {
+ PyStackRef_CLOSE(pop);
+ value = PyStackRef_FromPyObjectNew(ptr);
+ }
+
tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
value = PyStackRef_FromPyObjectImmortal(ptr);
}
@@ -5060,38 +5030,6 @@ dummy_func(
DEOPT_IF(func->func_version != func_version);
}
- tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res)) {
- PyDictObject *dict = (PyDictObject *)GLOBALS();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- PyObject *res_o = entries[index].me_value;
- DEOPT_IF(res_o == NULL);
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
-
- tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res)) {
- PyDictObject *dict = (PyDictObject *)BUILTINS();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- PyObject *res_o = entries[index].me_value;
- DEOPT_IF(res_o == NULL);
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
-
- tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr)) {
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
- assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
- assert(index < dict->ma_keys->dk_nentries);
- PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
- PyObject *attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL);
- STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
- DECREF_INPUTS();
- }
-
tier2 op(_START_EXECUTOR, (executor/4 --)) {
Py_CLEAR(tstate->previous_executor);
#ifndef _Py_JIT
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 6ae88a81724..e164f11620d 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -2267,9 +2267,10 @@
break;
}
- case _GUARD_GLOBALS_VERSION_PUSH_KEYS: {
- PyDictKeysObject *globals_keys;
+ case _LOAD_GLOBAL_MODULE: {
+ _PyStackRef res;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
+ uint16_t index = (uint16_t)CURRENT_OPERAND1();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@@ -2280,44 +2281,10 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- globals_keys = keys;
- assert(DK_IS_UNICODE(globals_keys));
- stack_pointer[0].bits = (uintptr_t)globals_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _GUARD_BUILTINS_VERSION_PUSH_KEYS: {
- PyDictKeysObject *builtins_keys;
- uint16_t version = (uint16_t)CURRENT_OPERAND0();
- PyDictObject *dict = (PyDictObject *)BUILTINS();
- if (!PyDict_CheckExact(dict)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- builtins_keys = keys;
- assert(DK_IS_UNICODE(builtins_keys));
- stack_pointer[0].bits = (uintptr_t)builtins_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
- PyDictKeysObject *globals_keys;
- _PyStackRef res;
- globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
+ assert(index < DK_SIZE(keys));
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (res_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -2329,8 +2296,7 @@
JUMP_TO_JUMP_TARGET();
}
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
stack_pointer[0] = res;
@@ -2339,15 +2305,23 @@
break;
}
- case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
- PyDictKeysObject *builtins_keys;
+ case _LOAD_GLOBAL_BUILTINS: {
_PyStackRef res;
- builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
+ uint16_t version = (uint16_t)CURRENT_OPERAND0();
+ uint16_t index = (uint16_t)CURRENT_OPERAND1();
+ PyDictObject *dict = (PyDictObject *)BUILTINS();
+ if (!PyDict_CheckExact(dict)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (res_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -2359,8 +2333,7 @@
JUMP_TO_JUMP_TARGET();
}
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
stack_pointer[0] = res;
@@ -3143,11 +3116,12 @@
break;
}
- case _CHECK_ATTR_MODULE_PUSH_KEYS: {
+ case _LOAD_ATTR_MODULE: {
_PyStackRef owner;
- PyDictKeysObject *mod_keys;
+ _PyStackRef attr;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)CURRENT_OPERAND0();
+ uint16_t index = (uint16_t)CURRENT_OPERAND1();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
UOP_STAT_INC(uopcode, miss);
@@ -3160,27 +3134,10 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- mod_keys = keys;
- stack_pointer[0].bits = (uintptr_t)mod_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_ATTR_MODULE_FROM_KEYS: {
- PyDictKeysObject *mod_keys;
- _PyStackRef owner;
- _PyStackRef attr;
- mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
- owner = stack_pointer[-2];
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
- assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
- PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
+ assert(keys->dk_kind == DICT_KEYS_UNICODE);
+ assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries));
+ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- // Clear mod_keys from stack in case we need to deopt
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3194,8 +3151,7 @@
}
}
#else
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
stack_pointer[-1] = attr;
@@ -3205,37 +3161,22 @@
break;
}
- case _CHECK_ATTR_WITH_HINT: {
+ case _LOAD_ATTR_WITH_HINT: {
_PyStackRef owner;
- PyDictObject *dict;
+ _PyStackRef attr;
+ oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
+ uint16_t hint = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
- if (dict_o == NULL) {
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ if (dict == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(PyDict_CheckExact((PyObject *)dict_o));
- dict = dict_o;
- stack_pointer[0].bits = (uintptr_t)dict;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_ATTR_WITH_HINT: {
- PyDictObject *dict;
- _PyStackRef owner;
- _PyStackRef attr;
- oparg = CURRENT_OPARG();
- dict = (PyDictObject *)stack_pointer[-1].bits;
- owner = stack_pointer[-2];
- uint16_t hint = (uint16_t)CURRENT_OPERAND0();
+ assert(PyDict_CheckExact((PyObject *)dict));
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3243,8 +3184,6 @@
}
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UNLOCK_OBJECT(dict);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3253,8 +3192,6 @@
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3263,8 +3200,6 @@
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3273,8 +3208,6 @@
attr_o = ep->me_value;
if (attr_o == NULL) {
UNLOCK_OBJECT(dict);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@@ -3283,15 +3216,10 @@
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
+ stack_pointer[-1] = attr;
_PyFrame_SetStackPointer(frame, stack_pointer);
- _PyStackRef tmp = owner;
- owner = attr;
- stack_pointer[-1] = owner;
- PyStackRef_CLOSE(tmp);
+ PyStackRef_CLOSE(owner);
stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer[-1] = attr;
break;
}
@@ -3395,7 +3323,7 @@
break;
}
- /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has unused cache entries */
+ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */
case _GUARD_DORV_NO_DICT: {
_PyStackRef owner;
@@ -6833,6 +6761,23 @@
break;
}
+ case _POP_TOP_LOAD_CONST_INLINE: {
+ _PyStackRef pop;
+ _PyStackRef value;
+ pop = stack_pointer[-1];
+ PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(pop);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ value = PyStackRef_FromPyObjectNew(ptr);
+ stack_pointer[0] = value;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
+
case _LOAD_CONST_INLINE_BORROW: {
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
@@ -6871,70 +6816,6 @@
break;
}
- case _LOAD_GLOBAL_MODULE: {
- _PyStackRef res;
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- PyDictObject *dict = (PyDictObject *)GLOBALS();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- PyObject *res_o = entries[index].me_value;
- if (res_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
- stack_pointer[0] = res;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_GLOBAL_BUILTINS: {
- _PyStackRef res;
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- PyDictObject *dict = (PyDictObject *)BUILTINS();
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- PyObject *res_o = entries[index].me_value;
- if (res_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
- stack_pointer[0] = res;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_ATTR_MODULE: {
- _PyStackRef owner;
- _PyStackRef attr;
- owner = stack_pointer[-1];
- uint16_t index = (uint16_t)CURRENT_OPERAND0();
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
- assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
- assert(index < dict->ma_keys->dk_nentries);
- PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
- PyObject *attr_o = ep->me_value;
- if (attr_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyStackRef tmp = owner;
- owner = attr;
- stack_pointer[-1] = owner;
- PyStackRef_CLOSE(tmp);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer[-1] = attr;
- break;
- }
-
case _START_EXECUTOR: {
PyObject *executor = (PyObject *)CURRENT_OPERAND0();
_PyFrame_SetStackPointer(frame, stack_pointer);
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 564e5be92ae..8c3c0e3910b 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -8173,14 +8173,14 @@
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
- PyDictKeysObject *mod_keys;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
- // _CHECK_ATTR_MODULE_PUSH_KEYS
+ // _LOAD_ATTR_MODULE
{
owner = stack_pointer[-1];
uint32_t dict_version = read_u32(&this_instr[2].cache);
+ uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
UPDATE_MISS_STATS(LOAD_ATTR);
@@ -8195,16 +8195,10 @@
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
- mod_keys = keys;
- }
- // _LOAD_ATTR_MODULE_FROM_KEYS
- {
- uint16_t index = read_u16(&this_instr[4].cache);
- assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
- assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
- PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
+ assert(keys->dk_kind == DICT_KEYS_UNICODE);
+ assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries));
+ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- // Clear mod_keys from stack in case we need to deopt
if (attr_o == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
@@ -8220,8 +8214,7 @@
}
}
#else
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
stack_pointer[-1] = attr;
@@ -8530,7 +8523,6 @@
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
- PyDictObject *dict;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
@@ -8546,22 +8538,18 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
- // _CHECK_ATTR_WITH_HINT
+ // _LOAD_ATTR_WITH_HINT
{
+ uint16_t hint = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
- if (dict_o == NULL) {
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ if (dict == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
- assert(PyDict_CheckExact((PyObject *)dict_o));
- dict = dict_o;
- }
- // _LOAD_ATTR_WITH_HINT
- {
- uint16_t hint = read_u16(&this_instr[4].cache);
+ assert(PyDict_CheckExact((PyObject *)dict));
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
if (true) {
@@ -8608,13 +8596,10 @@
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
+ stack_pointer[-1] = attr;
_PyFrame_SetStackPointer(frame, stack_pointer);
- _PyStackRef tmp = owner;
- owner = attr;
- stack_pointer[-1] = owner;
- PyStackRef_CLOSE(tmp);
+ PyStackRef_CLOSE(owner);
stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer[-1] = attr;
}
/* Skip 5 cache entries */
// _PUSH_NULL_CONDITIONAL
@@ -9052,7 +9037,6 @@
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
- PyDictKeysObject *builtins_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
@@ -9073,9 +9057,10 @@
}
assert(DK_IS_UNICODE(keys));
}
- // _GUARD_BUILTINS_VERSION_PUSH_KEYS
+ // _LOAD_GLOBAL_BUILTINS
{
uint16_t version = read_u16(&this_instr[3].cache);
+ uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
@@ -9088,13 +9073,8 @@
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
JUMP_TO_PREDICTED(LOAD_GLOBAL);
}
- builtins_keys = keys;
- assert(DK_IS_UNICODE(builtins_keys));
- }
- // _LOAD_GLOBAL_BUILTINS_FROM_KEYS
- {
- uint16_t index = read_u16(&this_instr[4].cache);
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
if (res_o == NULL) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
@@ -9109,8 +9089,7 @@
JUMP_TO_PREDICTED(LOAD_GLOBAL);
}
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
}
@@ -9136,13 +9115,16 @@
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
- PyDictKeysObject *globals_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
- // _GUARD_GLOBALS_VERSION_PUSH_KEYS
+ // _NOP
+ {
+ }
+ // _LOAD_GLOBAL_MODULE
{
uint16_t version = read_u16(&this_instr[2].cache);
+ uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
@@ -9155,14 +9137,9 @@
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
JUMP_TO_PREDICTED(LOAD_GLOBAL);
}
- globals_keys = keys;
- assert(DK_IS_UNICODE(globals_keys));
- }
- /* Skip 1 cache entry */
- // _LOAD_GLOBAL_MODULE_FROM_KEYS
- {
- uint16_t index = read_u16(&this_instr[4].cache);
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
+ assert(DK_IS_UNICODE(keys));
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
+ assert(index < DK_SIZE(keys));
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
if (res_o == NULL) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
@@ -9177,8 +9154,7 @@
JUMP_TO_PREDICTED(LOAD_GLOBAL);
}
#else
- Py_INCREF(res_o);
- res = PyStackRef_FromPyObjectSteal(res_o);
+ res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
}
diff --git a/Python/optimizer.c b/Python/optimizer.c
index d86d58d82e3..e05523451da 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -230,16 +230,18 @@ _PyUOpPrint(const _PyUOpInstruction *uop)
}
switch(uop->format) {
case UOP_FORMAT_TARGET:
- printf(" (%d, target=%d, operand=%#" PRIx64,
+ printf(" (%d, target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64,
uop->oparg,
uop->target,
- (uint64_t)uop->operand0);
+ (uint64_t)uop->operand0,
+ (uint64_t)uop->operand1);
break;
case UOP_FORMAT_JUMP:
- printf(" (%d, jump_target=%d, operand=%#" PRIx64,
+ printf(" (%d, jump_target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64,
uop->oparg,
uop->jump_target,
- (uint64_t)uop->operand0);
+ (uint64_t)uop->operand0,
+ (uint64_t)uop->operand1);
break;
default:
printf(" (%d, Unknown format)", uop->oparg);
@@ -682,7 +684,7 @@ translate_bytecode_to_trace(
// Add one to account for the actual opcode/oparg pair:
int offset = expansion->uops[i].offset + 1;
switch (expansion->uops[i].size) {
- case OPARG_FULL:
+ case OPARG_SIMPLE:
assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD);
break;
case OPARG_CACHE_1:
@@ -716,6 +718,21 @@ translate_bytecode_to_trace(
}
#endif
break;
+ case OPERAND1_1:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u16(&instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
+ case OPERAND1_2:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u32(&instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
+ case OPERAND1_4:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u64(&instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
default:
fprintf(stderr,
"opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n",
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 6c0aadb87e6..29a05088e62 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -93,26 +93,27 @@ type_watcher_callback(PyTypeObject* type)
}
static PyObject *
-convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj)
+convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop)
{
- assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE_FROM_KEYS);
+ assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE);
assert(PyDict_CheckExact(obj));
PyDictObject *dict = (PyDictObject *)obj;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- assert(inst->operand0 <= UINT16_MAX);
- if ((int)inst->operand0 >= dict->ma_keys->dk_nentries) {
+ int64_t index = inst->operand1;
+ assert(index <= UINT16_MAX);
+ if ((int)index >= dict->ma_keys->dk_nentries) {
return NULL;
}
- PyObject *res = entries[inst->operand0].me_value;
+ PyObject *res = entries[index].me_value;
if (res == NULL) {
return NULL;
}
if (_Py_IsImmortal(res)) {
- inst->opcode = _LOAD_CONST_INLINE_BORROW;
+ inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE_BORROW;
}
else {
- inst->opcode = _LOAD_CONST_INLINE;
+ inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE : _LOAD_CONST_INLINE;
}
if (inst->oparg & 1) {
assert(inst[1].opcode == _PUSH_NULL_CONDITIONAL);
@@ -198,21 +199,18 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
_PyUOpInstruction *inst = &buffer[pc];
int opcode = inst->opcode;
switch(opcode) {
- case _GUARD_BUILTINS_VERSION_PUSH_KEYS:
- if (incorrect_keys(inst, builtins)) {
+ case _GUARD_GLOBALS_VERSION:
+ if (incorrect_keys(inst, globals)) {
OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
- if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
- continue;
- }
- if (!check_next_uop(buffer, buffer_size, pc,
- _LOAD_GLOBAL_BUILTINS_FROM_KEYS)) {
+ if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
continue;
}
- if ((builtins_watched & 1) == 0) {
- PyDict_Watch(BUILTINS_WATCHER_ID, builtins);
- builtins_watched |= 1;
+ if ((globals_watched & 1) == 0) {
+ PyDict_Watch(GLOBALS_WATCHER_ID, globals);
+ _Py_BloomFilter_Add(dependencies, globals);
+ globals_watched |= 1;
}
if (function_checked & 1) {
buffer[pc].opcode = NOP;
@@ -222,24 +220,29 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
buffer[pc].operand0 = function_version;
function_checked |= 1;
}
- // We're no longer pushing the builtins keys; rewrite the
- // instruction that consumed the keys to load them from the
- // frame.
- buffer[pc + 1].opcode = _LOAD_GLOBAL_BUILTINS;
break;
- case _GUARD_GLOBALS_VERSION:
- case _GUARD_GLOBALS_VERSION_PUSH_KEYS:
- if (incorrect_keys(inst, globals)) {
+ case _LOAD_GLOBAL_BUILTINS:
+ if (incorrect_keys(inst, builtins)) {
OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
- uint64_t watched_mutations = get_mutations(globals);
- if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
+ if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
continue;
}
- if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS &&
- !check_next_uop(buffer, buffer_size, pc,
- _LOAD_GLOBAL_MODULE_FROM_KEYS)) {
+ if ((builtins_watched & 1) == 0) {
+ PyDict_Watch(BUILTINS_WATCHER_ID, builtins);
+ builtins_watched |= 1;
+ }
+ if (function_checked & globals_watched & 1) {
+ convert_global_to_const(inst, builtins, false);
+ }
+ break;
+ case _LOAD_GLOBAL_MODULE:
+ if (incorrect_keys(inst, globals)) {
+ OPT_STAT_INC(remove_globals_incorrect_keys);
+ return 0;
+ }
+ if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
continue;
}
if ((globals_watched & 1) == 0) {
@@ -247,29 +250,13 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
_Py_BloomFilter_Add(dependencies, globals);
globals_watched |= 1;
}
- if (function_checked & 1) {
- buffer[pc].opcode = NOP;
- }
- else {
- buffer[pc].opcode = _CHECK_FUNCTION;
- buffer[pc].operand0 = function_version;
+ if ((function_checked & 1) == 0 && buffer[pc-1].opcode == _NOP) {
+ buffer[pc-1].opcode = _CHECK_FUNCTION;
+ buffer[pc-1].operand0 = function_version;
function_checked |= 1;
}
- if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) {
- // We're no longer pushing the globals keys; rewrite the
- // instruction that consumed the keys to load them from the
- // frame.
- buffer[pc + 1].opcode = _LOAD_GLOBAL_MODULE;
- }
- break;
- case _LOAD_GLOBAL_BUILTINS:
- if (function_checked & globals_watched & builtins_watched & 1) {
- convert_global_to_const(inst, builtins);
- }
- break;
- case _LOAD_GLOBAL_MODULE:
- if (function_checked & globals_watched & 1) {
- convert_global_to_const(inst, globals);
+ if (function_checked & 1) {
+ convert_global_to_const(inst, globals, false);
}
break;
case _PUSH_FRAME:
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 41eb59c931a..f3625a1492c 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -492,6 +492,14 @@ dummy_func(void) {
value = sym_new_const(ctx, ptr);
}
+ op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) {
+ value = sym_new_const(ctx, ptr);
+ }
+
+ op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) {
+ value = sym_new_const(ctx, ptr);
+ }
+
op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) {
assert(oparg > 0);
top = bottom;
@@ -509,22 +517,27 @@ dummy_func(void) {
(void)offset;
}
- op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) {
+ op(_LOAD_ATTR_MODULE, (dict_version/2, owner, index/1 -- attr)) {
(void)dict_version;
- mod_keys = sym_new_not_null(ctx);
+ (void)index;
+ attr = NULL;
if (sym_is_const(owner)) {
- PyObject *cnst = sym_get_const(owner);
- if (PyModule_CheckExact(cnst)) {
- PyModuleObject *mod = (PyModuleObject *)cnst;
+ PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
+ if (PyModule_CheckExact(mod)) {
PyObject *dict = mod->md_dict;
uint64_t watched_mutations = get_mutations(dict);
if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
PyDict_Watch(GLOBALS_WATCHER_ID, dict);
_Py_BloomFilter_Add(dependencies, dict);
- this_instr->opcode = _NOP;
+ PyObject *res = convert_global_to_const(this_instr, dict, true);
+ attr = sym_new_const(ctx, res);
}
}
}
+ if (attr == NULL) {
+ /* No conversion made. We don't know what `attr` is. */
+ attr = sym_new_not_null(ctx);
+ }
}
op (_PUSH_NULL_CONDITIONAL, ( -- null if (oparg & 1))) {
@@ -541,35 +554,7 @@ dummy_func(void) {
}
}
- op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) {
- (void)index;
- attr = NULL;
- if (this_instr[-1].opcode == _NOP) {
- // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
- assert(sym_is_const(owner));
- PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
- assert(PyModule_CheckExact(mod));
- PyObject *dict = mod->md_dict;
- PyObject *res = convert_global_to_const(this_instr, dict);
- if (res != NULL) {
- this_instr[-1].opcode = _POP_TOP;
- attr = sym_new_const(ctx, res);
- }
- else {
- this_instr->opcode = _LOAD_ATTR_MODULE;
- }
- }
- if (attr == NULL) {
- /* No conversion made. We don't know what `attr` is. */
- attr = sym_new_not_null(ctx);
- }
- }
-
- op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) {
- dict = sym_new_not_null(ctx);
- }
-
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) {
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) {
attr = sym_new_not_null(ctx);
(void)hint;
}
@@ -888,16 +873,6 @@ dummy_func(void) {
ctx->done = true;
}
- op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version/1 -- globals_keys)) {
- globals_keys = sym_new_unknown(ctx);
- (void)version;
- }
-
- op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version/1 -- builtins_keys)) {
- builtins_keys = sym_new_unknown(ctx);
- (void)version;
- }
-
op(_REPLACE_WITH_TRUE, (value -- res)) {
res = sym_new_const(ctx, Py_True);
}
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 51d0fa63e64..0372870b94e 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -932,39 +932,21 @@
break;
}
- case _GUARD_GLOBALS_VERSION_PUSH_KEYS: {
- JitOptSymbol *globals_keys;
- uint16_t version = (uint16_t)this_instr->operand0;
- globals_keys = sym_new_unknown(ctx);
- (void)version;
- stack_pointer[0] = globals_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _GUARD_BUILTINS_VERSION_PUSH_KEYS: {
- JitOptSymbol *builtins_keys;
- uint16_t version = (uint16_t)this_instr->operand0;
- builtins_keys = sym_new_unknown(ctx);
- (void)version;
- stack_pointer[0] = builtins_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
+ case _LOAD_GLOBAL_MODULE: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
- case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
+ case _LOAD_GLOBAL_BUILTINS: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
@@ -1168,80 +1150,34 @@
break;
}
- case _CHECK_ATTR_MODULE_PUSH_KEYS: {
+ case _LOAD_ATTR_MODULE: {
JitOptSymbol *owner;
- JitOptSymbol *mod_keys;
+ JitOptSymbol *attr;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)this_instr->operand0;
+ uint16_t index = (uint16_t)this_instr->operand0;
(void)dict_version;
- mod_keys = sym_new_not_null(ctx);
+ (void)index;
+ attr = NULL;
if (sym_is_const(owner)) {
- PyObject *cnst = sym_get_const(owner);
- if (PyModule_CheckExact(cnst)) {
- PyModuleObject *mod = (PyModuleObject *)cnst;
+ PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
+ if (PyModule_CheckExact(mod)) {
PyObject *dict = mod->md_dict;
- stack_pointer[0] = mod_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
+ stack_pointer[-1] = attr;
uint64_t watched_mutations = get_mutations(dict);
if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
PyDict_Watch(GLOBALS_WATCHER_ID, dict);
_Py_BloomFilter_Add(dependencies, dict);
- this_instr->opcode = _NOP;
+ PyObject *res = convert_global_to_const(this_instr, dict, true);
+ attr = sym_new_const(ctx, res);
}
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
}
}
- stack_pointer[0] = mod_keys;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_ATTR_MODULE_FROM_KEYS: {
- JitOptSymbol *owner;
- JitOptSymbol *attr;
- owner = stack_pointer[-2];
- uint16_t index = (uint16_t)this_instr->operand0;
- (void)index;
- attr = NULL;
- if (this_instr[-1].opcode == _NOP) {
- // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
- assert(sym_is_const(owner));
- PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
- assert(PyModule_CheckExact(mod));
- PyObject *dict = mod->md_dict;
- stack_pointer[-2] = attr;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- PyObject *res = convert_global_to_const(this_instr, dict);
- if (res != NULL) {
- this_instr[-1].opcode = _POP_TOP;
- attr = sym_new_const(ctx, res);
- }
- else {
- this_instr->opcode = _LOAD_ATTR_MODULE;
- }
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- }
if (attr == NULL) {
/* No conversion made. We don't know what `attr` is. */
attr = sym_new_not_null(ctx);
}
- stack_pointer[-2] = attr;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _CHECK_ATTR_WITH_HINT: {
- JitOptSymbol *dict;
- dict = sym_new_not_null(ctx);
- stack_pointer[0] = dict;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
+ stack_pointer[-1] = attr;
break;
}
@@ -1250,9 +1186,7 @@
uint16_t hint = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
(void)hint;
- stack_pointer[-2] = attr;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
+ stack_pointer[-1] = attr;
break;
}
@@ -2420,6 +2354,14 @@
break;
}
+ case _POP_TOP_LOAD_CONST_INLINE: {
+ JitOptSymbol *value;
+ PyObject *ptr = (PyObject *)this_instr->operand0;
+ value = sym_new_const(ctx, ptr);
+ stack_pointer[-1] = value;
+ break;
+ }
+
case _LOAD_CONST_INLINE_BORROW: {
JitOptSymbol *value;
PyObject *ptr = (PyObject *)this_instr->operand0;
@@ -2432,7 +2374,8 @@
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
JitOptSymbol *value;
- value = sym_new_not_null(ctx);
+ PyObject *ptr = (PyObject *)this_instr->operand0;
+ value = sym_new_const(ctx, ptr);
stack_pointer[-1] = value;
break;
}
@@ -2441,31 +2384,6 @@
break;
}
- case _LOAD_GLOBAL_MODULE: {
- JitOptSymbol *res;
- res = sym_new_not_null(ctx);
- stack_pointer[0] = res;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_GLOBAL_BUILTINS: {
- JitOptSymbol *res;
- res = sym_new_not_null(ctx);
- stack_pointer[0] = res;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
- case _LOAD_ATTR_MODULE: {
- JitOptSymbol *attr;
- attr = sym_new_not_null(ctx);
- stack_pointer[-1] = attr;
- break;
- }
-
case _START_EXECUTOR: {
break;
}