diff options
author | mpage <mpage@meta.com> | 2025-01-14 11:56:11 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-01-14 11:56:11 -0800 |
commit | b5ee0258bf5bb60a5a5a65c64717853e06b64808 (patch) | |
tree | f5f7dd71e3fbed86e867f454c742dd34040f7129 /Python | |
parent | 1c13c56a34fc4c4d8969f0b6dc93d5208a50d61b (diff) | |
download | cpython-b5ee0258bf5bb60a5a5a65c64717853e06b64808.tar.gz cpython-b5ee0258bf5bb60a5a5a65c64717853e06b64808.zip |
gh-115999: Specialize `LOAD_ATTR` for instance and class receivers in free-threaded builds (#128164)
Finish specialization for LOAD_ATTR in the free-threaded build by adding support for class and instance receivers.
Diffstat (limited to 'Python')
-rw-r--r-- | Python/bytecodes.c | 85 | ||||
-rw-r--r-- | Python/executor_cases.c.h | 141 | ||||
-rw-r--r-- | Python/generated_cases.c.h | 81 | ||||
-rw-r--r-- | Python/optimizer_bytecodes.c | 8 | ||||
-rw-r--r-- | Python/optimizer_cases.c.h | 19 | ||||
-rw-r--r-- | Python/specialize.c | 229 |
6 files changed, 378 insertions, 185 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cec530fefff..a906ded3656 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2190,18 +2190,23 @@ dummy_func( PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); } split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -2227,9 +2232,8 @@ dummy_func( assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - DEAD(mod_keys); // Clear mod_keys from stack in case we need to deopt - POP_DEAD_INPUTS(); + POP_INPUT(mod_keys); DEOPT_IF(attr_o == NULL); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -2251,30 +2255,50 @@ dummy_func( _LOAD_ATTR_MODULE_FROM_KEYS + unused/5; - op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - EXIT_IF(dict == NULL); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + EXIT_IF(dict_o == NULL); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) { PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + POP_INPUT(dict); + DEOPT_IF(true); + } - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + DEAD(dict); null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2289,12 +2313,17 @@ dummy_func( split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -2309,7 +2338,7 @@ dummy_func( EXIT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version); } split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { @@ -2363,7 +2392,7 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -3281,13 +3310,15 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); } split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -3357,7 +3388,7 @@ dummy_func( op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1aa80f398d7..cda01bb768c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2652,7 +2652,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2668,15 +2668,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2691,15 +2699,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2778,55 +2794,88 @@ case _CHECK_ATTR_WITH_HINT: { _PyStackRef owner; + PyDictObject *dict; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + if (dict_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; + stack_pointer[0].bits = (uintptr_t)dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + PyDictObject *dict; _PyStackRef owner; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + dict = (PyDictObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (!LOCK_OBJECT(dict)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } if (hint >= (size_t)dict->ma_keys->dk_nentries) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } attr_o = ep->me_value; if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2839,15 +2888,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2861,15 +2918,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2890,7 +2955,7 @@ JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); - if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3924,7 +3989,8 @@ owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3937,7 +4003,8 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4022,7 +4089,7 @@ owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ if (dict != NULL) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 810beb61d0d..81408380d6b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5345,7 +5345,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5380,7 +5380,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5421,7 +5421,7 @@ DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -5463,19 +5463,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5507,7 +5512,7 @@ { uint16_t dictoffset = read_u16(&this_instr[4].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } @@ -5586,14 +5591,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5716,14 +5723,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5824,12 +5833,17 @@ { uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed, LOAD_ATTR); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5846,6 +5860,7 @@ INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + PyDictObject *dict; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -5861,26 +5876,40 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict_o == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + if (!LOCK_OBJECT(dict)) { + DEOPT_IF(true, LOAD_ATTR); + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 788adecca8a..4d96ada5acf 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -582,11 +582,17 @@ dummy_func(void) { } } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { + dict = sym_new_not_null(ctx); + (void)owner; + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; + (void)dict; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1a7cc6becfe..aff4493fdc4 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1250,22 +1250,33 @@ } case _CHECK_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *dict; + owner = stack_pointer[-1]; + dict = sym_new_not_null(ctx); + (void)owner; + stack_pointer[0] = dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *dict; _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; + dict = stack_pointer[-1]; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + (void)dict; + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index 897005c4f10..8d9f19c8895 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() @@ -537,6 +538,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 #define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34 #define SPEC_FAIL_ATTR_SPLIT_DICT 35 +#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36 /* Binary subscr and store subscr */ @@ -729,11 +731,8 @@ unspecialize(_Py_CODEUNIT *instr) } static int function_kind(PyCodeObject *code); -#ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -static uint32_t type_get_version(PyTypeObject *t, int opcode); -#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -879,10 +878,11 @@ descriptor_is_class(PyObject *descriptor, PyObject *name) (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); } -#ifndef Py_GIL_DISABLED static DescriptorClassification -analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) { bool has_getattr = false; + bool have_ga_version = false; + unsigned int ga_version; getattrofunc getattro_slot = type->tp_getattro; if (getattro_slot == PyObject_GenericGetAttr) { /* Normal attribute lookup; */ @@ -892,24 +892,27 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { getattro_slot == _Py_slot_tp_getattro) { /* One or both of __getattribute__ or __getattr__ may have been overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyObject *getattribute = _PyType_LookupRefAndVersion(type, + &_Py_ID(__getattribute__), &ga_version); + have_ga_version = true; PyInterpreterState *interp = _PyInterpreterState_GET(); bool has_custom_getattribute = getattribute != NULL && getattribute != interp->callable_cache.object__getattribute__; - PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; - Py_XDECREF(getattr); if (has_custom_getattribute) { if (getattro_slot == _Py_slot_tp_getattro && !has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; + *tp_version = ga_version; return GETATTRIBUTE_IS_PYTHON_FUNCTION; } /* Potentially both __getattr__ and __getattribute__ are set. Too complicated */ Py_DECREF(getattribute); *descr = NULL; + *tp_version = ga_version; return GETSET_OVERRIDDEN; } /* Potentially has __getattr__ but no custom __getattribute__. @@ -923,16 +926,18 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { } else { *descr = NULL; + *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag); return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_LookupRef(type, name); + unsigned int descr_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version); *descr = descriptor; + *tp_version = have_ga_version ? ga_version : descr_version; if (descriptor_is_class(descriptor, name)) { return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } -#endif //!Py_GIL_DISABLED static DescriptorClassification analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) @@ -952,12 +957,13 @@ analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, u static int specialize_dict_access_inline( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int values_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner); Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); assert (index != DKIX_ERROR); if (index == DKIX_EMPTY) { @@ -965,6 +971,7 @@ specialize_dict_access_inline( return 0; } assert(index >= 0); + assert(_PyObject_InlineValues(owner)->valid); char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; Py_ssize_t offset = value_addr - (char *)owner; if (offset != (uint16_t)offset) { @@ -980,10 +987,13 @@ specialize_dict_access_inline( static int specialize_dict_access_hint( PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int hint_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + // We found an instance with a __dict__. if (_PyDict_HasSplitTable(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); @@ -1027,7 +1037,7 @@ specialize_dict_access( PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL) { // managed dict, not materialized, inline values valid - res = specialize_dict_access_inline(owner, instr, type, kind, name, + res = specialize_dict_access_inline(owner, instr, type, name, tp_version, base_op, values_op); } else { @@ -1047,16 +1057,19 @@ specialize_dict_access( int res; Py_BEGIN_CRITICAL_SECTION(dict); // materialized managed dict - res = specialize_dict_access_hint(dict, instr, type, kind, name, + res = specialize_dict_access_hint(dict, instr, type, name, tp_version, base_op, hint_op); Py_END_CRITICAL_SECTION(); return res; } } -#ifndef Py_GIL_DISABLED -static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, - PyObject* descr, DescriptorClassification kind, bool is_method); +static int +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); /* Returns true if instances of obj's class are @@ -1065,7 +1078,7 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb * For other objects, we check their actual dictionary. */ static bool -instance_has_key(PyObject *obj, PyObject* name) +instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) { PyTypeObject *cls = Py_TYPE(obj); if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { @@ -1073,36 +1086,38 @@ instance_has_key(PyObject *obj, PyObject* name) } if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys; - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t index = + _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version); return index >= 0; } PyDictObject *dict = _PyObject_GetManagedDict(obj); if (dict == NULL || !PyDict_CheckExact(dict)) { return false; } + bool result; + Py_BEGIN_CRITICAL_SECTION(dict); if (dict->ma_values) { - return false; + result = false; } - Py_ssize_t index = _PyDict_LookupIndex(dict, name); - if (index < 0) { - return false; + else { + result = (_PyDict_LookupIndex(dict, name) >= 0); } - return true; + Py_END_CRITICAL_SECTION(); + return result; } static int -specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, + bool shadow, uint32_t shared_keys_version, + DescriptorClassification kind, PyObject *descr, unsigned int tp_version) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); - bool shadow = instance_has_key(owner, name); - PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); - if (type_get_version(type, LOAD_ATTR) == 0) { + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1112,9 +1127,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - int oparg = instr->op.arg; if (oparg & 1) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, true, + shared_keys_version)) { return 0; } else { @@ -1140,7 +1156,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (!function_check_args(fget, 1, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1149,8 +1165,14 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } - assert(type->tp_version_tag != 0); - write_u32(lm_cache->type_version, type->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(fget)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif + assert(tp_version != 0); + write_u32(lm_cache->type_version, tp_version); /* borrowed */ write_obj(lm_cache->descr, fget); specialize(instr, LOAD_ATTR_PROPERTY); @@ -1176,7 +1198,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1185,7 +1207,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1200,13 +1222,18 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { + #ifndef Py_GIL_DISABLED + // In free-threaded builds it's possible for tp_getattro to change + // after the call to analyze_descriptor. That is fine: the version + // guard will fail. assert(type->tp_getattro == _Py_slot_tp_getattro); + #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1219,10 +1246,16 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif write_u32(lm_cache->keys_version, version); /* borrowed */ write_obj(lm_cache->descr, descr); - write_u32(lm_cache->type_version, type->tp_version_tag); + write_u32(lm_cache->type_version, tp_version); specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); return 0; } @@ -1237,8 +1270,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - if ((instr->op.arg & 1) == 0) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) { + if ((oparg & 1) == 0) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, false, + shared_keys_version)) { return 0; } } @@ -1252,14 +1287,28 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } return -1; } -#endif // Py_GIL_DISABLED + +static int +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +{ + // 0 is not a valid version + uint32_t shared_keys_version = 0; + bool shadow = instance_has_key(owner, name, &shared_keys_version); + PyObject *descr = NULL; + unsigned int tp_version = 0; + PyTypeObject *type = Py_TYPE(owner); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); + Py_XDECREF(descr); + return result; +} void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) @@ -1281,20 +1330,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_module_load_attr(owner, instr, name); } else if (PyType_Check(owner)) { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_class_load_attr(owner, instr, name); - #endif } else { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_instance_load_attr(owner, instr, name); - #endif } if (fail) { @@ -1402,8 +1441,6 @@ success: return; } -#ifndef Py_GIL_DISABLED - #ifdef Py_STATS static int load_attr_fail_kind(DescriptorClassification kind) @@ -1452,8 +1489,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN); return -1; } - PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name); + unsigned int meta_version = 0; + PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version); DescriptorClassification metakind = classify_descriptor(metadescriptor, false); + Py_XDECREF(metadescriptor); switch (metakind) { case METHOD: case NON_DESCRIPTOR: @@ -1468,38 +1507,52 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor_load(cls, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - if (type_get_version(cls, LOAD_ATTR) == 0) { + unsigned int tp_version = 0; + kind = analyze_descriptor_load(cls, name, &descr, &tp_version); + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } bool metaclass_check = false; if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { metaclass_check = true; - if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) { + if (meta_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } } switch (kind) { case METHOD: case NON_DESCRIPTOR: - write_u32(cache->type_version, cls->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + Py_XDECREF(descr); + return -1; + } + #endif + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); if (metaclass_check) { - write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag); + write_u32(cache->keys_version, meta_version); specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); } else { specialize(instr, LOAD_ATTR_CLASS); } + Py_XDECREF(descr); return 0; #ifdef Py_STATS case ABSENT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + Py_XDECREF(descr); return -1; #endif default: SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind)); + Py_XDECREF(descr); return -1; } } @@ -1508,29 +1561,41 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. static int -specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, -PyObject *descr, DescriptorClassification kind, bool is_method) +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version) { _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - assert(_PyDictKeys_StringLookup(keys, name) < 0); - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), keys); - if (keys_version == 0) { + + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return 0; + } + #endif + + unsigned long tp_flags = PyType_GetFlags(owner_cls); + if (tp_flags & Py_TPFLAGS_INLINE_VALUES) { + #ifndef Py_GIL_DISABLED + assert(_PyDictKeys_StringLookup( + ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); + #endif + if (shared_keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } - write_u32(cache->keys_version, keys_version); + write_u32(cache->keys_version, shared_keys_version); specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + if (tp_flags & Py_TPFLAGS_MANAGED_DICT) { dictoffset = MANAGED_DICT_OFFSET; } else { @@ -1576,13 +1641,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method) * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ - write_u32(cache->type_version, owner_cls->tp_version_tag); + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); return 1; } -#endif // Py_GIL_DISABLED - static void specialize_load_global_lock_held( @@ -1729,7 +1792,6 @@ function_kind(PyCodeObject *code) { return SIMPLE_FUNCTION; } -#ifndef Py_GIL_DISABLED /* Returning false indicates a failure. */ static bool function_check_args(PyObject *o, int expected_argcount, int opcode) @@ -1763,19 +1825,6 @@ function_get_version(PyObject *o, int opcode) return version; } -/* Returning 0 indicates a failure. */ -static uint32_t -type_get_version(PyTypeObject *t, int opcode) -{ - uint32_t version = t->tp_version_tag; - if (version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); - return 0; - } - return version; -} -#endif // Py_GIL_DISABLED - void _Py_Specialize_BinarySubscr( _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) |