aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Python
diff options
context:
space:
mode:
authormpage <mpage@meta.com>2025-01-14 11:56:11 -0800
committerGitHub <noreply@github.com>2025-01-14 11:56:11 -0800
commitb5ee0258bf5bb60a5a5a65c64717853e06b64808 (patch)
treef5f7dd71e3fbed86e867f454c742dd34040f7129 /Python
parent1c13c56a34fc4c4d8969f0b6dc93d5208a50d61b (diff)
downloadcpython-b5ee0258bf5bb60a5a5a65c64717853e06b64808.tar.gz
cpython-b5ee0258bf5bb60a5a5a65c64717853e06b64808.zip
gh-115999: Specialize `LOAD_ATTR` for instance and class receivers in free-threaded builds (#128164)
Finish specialization for LOAD_ATTR in the free-threaded build by adding support for class and instance receivers.
Diffstat (limited to 'Python')
-rw-r--r--Python/bytecodes.c85
-rw-r--r--Python/executor_cases.c.h141
-rw-r--r--Python/generated_cases.c.h81
-rw-r--r--Python/optimizer_bytecodes.c8
-rw-r--r--Python/optimizer_cases.c.h19
-rw-r--r--Python/specialize.c229
6 files changed, 378 insertions, 185 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index cec530fefff..a906ded3656 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2190,18 +2190,23 @@ dummy_func(
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid));
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
DECREF_INPUTS();
}
@@ -2227,9 +2232,8 @@ dummy_func(
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- DEAD(mod_keys);
// Clear mod_keys from stack in case we need to deopt
- POP_DEAD_INPUTS();
+ POP_INPUT(mod_keys);
DEOPT_IF(attr_o == NULL);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
@@ -2251,30 +2255,50 @@ dummy_func(
_LOAD_ATTR_MODULE_FROM_KEYS +
unused/5;
- op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- EXIT_IF(dict == NULL);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ EXIT_IF(dict_o == NULL);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) {
PyObject *attr_o;
+ if (!LOCK_OBJECT(dict)) {
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- DEOPT_IF(ep->me_key != name);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
+ DEAD(dict);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@@ -2289,12 +2313,17 @@ dummy_func(
split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ DEOPT_IF(!increfed);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
DECREF_INPUTS();
}
@@ -2309,7 +2338,7 @@ dummy_func(
EXIT_IF(!PyType_Check(owner_o));
assert(type_version != 0);
- EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version);
+ EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version);
}
split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) {
@@ -2363,7 +2392,7 @@ dummy_func(
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
@@ -3281,13 +3310,15 @@ dummy_func(
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid));
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version);
}
split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) {
@@ -3357,7 +3388,7 @@ dummy_func(
op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
}
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 1aa80f398d7..cda01bb768c 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -2652,7 +2652,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -2668,15 +2668,23 @@
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
@@ -2691,15 +2699,23 @@
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
@@ -2778,55 +2794,88 @@
case _CHECK_ATTR_WITH_HINT: {
_PyStackRef owner;
+ PyDictObject *dict;
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- if (dict == NULL) {
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ if (dict_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(PyDict_CheckExact((PyObject *)dict));
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
+ stack_pointer[0].bits = (uintptr_t)dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ PyDictObject *dict;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
- owner = stack_pointer[-1];
+ dict = (PyDictObject *)stack_pointer[-1].bits;
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ if (!LOCK_OBJECT(dict)) {
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- if (!DK_IS_UNICODE(dict->ma_keys)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
attr_o = ep->me_value;
if (attr_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@@ -2839,15 +2888,23 @@
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
@@ -2861,15 +2918,23 @@
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
@@ -2890,7 +2955,7 @@
JUMP_TO_JUMP_TARGET();
}
assert(type_version != 0);
- if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) {
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -3924,7 +3989,8 @@
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -3937,7 +4003,8 @@
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- if (owner_heap_type->ht_cached_keys->dk_version != keys_version) {
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4022,7 +4089,7 @@
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
if (dict != NULL) {
UOP_STAT_INC(uopcode, miss);
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 810beb61d0d..81408380d6b 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5345,7 +5345,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
@@ -5380,7 +5380,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_TYPE_VERSION
{
@@ -5421,7 +5421,7 @@
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
@@ -5463,19 +5463,24 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t offset = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
@@ -5507,7 +5512,7 @@
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
@@ -5586,14 +5591,16 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
@@ -5716,14 +5723,16 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
@@ -5824,12 +5833,17 @@
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ DEOPT_IF(!increfed, LOAD_ATTR);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
@@ -5846,6 +5860,7 @@
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
+ PyDictObject *dict;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
@@ -5861,26 +5876,40 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict == NULL, LOAD_ATTR);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ DEOPT_IF(dict_o == NULL, LOAD_ATTR);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
+ if (!LOCK_OBJECT(dict)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR);
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- DEOPT_IF(ep->me_key != name, LOAD_ATTR);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 788adecca8a..4d96ada5acf 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -582,11 +582,17 @@ dummy_func(void) {
}
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) {
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ }
+
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
+ (void)dict;
}
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 1a7cc6becfe..aff4493fdc4 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -1250,22 +1250,33 @@
}
case _CHECK_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *owner;
+ _Py_UopsSymbol *dict;
+ owner = stack_pointer[-1];
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ stack_pointer[0] = dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *dict;
_Py_UopsSymbol *owner;
_Py_UopsSymbol *attr;
_Py_UopsSymbol *null = NULL;
- owner = stack_pointer[-1];
+ dict = stack_pointer[-1];
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ (void)dict;
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
diff --git a/Python/specialize.c b/Python/specialize.c
index 897005c4f10..8d9f19c8895 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -3,6 +3,7 @@
#include "opcode.h"
#include "pycore_code.h"
+#include "pycore_critical_section.h"
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // DICT_KEYS_UNICODE
#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
@@ -537,6 +538,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts,
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34
#define SPEC_FAIL_ATTR_SPLIT_DICT 35
+#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36
/* Binary subscr and store subscr */
@@ -729,11 +731,8 @@ unspecialize(_Py_CODEUNIT *instr)
}
static int function_kind(PyCodeObject *code);
-#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
-static uint32_t type_get_version(PyTypeObject *t, int opcode);
-#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
@@ -879,10 +878,11 @@ descriptor_is_class(PyObject *descriptor, PyObject *name)
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
-#ifndef Py_GIL_DISABLED
static DescriptorClassification
-analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
+analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) {
bool has_getattr = false;
+ bool have_ga_version = false;
+ unsigned int ga_version;
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
@@ -892,24 +892,27 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
- PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__));
+ PyObject *getattribute = _PyType_LookupRefAndVersion(type,
+ &_Py_ID(__getattribute__), &ga_version);
+ have_ga_version = true;
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
- PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
+ PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
- Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
+ *tp_version = ga_version;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
+ *tp_version = ga_version;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
@@ -923,16 +926,18 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
}
else {
*descr = NULL;
+ *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);
return GETSET_OVERRIDDEN;
}
- PyObject *descriptor = _PyType_LookupRef(type, name);
+ unsigned int descr_version;
+ PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version);
*descr = descriptor;
+ *tp_version = have_ga_version ? ga_version : descr_version;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, has_getattr);
}
-#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
@@ -952,12 +957,13 @@ analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, u
static int
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
@@ -965,6 +971,7 @@ specialize_dict_access_inline(
return 0;
}
assert(index >= 0);
+ assert(_PyObject_InlineValues(owner)->valid);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
@@ -980,10 +987,13 @@ specialize_dict_access_inline(
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
+
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);
+
// We found an instance with a __dict__.
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
@@ -1027,7 +1037,7 @@ specialize_dict_access(
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
- res = specialize_dict_access_inline(owner, instr, type, kind, name,
+ res = specialize_dict_access_inline(owner, instr, type, name,
tp_version, base_op, values_op);
}
else {
@@ -1047,16 +1057,19 @@ specialize_dict_access(
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
- res = specialize_dict_access_hint(dict, instr, type, kind, name,
+ res = specialize_dict_access_hint(dict, instr, type, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
}
-#ifndef Py_GIL_DISABLED
-static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
- PyObject* descr, DescriptorClassification kind, bool is_method);
+static int
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version);
static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
/* Returns true if instances of obj's class are
@@ -1065,7 +1078,7 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb
* For other objects, we check their actual dictionary.
*/
static bool
-instance_has_key(PyObject *obj, PyObject* name)
+instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)
{
PyTypeObject *cls = Py_TYPE(obj);
if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
@@ -1073,36 +1086,38 @@ instance_has_key(PyObject *obj, PyObject* name)
}
if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys;
- Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
+ Py_ssize_t index =
+ _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version);
return index >= 0;
}
PyDictObject *dict = _PyObject_GetManagedDict(obj);
if (dict == NULL || !PyDict_CheckExact(dict)) {
return false;
}
+ bool result;
+ Py_BEGIN_CRITICAL_SECTION(dict);
if (dict->ma_values) {
- return false;
+ result = false;
}
- Py_ssize_t index = _PyDict_LookupIndex(dict, name);
- if (index < 0) {
- return false;
+ else {
+ result = (_PyDict_LookupIndex(dict, name) >= 0);
}
- return true;
+ Py_END_CRITICAL_SECTION();
+ return result;
}
static int
-specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
+do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
+ bool shadow, uint32_t shared_keys_version,
+ DescriptorClassification kind, PyObject *descr, unsigned int tp_version)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
- bool shadow = instance_has_key(owner, name);
- PyObject *descr = NULL;
- DescriptorClassification kind = analyze_descriptor_load(type, name, &descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
- if (type_get_version(type, LOAD_ATTR) == 0) {
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
+ uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@@ -1112,9 +1127,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
if (shadow) {
goto try_instance;
}
- int oparg = instr->op.arg;
if (oparg & 1) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, true,
+ shared_keys_version)) {
return 0;
}
else {
@@ -1140,7 +1156,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
if (!function_check_args(fget, 1, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
@@ -1149,8 +1165,14 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
- assert(type->tp_version_tag != 0);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(fget)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
+ assert(tp_version != 0);
+ write_u32(lm_cache->type_version, tp_version);
/* borrowed */
write_obj(lm_cache->descr, fget);
specialize(instr, LOAD_ATTR_PROPERTY);
@@ -1176,7 +1198,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
@@ -1185,7 +1207,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
Py_ssize_t offset = offsetof(PyObject, ob_type);
assert(offset == (uint16_t)offset);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
@@ -1200,13 +1222,18 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
return -1;
case GETATTRIBUTE_IS_PYTHON_FUNCTION:
{
+ #ifndef Py_GIL_DISABLED
+ // In free-threaded builds it's possible for tp_getattro to change
+ // after the call to analyze_descriptor. That is fine: the version
+ // guard will fail.
assert(type->tp_getattro == _Py_slot_tp_getattro);
+ #endif
assert(Py_IS_TYPE(descr, &PyFunction_Type));
_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
if (!function_check_args(descr, 2, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
@@ -1219,10 +1246,16 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
write_u32(lm_cache->keys_version, version);
/* borrowed */
write_obj(lm_cache->descr, descr);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ write_u32(lm_cache->type_version, tp_version);
specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
return 0;
}
@@ -1237,8 +1270,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
if (shadow) {
goto try_instance;
}
- if ((instr->op.arg & 1) == 0) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) {
+ if ((oparg & 1) == 0) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, false,
+ shared_keys_version)) {
return 0;
}
}
@@ -1252,14 +1287,28 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
}
Py_UNREACHABLE();
try_instance:
- if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag,
+ if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
return 0;
}
return -1;
}
-#endif // Py_GIL_DISABLED
+
+static int
+specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
+{
+ // 0 is not a valid version
+ uint32_t shared_keys_version = 0;
+ bool shadow = instance_has_key(owner, name, &shared_keys_version);
+ PyObject *descr = NULL;
+ unsigned int tp_version = 0;
+ PyTypeObject *type = Py_TYPE(owner);
+ DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);
+ int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);
+ Py_XDECREF(descr);
+ return result;
+}
void
_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
@@ -1281,20 +1330,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam
fail = specialize_module_load_attr(owner, instr, name);
}
else if (PyType_Check(owner)) {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_class_load_attr(owner, instr, name);
- #endif
}
else {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_instance_load_attr(owner, instr, name);
- #endif
}
if (fail) {
@@ -1402,8 +1441,6 @@ success:
return;
}
-#ifndef Py_GIL_DISABLED
-
#ifdef Py_STATS
static int
load_attr_fail_kind(DescriptorClassification kind)
@@ -1452,8 +1489,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN);
return -1;
}
- PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name);
+ unsigned int meta_version = 0;
+ PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version);
DescriptorClassification metakind = classify_descriptor(metadescriptor, false);
+ Py_XDECREF(metadescriptor);
switch (metakind) {
case METHOD:
case NON_DESCRIPTOR:
@@ -1468,38 +1507,52 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
- kind = analyze_descriptor_load(cls, name, &descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- if (type_get_version(cls, LOAD_ATTR) == 0) {
+ unsigned int tp_version = 0;
+ kind = analyze_descriptor_load(cls, name, &descr, &tp_version);
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
bool metaclass_check = false;
if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) {
metaclass_check = true;
- if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) {
+ if (meta_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
}
switch (kind) {
case METHOD:
case NON_DESCRIPTOR:
- write_u32(cache->type_version, cls->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ Py_XDECREF(descr);
+ return -1;
+ }
+ #endif
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
if (metaclass_check) {
- write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag);
+ write_u32(cache->keys_version, meta_version);
specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
}
else {
specialize(instr, LOAD_ATTR_CLASS);
}
+ Py_XDECREF(descr);
return 0;
#ifdef Py_STATS
case ABSENT:
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
+ Py_XDECREF(descr);
return -1;
#endif
default:
SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
+ Py_XDECREF(descr);
return -1;
}
}
@@ -1508,29 +1561,41 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
// can cause a significant drop in cache hits. A possible test is
// python.exe -m test_typing test_re test_dis test_zlib.
static int
-specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
-PyObject *descr, DescriptorClassification kind, bool is_method)
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version)
{
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
assert(descr != NULL);
assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));
- if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
- assert(_PyDictKeys_StringLookup(keys, name) < 0);
- uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
- _PyInterpreterState_GET(), keys);
- if (keys_version == 0) {
+
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return 0;
+ }
+ #endif
+
+ unsigned long tp_flags = PyType_GetFlags(owner_cls);
+ if (tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ #ifndef Py_GIL_DISABLED
+ assert(_PyDictKeys_StringLookup(
+ ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0);
+ #endif
+ if (shared_keys_version == 0) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return 0;
}
- write_u32(cache->keys_version, keys_version);
+ write_u32(cache->keys_version, shared_keys_version);
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {
Py_ssize_t dictoffset;
- if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ if (tp_flags & Py_TPFLAGS_MANAGED_DICT) {
dictoffset = MANAGED_DICT_OFFSET;
}
else {
@@ -1576,13 +1641,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
* PyType_Modified usages in typeobject.c). The MCACHE has been
* working since Python 2.6 and it's battle-tested.
*/
- write_u32(cache->type_version, owner_cls->tp_version_tag);
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
return 1;
}
-#endif // Py_GIL_DISABLED
-
static void
specialize_load_global_lock_held(
@@ -1729,7 +1792,6 @@ function_kind(PyCodeObject *code) {
return SIMPLE_FUNCTION;
}
-#ifndef Py_GIL_DISABLED
/* Returning false indicates a failure. */
static bool
function_check_args(PyObject *o, int expected_argcount, int opcode)
@@ -1763,19 +1825,6 @@ function_get_version(PyObject *o, int opcode)
return version;
}
-/* Returning 0 indicates a failure. */
-static uint32_t
-type_get_version(PyTypeObject *t, int opcode)
-{
- uint32_t version = t->tp_version_tag;
- if (version == 0) {
- SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
- return 0;
- }
- return version;
-}
-#endif // Py_GIL_DISABLED
-
void
_Py_Specialize_BinarySubscr(
_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)