diff options
Diffstat (limited to 'Objects/dictobject.c')
| -rw-r--r-- | Objects/dictobject.c | 464 |
1 files changed, 237 insertions, 227 deletions
diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 536746ca41e..58a3d979339 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -361,6 +361,10 @@ static int dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, PyObject **result, int incref_result); +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); +#endif + #include "clinic/dictobject.c.h" @@ -624,8 +628,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { @@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) PyMem_Free(keys); } +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; +} + +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = DICT_VALUES_SIZE(values); + assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(((char *)values)-prefix_size); + _PyMem_FreeDelayed(values); return; } #endif - PyMem_Free(((char *)values)-prefix_size); + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ -#ifdef Py_GIL_DISABLED - // dk_usable will decrease for each instance that is created and each - // value that is added. dk_nentries will increase for each value that - // is added. We want to always return the right value or larger. - // We therefore increase dk_nentries first and we decrease dk_usable - // second, and conversely here we read dk_usable first and dk_entries - // second (to avoid the case where we read entries before the increment - // and read usable after the decrement) - return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + - _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); -#else - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -#endif -} - /* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) @@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb if (values == NULL) goto read_failed; - uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); if (ix >= (Py_ssize_t)capacity) goto read_failed; @@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op) return; mp = (PyDictObject *) op; + ASSERT_CONSISTENT(mp); numentries = mp->ma_keys->dk_nentries; if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < numentries; i++) { @@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, set_keys(mp, newkeys); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + oldvalues->valid = 0; + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); + } } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { @@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } static int @@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op) mp->ma_version_tag = new_version; /* ...then clear the keys and values */ if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (!oldvalues->embedded) { + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) + Py_CLEAR(oldvalues->values[i]); + free_values(oldvalues, IS_DICT_SHARED(mp)); + } dictkeys_decref(interp, oldkeys, false); } else { @@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values, false); dictkeys_decref(interp, keys, false); } else if (keys != NULL) { @@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe PyDictKeysObject *okeys = other->ma_keys; // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); @@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (_PyDict_HasSplitTable(mp)) { - free_values(mp->ma_values, IS_DICT_SHARED(mp)); - mp->ma_values = NULL; - } - mp->ma_used = other->ma_used; mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); @@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } +/* Copies the values, but does not change the reference + * counts of the objects in the array. */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + PyErr_NoMemory(); + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + static PyObject * copy_lock_held(PyObject *o) { @@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; @@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, return 0; } -static Py_ssize_t -load_values_used_size(PyDictValues *values) -{ - return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); -} - static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) @@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, goto concurrent_modification; } - Py_ssize_t used = load_values_used_size(values); + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); if (i >= used) { goto fail; } @@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void) return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); #ifdef Py_GIL_DISABLED Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); if (usable > 1) { @@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } #endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; -} - -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { @@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MakeDictFromInstanceAttributes(PyObject *obj) { + PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); return make_dict_from_instance_attributes(interp, keys, values); } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) -{ - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; - } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || - !has_unique_reference((PyObject *)dict)) - { - return false; - } - ensure_shared_on_resize(dict); - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; -} int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { Py_hash_t hash = unicode_get_hash(name); @@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } #endif } + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (ix == DKIX_EMPTY) { - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); if (dict == NULL) { - return -1; + dict = make_dict_from_instance_attributes( + interp, keys, values); + if (dict == NULL) { + return -1; + } + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } - _PyObject_DictOrValuesPointer(obj)->dict = dict; if (value == NULL) { - return PyDict_DelItem(dict, name); + return PyDict_DelItem((PyObject *)dict, name); } else { - return PyDict_SetItem(dict, name, value); + return PyDict_SetItem((PyObject *)dict, name, value); } } PyObject *old_value = values->values[ix]; @@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return -1; } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used++; + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used--; + } } Py_DECREF(old_value); } @@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); int size = ((uint8_t *)values)[-2]; int count = 0; PyDictKeysObject *keys = CACHED_KEYS(tp); @@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) CHECK(size == count); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; @@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (values->values[i] != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_ManagedDictPointer(obj)->dict; + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = managed_dict->dict; } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; @@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) return ((PyDictObject *)dict)->ma_used == 0; } -void -_PyObject_FreeInstanceAttributes(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; - } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); - } - free_values(values, IS_DICT_SHARED((PyDictObject*)self)); -} - int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { @@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; } } - else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); - } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } void PyObject_ClearManagedDict(PyObject *obj) { + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; - } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); - } - dorv_ptr->dict = NULL; - free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); - } - else { - PyObject *dict = dorv_ptr->dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (dict) { - dorv_ptr->dict = NULL; + _PyDict_DetachFromObject(dict, obj); + _PyObject_ManagedDictPointer(obj)->dict = NULL; Py_DECREF(dict); } + else { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } + values->valid = 0; + } + } + } + else { + Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict); + } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) { + return 0; + } + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + Py_BEGIN_CRITICAL_SECTION(mp); + mp->ma_values = copy_values(mp->ma_values); + _PyObject_InlineValues(obj)->valid = 0; + Py_END_CRITICAL_SECTION(); + if (mp->ma_values == NULL) { + return -1; } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + PyDictObject *dict = managed_dict->dict; + if (dict == NULL && + (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid + ) { + PyDictValues *values = _PyObject_InlineValues(obj); OBJECT_STAT_INC(dict_materialized_on_request); dict = make_dict_from_instance_attributes( interp, CACHED_KEYS(tp), values); if (dict != NULL) { - dorv_ptr->dict = dict; + managed_dict->dict = (PyDictObject *)dict; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = managed_dict->dict; if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; + dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); + managed_dict->dict = (PyDictObject *)dict; } } + return Py_XNewRef((PyObject *)dict); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; + PyObject *dict = *dictptr; if (dict == NULL) { PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { @@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) *dictptr = dict = PyDict_New(); } } + return Py_XNewRef(dict); } - return Py_XNewRef(dict); } int @@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, assert(dictptr != NULL); dict = *dictptr; if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) @@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif |