aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Objects
diff options
context:
space:
mode:
Diffstat (limited to 'Objects')
-rw-r--r--Objects/dictobject.c464
-rw-r--r--Objects/object.c100
-rw-r--r--Objects/object_layout.md91
-rw-r--r--Objects/object_layout_312.gv1
-rw-r--r--Objects/object_layout_312.pngbin30688 -> 33040 bytes
-rw-r--r--Objects/object_layout_313.gv45
-rw-r--r--Objects/object_layout_313.pngbin0 -> 35055 bytes
-rw-r--r--Objects/object_layout_full_313.gv25
-rw-r--r--Objects/object_layout_full_313.pngbin0 -> 16983 bytes
-rw-r--r--Objects/typeobject.c84
10 files changed, 477 insertions, 333 deletions
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 536746ca41e..58a3d979339 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -361,6 +361,10 @@ static int
dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value,
PyObject **result, int incref_result);
+#ifndef NDEBUG
+static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
+#endif
+
#include "clinic/dictobject.c.h"
@@ -624,8 +628,9 @@ static inline int
get_index_from_order(PyDictObject *mp, Py_ssize_t i)
{
assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
- assert(i < (((char *)mp->ma_values)[-2]));
- return ((char *)mp->ma_values)[-3-i];
+ assert(i < mp->ma_values->size);
+ uint8_t *array = get_insertion_order_array(mp->ma_values);
+ return array[i];
}
#ifdef DEBUG_PYDICT
@@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
else {
CHECK(keys->dk_kind == DICT_KEYS_SPLIT);
CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
+ if (mp->ma_values->embedded) {
+ CHECK(mp->ma_values->embedded == 1);
+ CHECK(mp->ma_values->valid == 1);
+ }
}
if (check_content) {
@@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
PyMem_Free(keys);
}
+static size_t
+values_size_from_count(size_t count)
+{
+ assert(count >= 1);
+ size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *));
+ assert(suffix_size < 128);
+ assert(suffix_size % sizeof(PyObject *) == 0);
+ return (count + 1) * sizeof(PyObject *) + suffix_size;
+}
+
+#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
+
static inline PyDictValues*
new_values(size_t size)
{
- assert(size >= 1);
- size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *));
- assert(prefix_size < 256);
- size_t n = prefix_size + size * sizeof(PyObject *);
- uint8_t *mem = PyMem_Malloc(n);
- if (mem == NULL) {
+ size_t n = values_size_from_count(size);
+ PyDictValues *res = (PyDictValues *)PyMem_Malloc(n);
+ if (res == NULL) {
return NULL;
}
- assert(prefix_size % sizeof(PyObject *) == 0);
- mem[prefix_size-1] = (uint8_t)prefix_size;
- return (PyDictValues*)(mem + prefix_size);
+ res->embedded = 0;
+ res->size = 0;
+ assert(size < 256);
+ res->capacity = (uint8_t)size;
+ return res;
}
static inline void
free_values(PyDictValues *values, bool use_qsbr)
{
- int prefix_size = DICT_VALUES_SIZE(values);
+ assert(values->embedded == 0);
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
- _PyMem_FreeDelayed(((char *)values)-prefix_size);
+ _PyMem_FreeDelayed(values);
return;
}
#endif
- PyMem_Free(((char *)values)-prefix_size);
+ PyMem_Free(values);
}
/* Consumes a reference to the keys object */
@@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp,
return (PyObject *)mp;
}
-static inline size_t
-shared_keys_usable_size(PyDictKeysObject *keys)
-{
-#ifdef Py_GIL_DISABLED
- // dk_usable will decrease for each instance that is created and each
- // value that is added. dk_nentries will increase for each value that
- // is added. We want to always return the right value or larger.
- // We therefore increase dk_nentries first and we decrease dk_usable
- // second, and conversely here we read dk_usable first and dk_entries
- // second (to avoid the case where we read entries before the increment
- // and read usable after the decrement)
- return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
- _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
-#else
- return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
-#endif
-}
-
/* Consumes a reference to the keys object */
static PyObject *
new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
@@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
dictkeys_decref(interp, keys, false);
return PyErr_NoMemory();
}
- ((char *)values)[-2] = 0;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
@@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb
if (values == NULL)
goto read_failed;
- uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values));
+ uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity);
if (ix >= (Py_ssize_t)capacity)
goto read_failed;
@@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op)
return;
mp = (PyDictObject *) op;
+ ASSERT_CONSISTENT(mp);
numentries = mp->ma_keys->dk_nentries;
if (_PyDict_HasSplitTable(mp)) {
for (i = 0; i < numentries; i++) {
@@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
set_keys(mp, newkeys);
dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp));
set_values(mp, NULL);
- free_values(oldvalues, IS_DICT_SHARED(mp));
+ if (oldvalues->embedded) {
+ assert(oldvalues->embedded == 1);
+ assert(oldvalues->valid == 1);
+ oldvalues->valid = 0;
+ }
+ else {
+ free_values(oldvalues, IS_DICT_SHARED(mp));
+ }
}
else { // oldkeys is combined.
if (oldkeys->dk_kind == DICT_KEYS_GENERAL) {
@@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value,
static void
delete_index_from_values(PyDictValues *values, Py_ssize_t ix)
{
- uint8_t *size_ptr = ((uint8_t *)values)-2;
- int size = *size_ptr;
+ uint8_t *array = get_insertion_order_array(values);
+ int size = values->size;
+ assert(size <= values->capacity);
int i;
- for (i = 1; size_ptr[-i] != ix; i++) {
- assert(i <= size);
+ for (i = 0; array[i] != ix; i++) {
+ assert(i < size);
}
- assert(i <= size);
+ assert(i < size);
+ size--;
for (; i < size; i++) {
- size_ptr[-i] = size_ptr[-i-1];
+ array[i] = array[i+1];
}
- *size_ptr = size -1;
+ values->size = size;
}
static int
@@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op)
mp->ma_version_tag = new_version;
/* ...then clear the keys and values */
if (oldvalues != NULL) {
- n = oldkeys->dk_nentries;
- for (i = 0; i < n; i++)
- Py_CLEAR(oldvalues->values[i]);
- free_values(oldvalues, IS_DICT_SHARED(mp));
+ if (!oldvalues->embedded) {
+ n = oldkeys->dk_nentries;
+ for (i = 0; i < n; i++)
+ Py_CLEAR(oldvalues->values[i]);
+ free_values(oldvalues, IS_DICT_SHARED(mp));
+ }
dictkeys_decref(interp, oldkeys, false);
}
else {
@@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self)
PyObject_GC_UnTrack(mp);
Py_TRASHCAN_BEGIN(mp, dict_dealloc)
if (values != NULL) {
- for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
- Py_XDECREF(values->values[i]);
+ if (values->embedded == 0) {
+ for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
+ Py_XDECREF(values->values[i]);
+ }
+ free_values(values, false);
}
- free_values(values, false);
dictkeys_decref(interp, keys, false);
}
else if (keys != NULL) {
@@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
PyDictKeysObject *okeys = other->ma_keys;
// If other is clean, combined, and just allocated, just clone it.
- if (other->ma_values == NULL &&
- other->ma_used == okeys->dk_nentries &&
- (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
- USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) {
+ if (mp->ma_values == NULL &&
+ other->ma_values == NULL &&
+ other->ma_used == okeys->dk_nentries &&
+ (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
+ USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
+ ) {
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
PyDictKeysObject *keys = clone_combined_dict_keys(other);
@@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
ensure_shared_on_resize(mp);
dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp));
mp->ma_keys = keys;
- if (_PyDict_HasSplitTable(mp)) {
- free_values(mp->ma_values, IS_DICT_SHARED(mp));
- mp->ma_values = NULL;
- }
-
mp->ma_used = other->ma_used;
mp->ma_version_tag = new_version;
ASSERT_CONSISTENT(mp);
@@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self)
return PyDict_Copy((PyObject *)self);
}
+/* Copies the values, but does not change the reference
+ * counts of the objects in the array. */
+static PyDictValues *
+copy_values(PyDictValues *values)
+{
+ PyDictValues *newvalues = new_values(values->capacity);
+ if (newvalues == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ newvalues->size = values->size;
+ uint8_t *values_order = get_insertion_order_array(values);
+ uint8_t *new_values_order = get_insertion_order_array(newvalues);
+ memcpy(new_values_order, values_order, values->capacity);
+ for (int i = 0; i < values->capacity; i++) {
+ newvalues->values[i] = values->values[i];
+ }
+ assert(newvalues->embedded == 0);
+ return newvalues;
+}
+
static PyObject *
copy_lock_held(PyObject *o)
{
@@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o)
if (_PyDict_HasSplitTable(mp)) {
PyDictObject *split_copy;
- size_t size = shared_keys_usable_size(mp->ma_keys);
- PyDictValues *newvalues = new_values(size);
- if (newvalues == NULL)
+ PyDictValues *newvalues = copy_values(mp->ma_values);
+ if (newvalues == NULL) {
return PyErr_NoMemory();
+ }
split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (split_copy == NULL) {
free_values(newvalues, false);
return NULL;
}
- size_t prefix_size = ((uint8_t *)newvalues)[-1];
- memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1);
+ for (size_t i = 0; i < newvalues->capacity; i++) {
+ Py_XINCREF(newvalues->values[i]);
+ }
split_copy->ma_values = newvalues;
split_copy->ma_keys = mp->ma_keys;
split_copy->ma_used = mp->ma_used;
split_copy->ma_version_tag = DICT_NEXT_VERSION(interp);
dictkeys_incref(mp->ma_keys);
- for (size_t i = 0; i < size; i++) {
- PyObject *value = mp->ma_values->values[i];
- split_copy->ma_values->values[i] = Py_XNewRef(value);
- }
if (_PyObject_GC_IS_TRACKED(mp))
_PyObject_GC_TRACK(split_copy);
return (PyObject *)split_copy;
@@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg)
if (DK_IS_UNICODE(keys)) {
if (_PyDict_HasSplitTable(mp)) {
- for (i = 0; i < n; i++) {
- Py_VISIT(mp->ma_values->values[i]);
+ if (!mp->ma_values->embedded) {
+ for (i = 0; i < n; i++) {
+ Py_VISIT(mp->ma_values->values[i]);
+ }
}
}
else {
@@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc,
return 0;
}
-static Py_ssize_t
-load_values_used_size(PyDictValues *values)
-{
- return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values));
-}
-
static int
dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
PyObject **out_key, PyObject **out_value)
@@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
goto concurrent_modification;
}
- Py_ssize_t used = load_values_used_size(values);
+ Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size);
if (i >= used) {
goto fail;
}
@@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void)
return keys;
}
-#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
-
-int
+void
_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
{
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictKeysObject *keys = CACHED_KEYS(tp);
assert(keys != NULL);
+ OBJECT_STAT_INC(inline_values);
#ifdef Py_GIL_DISABLED
Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable);
if (usable > 1) {
@@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
}
#endif
size_t size = shared_keys_usable_size(keys);
- PyDictValues *values = new_values(size);
- if (values == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- assert(((uint8_t *)values)[-1] >= (size + 2));
- ((uint8_t *)values)[-2] = 0;
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ assert(size < 256);
+ values->capacity = (uint8_t)size;
+ values->size = 0;
+ values->embedded = 1;
+ values->valid = 1;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
- _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values);
- return 0;
-}
-
-int
-_PyObject_InitializeDict(PyObject *obj)
-{
- PyInterpreterState *interp = _PyInterpreterState_GET();
- PyTypeObject *tp = Py_TYPE(obj);
- if (tp->tp_dictoffset == 0) {
- return 0;
- }
- if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- OBJECT_STAT_INC(new_values);
- return _PyObject_InitInlineValues(obj, tp);
- }
- PyObject *dict;
- if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
- dictkeys_incref(CACHED_KEYS(tp));
- dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
- }
- else {
- dict = PyDict_New();
- }
- if (dict == NULL) {
- return -1;
- }
- PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
- *dictptr = dict;
- return 0;
+ _PyObject_ManagedDictPointer(obj)->dict = NULL;
}
-static PyObject *
+static PyDictObject *
make_dict_from_instance_attributes(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues *values)
{
@@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp,
track += _PyObject_GC_MAY_BE_TRACKED(val);
}
}
- PyObject *res = new_dict(interp, keys, values, used, 0);
+ PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0);
if (track && res) {
_PyObject_GC_TRACK(res);
}
return res;
}
-PyObject *
-_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
+PyDictObject *
+_PyObject_MakeDictFromInstanceAttributes(PyObject *obj)
{
+ PyDictValues *values = _PyObject_InlineValues(obj);
PyInterpreterState *interp = _PyInterpreterState_GET();
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
OBJECT_STAT_INC(dict_materialized_on_request);
return make_dict_from_instance_attributes(interp, keys, values);
}
-// Return true if the dict was dematerialized, false otherwise.
-bool
-_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
-{
- assert(_PyObject_DictOrValuesPointer(obj) == dorv);
- assert(!_PyDictOrValues_IsValues(*dorv));
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
- if (dict == NULL) {
- return false;
- }
- // It's likely that this dict still shares its keys (if it was materialized
- // on request and not heavily modified):
- if (!PyDict_CheckExact(dict)) {
- return false;
- }
- assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE));
- if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) ||
- !has_unique_reference((PyObject *)dict))
- {
- return false;
- }
- ensure_shared_on_resize(dict);
- assert(dict->ma_values);
- // We have an opportunity to do something *really* cool: dematerialize it!
- _PyDictKeys_DecRef(dict->ma_keys);
- _PyDictOrValues_SetValues(dorv, dict->ma_values);
- OBJECT_STAT_INC(dict_dematerialized);
- // Don't try this at home, kids:
- dict->ma_keys = NULL;
- dict->ma_values = NULL;
- Py_DECREF(dict);
- return true;
-}
int
_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
assert(values != NULL);
- assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
Py_ssize_t ix = DKIX_EMPTY;
if (PyUnicode_CheckExact(name)) {
Py_hash_t hash = unicode_get_hash(name);
@@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
}
#endif
}
+ PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (ix == DKIX_EMPTY) {
- PyObject *dict = make_dict_from_instance_attributes(
- interp, keys, values);
if (dict == NULL) {
- return -1;
+ dict = make_dict_from_instance_attributes(
+ interp, keys, values);
+ if (dict == NULL) {
+ return -1;
+ }
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
- _PyObject_DictOrValuesPointer(obj)->dict = dict;
if (value == NULL) {
- return PyDict_DelItem(dict, name);
+ return PyDict_DelItem((PyObject *)dict, name);
}
else {
- return PyDict_SetItem(dict, name, value);
+ return PyDict_SetItem((PyObject *)dict, name, value);
}
}
PyObject *old_value = values->values[ix];
@@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
return -1;
}
_PyDictValues_AddToInsertionOrder(values, ix);
+ if (dict) {
+ assert(dict->ma_values == values);
+ dict->ma_used++;
+ }
}
else {
if (value == NULL) {
delete_index_from_values(values, ix);
+ if (dict) {
+ assert(dict->ma_values == values);
+ dict->ma_used--;
+ }
}
Py_DECREF(old_value);
}
@@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
{
PyTypeObject *tp = Py_TYPE(obj);
CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ if (_PyManagedDictPointer_IsValues(*managed_dict)) {
+ PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict);
int size = ((uint8_t *)values)[-2];
int count = 0;
PyDictKeysObject *keys = CACHED_KEYS(tp);
@@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
CHECK(size == count);
}
else {
- if (dorv_ptr->dict != NULL) {
- CHECK(PyDict_Check(dorv_ptr->dict));
+ if (managed_dict->dict != NULL) {
+ CHECK(PyDict_Check(managed_dict->dict));
}
}
return 1;
@@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
if (tp->tp_dictoffset == 0) {
return 1;
}
- PyObject *dict;
- if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(dorv)) {
+ PyDictObject *dict;
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) {
+ if (values->values[i] != NULL) {
return 0;
}
}
return 1;
}
- dict = _PyDictOrValues_GetDict(dorv);
+ dict = _PyObject_ManagedDictPointer(obj)->dict;
+ }
+ else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+ dict = managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
- dict = *dictptr;
+ dict = (PyDictObject *)*dictptr;
}
if (dict == NULL) {
return 1;
@@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
return ((PyDictObject *)dict)->ma_used == 0;
}
-void
-_PyObject_FreeInstanceAttributes(PyObject *self)
-{
- PyTypeObject *tp = Py_TYPE(self);
- assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self);
- if (!_PyDictOrValues_IsValues(dorv)) {
- return;
- }
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_XDECREF(values->values[i]);
- }
- free_values(values, IS_DICT_SHARED((PyDictObject*)self));
-}
-
int
PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
{
@@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return 0;
}
- assert(tp->tp_dictoffset);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(dorv)) {
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_VISIT(values->values[i]);
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
+ for (Py_ssize_t i = 0; i < values->capacity; i++) {
+ Py_VISIT(values->values[i]);
+ }
+ return 0;
}
}
- else {
- PyObject *dict = _PyDictOrValues_GetDict(dorv);
- Py_VISIT(dict);
- }
+ Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict);
return 0;
}
void
PyObject_ClearManagedDict(PyObject *obj)
{
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
PyTypeObject *tp = Py_TYPE(obj);
- if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
- return;
- }
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_CLEAR(values->values[i]);
- }
- dorv_ptr->dict = NULL;
- free_values(values, IS_DICT_SHARED((PyDictObject*)obj));
- }
- else {
- PyObject *dict = dorv_ptr->dict;
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (dict) {
- dorv_ptr->dict = NULL;
+ _PyDict_DetachFromObject(dict, obj);
+ _PyObject_ManagedDictPointer(obj)->dict = NULL;
Py_DECREF(dict);
}
+ else {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
+ for (Py_ssize_t i = 0; i < values->capacity; i++) {
+ Py_CLEAR(values->values[i]);
+ }
+ values->valid = 0;
+ }
+ }
+ }
+ else {
+ Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
+ }
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
+{
+ assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+ if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) {
+ return 0;
+ }
+ assert(mp->ma_values->embedded == 1);
+ assert(mp->ma_values->valid == 1);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ Py_BEGIN_CRITICAL_SECTION(mp);
+ mp->ma_values = copy_values(mp->ma_values);
+ _PyObject_InlineValues(obj)->valid = 0;
+ Py_END_CRITICAL_SECTION();
+ if (mp->ma_values == NULL) {
+ return -1;
}
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+ ASSERT_CONSISTENT(mp);
+ return 0;
}
PyObject *
PyObject_GenericGetDict(PyObject *obj, void *context)
{
- PyObject *dict;
PyInterpreterState *interp = _PyInterpreterState_GET();
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ PyDictObject *dict = managed_dict->dict;
+ if (dict == NULL &&
+ (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_InlineValues(obj)->valid
+ ) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
OBJECT_STAT_INC(dict_materialized_on_request);
dict = make_dict_from_instance_attributes(
interp, CACHED_KEYS(tp), values);
if (dict != NULL) {
- dorv_ptr->dict = dict;
+ managed_dict->dict = (PyDictObject *)dict;
}
}
else {
- dict = _PyDictOrValues_GetDict(*dorv_ptr);
+ dict = managed_dict->dict;
if (dict == NULL) {
dictkeys_incref(CACHED_KEYS(tp));
OBJECT_STAT_INC(dict_materialized_on_request);
- dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
- dorv_ptr->dict = dict;
+ dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
+ managed_dict->dict = (PyDictObject *)dict;
}
}
+ return Py_XNewRef((PyObject *)dict);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
"This object has no __dict__");
return NULL;
}
- dict = *dictptr;
+ PyObject *dict = *dictptr;
if (dict == NULL) {
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
@@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
*dictptr = dict = PyDict_New();
}
}
+ return Py_XNewRef(dict);
}
- return Py_XNewRef(dict);
}
int
@@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr,
assert(dictptr != NULL);
dict = *dictptr;
if (dict == NULL) {
- assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT));
+ assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES));
dictkeys_incref(cached);
dict = new_dict_with_shared_keys(interp, cached);
if (dict == NULL)
@@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits,
watcher_bits >>= 1;
}
}
+
+#ifndef NDEBUG
+static int
+_PyObject_InlineValuesConsistencyCheck(PyObject *obj)
+{
+ if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
+ return 1;
+ }
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict;
+ if (dict == NULL) {
+ return 1;
+ }
+ if (dict->ma_values == _PyObject_InlineValues(obj) ||
+ _PyObject_InlineValues(obj)->valid == 0) {
+ return 1;
+ }
+ assert(0);
+ return 0;
+}
+#endif
diff --git a/Objects/object.c b/Objects/object.c
index b4f0fd4d7db..60642d899bc 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj)
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return _PyObject_ComputedDictPointer(obj);
}
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr));
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
if (dict == NULL) {
PyErr_Clear();
return NULL;
}
- dorv_ptr->dict = dict;
+ managed_dict->dict = dict;
}
- return &dorv_ptr->dict;
+ return (PyObject **)&managed_dict->dict;
}
PyObject *
@@ -1474,21 +1474,19 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
}
}
PyObject *dict;
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
- if (attr != NULL) {
- *method = attr;
- Py_XDECREF(descr);
- return 0;
- }
- dict = NULL;
- }
- else {
- dict = dorv_ptr->dict;
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
+ if (attr != NULL) {
+ *method = attr;
+ Py_XDECREF(descr);
+ return 0;
}
+ dict = NULL;
+ }
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+ dict = (PyObject *)managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1581,29 +1579,27 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
}
}
if (dict == NULL) {
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- if (PyUnicode_CheckExact(name)) {
- res = _PyObject_GetInstanceAttribute(obj, values, name);
- if (res != NULL) {
- goto done;
- }
- }
- else {
- dict = _PyObject_MakeDictFromInstanceAttributes(obj, values);
- if (dict == NULL) {
- res = NULL;
- goto done;
- }
- dorv_ptr->dict = dict;
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (PyUnicode_CheckExact(name)) {
+ res = _PyObject_GetInstanceAttribute(obj, values, name);
+ if (res != NULL) {
+ goto done;
}
}
else {
- dict = _PyDictOrValues_GetDict(*dorv_ptr);
+ dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
+ if (dict == NULL) {
+ res = NULL;
+ goto done;
+ }
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
}
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+ dict = (PyObject *)managed_dict->dict;
+ }
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
if (dictptr) {
@@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name,
if (dict == NULL) {
PyObject **dictptr;
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- res = _PyObject_StoreInstanceAttribute(
- obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
- goto error_check;
- }
- dictptr = &dorv_ptr->dict;
- if (*dictptr == NULL) {
- if (_PyObject_InitInlineValues(obj, tp) < 0) {
- goto done;
- }
- res = _PyObject_StoreInstanceAttribute(
- obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
- goto error_check;
- }
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
+ res = _PyObject_StoreInstanceAttribute(
+ obj, _PyObject_InlineValues(obj), name, value);
+ goto error_check;
+ }
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ dictptr = (PyObject **)&managed_dict->dict;
}
else {
dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context)
{
PyObject **dictptr = _PyObject_GetDictPtr(obj);
if (dictptr == NULL) {
- if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) &&
- _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj)))
- {
+ if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_ManagedDictPointer(obj)->dict == NULL
+ ) {
/* Was unable to convert to dict */
PyErr_NoMemory();
}
diff --git a/Objects/object_layout.md b/Objects/object_layout.md
index 4f379bed8d7..352409425ee 100644
--- a/Objects/object_layout.md
+++ b/Objects/object_layout.md
@@ -16,7 +16,45 @@ Since the introduction of the cycle GC, there has also been a pre-header.
Before 3.11, this pre-header was two words in size.
It should be considered opaque to all code except the cycle GC.
-## 3.11 pre-header
+### 3.13
+
+In 3.13, the values array is embedded into the object, so there is no
+need for a values pointer (it is just a fixed offset into the object).
+So the pre-header is these two fields:
+
+* weakreflist
+* dict_pointer
+
+If the object has no physical dictionary, then the ``dict_pointer``
+is set to `NULL`.
+
+
+<details>
+<summary> 3.12 </summary>
+
+### 3.12
+
+In 3.12, the pointer to the list of weak references is added to the
+pre-header. In order to make space for it, the ``dict`` and ``values``
+pointers are combined into a single tagged pointer:
+
+* weakreflist
+* dict_or_values
+
+If the object has no physical dictionary, then the ``dict_or_values``
+has its low bit set to one, and points to the values array.
+If the object has a physical dictionary, then the ``dict_or_values``
+has its low bit set to zero, and points to the dictionary.
+
+The untagged form is chosen for the dictionary pointer, rather than
+the values pointer, to enable the (legacy) C-API function
+`_PyObject_GetDictPtr(PyObject *obj)` to work.
+</details>
+
+<details>
+<summary> 3.11 </summary>
+
+### 3.11
In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``.
The reason for moving the ``__dict__`` to the pre-header is that it allows
@@ -33,27 +71,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the
values of the objects's attributes.
Should the dictionary be needed, then ``values`` is set to ``NULL``
and the ``dict`` field points to the dictionary.
+</details>
-## 3.12 pre-header
+## Layout of a "normal" Python object
-In 3.12, the pointer to the list of weak references is added to the
-pre-header. In order to make space for it, the ``dict`` and ``values``
-pointers are combined into a single tagged pointer:
+A "normal" Python object is one that doesn't inherit from a builtin
+class, doesn't have slots.
+
+### 3.13
+
+In 3.13 the values are embedded into the object, as follows:
* weakreflist
* dict_or_values
+* GC 1
+* GC 2
+* ob_refcnt
+* ob_type
+* Inlined values:
+ * Flags
+ * values 0
+ * values 1
+ * ...
+ * Insertion order bytes
-If the object has no physical dictionary, then the ``dict_or_values``
-has its low bit set to one, and points to the values array.
-If the object has a physical dictionary, then the ``dict_or_values``
-has its low bit set to zero, and points to the dictionary.
+This has all the advantages of the layout used in 3.12, plus:
+* Access to values is even faster as there is one less load
+* Fast access is mostly maintained when the `__dict__` is materialized
-The untagged form is chosen for the dictionary pointer, rather than
-the values pointer, to enable the (legacy) C-API function
-`_PyObject_GetDictPtr(PyObject *obj)` to work.
+![Layout of "normal" object in 3.13](./object_layout_313.png)
+
+For objects with opaque parts defined by a C extension,
+the layout is much the same as for 3.12
+![Layout of "full" object in 3.13](./object_layout_full_313.png)
-## Layout of a "normal" Python object in 3.12:
+
+<details>
+<summary> 3.12 </summary>
+
+### 3.12:
+
+In 3.12, the header and pre-header form the entire object for "normal"
+Python objects:
* weakreflist
* dict_or_values
@@ -62,9 +122,6 @@ the values pointer, to enable the (legacy) C-API function
* ob_refcnt
* ob_type
-For a "normal" Python object, one that doesn't inherit from a builtin
-class or have slots, the header and pre-header form the entire object.
-
![Layout of "normal" object in 3.12](./object_layout_312.png)
There are several advantages to this layout:
@@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension,
and `__slots__` looks like this:
![Layout of "full" object in 3.12](./object_layout_full_312.png)
+</details>
+
diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv
index c0068d78568..731a25332b3 100644
--- a/Objects/object_layout_312.gv
+++ b/Objects/object_layout_312.gv
@@ -20,6 +20,7 @@ digraph ideal {
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>values</b></td></tr>
+ <tr><td border="1">Insertion order</td></tr>
<tr><td port="0" border="1">values[0]</td></tr>
<tr><td border="1">values[1]</td></tr>
<tr><td border="1">...</td></tr>
diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png
index 396dab183b3..a63d095ea0b 100644
--- a/Objects/object_layout_312.png
+++ b/Objects/object_layout_312.png
Binary files differ
diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv
new file mode 100644
index 00000000000..d85352876f2
--- /dev/null
+++ b/Objects/object_layout_313.gv
@@ -0,0 +1,45 @@
+digraph ideal {
+
+ rankdir = "LR"
+
+
+ object [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>object</b></td></tr>
+ <tr><td port="w" border="1">weakrefs</td></tr>
+ <tr><td port="dv" border="1">dict pointer</td></tr>
+ <tr><td border="1" >GC info 0</td></tr>
+ <tr><td border="1" >GC info 1</td></tr>
+ <tr><td port="r" border="1" >refcount</td></tr>
+ <tr><td port="h" border="1" >__class__</td></tr>
+ <tr><td border="1">values flags</td></tr>
+ <tr><td port="0" border="1">values[0]</td></tr>
+ <tr><td border="1">values[1]</td></tr>
+ <tr><td border="1">...</td></tr>
+ <tr><td border="1">Insertion order</td></tr>
+ </table>>
+
+ ]
+
+ class [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>class</b></td></tr>
+ <tr><td port="head" bgcolor="lightgreen" border="1">...</td></tr>
+ <tr><td border="1" bgcolor="lightgreen">dict_offset</td></tr>
+ <tr><td border="1" bgcolor="lightgreen">...</td></tr>
+ <tr><td port="k" border="1" bgcolor="lightgreen">cached_keys</td></tr>
+ </table>>
+ ]
+
+ keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"]
+ NULL [ label = " NULL"; shape="plain"]
+ object:w -> NULL
+ object:h -> class:head
+ object:dv -> NULL
+ class:k -> keys
+
+ oop [ label = "pointer"; shape="plain"]
+ oop -> object:r
+}
diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png
new file mode 100644
index 00000000000..f7059c286c8
--- /dev/null
+++ b/Objects/object_layout_313.png
Binary files differ
diff --git a/Objects/object_layout_full_313.gv b/Objects/object_layout_full_313.gv
new file mode 100644
index 00000000000..551312d51a7
--- /dev/null
+++ b/Objects/object_layout_full_313.gv
@@ -0,0 +1,25 @@
+digraph ideal {
+
+ rankdir = "LR"
+
+
+ object [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>object</b></td></tr>
+ <tr><td port="w" border="1">weakrefs</td></tr>
+ <tr><td port="dv" border="1">dict pointer</td></tr>
+ <tr><td border="1" >GC info 0</td></tr>
+ <tr><td border="1" >GC info 1</td></tr>
+ <tr><td port="r" border="1" >refcount</td></tr>
+ <tr><td port="h" border="1" >__class__</td></tr>
+ <tr><td border="1">opaque (extension) data </td></tr>
+ <tr><td border="1">...</td></tr>
+ <tr><td border="1">__slot__ 0</td></tr>
+ <tr><td border="1">...</td></tr>
+ </table>>
+ ]
+
+ oop [ label = "pointer"; shape="plain"]
+ oop -> object:r
+}
diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png
new file mode 100644
index 00000000000..7352ec69e5d
--- /dev/null
+++ b/Objects/object_layout_full_313.png
Binary files differ
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 2ef79fbf17b..6df9986b82e 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *
_PyType_NewManagedObject(PyTypeObject *type)
{
- assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(_PyType_IS_GC(type));
assert(type->tp_new == PyBaseObject_Type.tp_new);
assert(type->tp_alloc == PyType_GenericAlloc);
@@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type)
if (obj == NULL) {
return PyErr_NoMemory();
}
- _PyObject_DictOrValuesPointer(obj)->dict = NULL;
- if (_PyObject_InitInlineValues(obj, type)) {
- Py_DECREF(obj);
- return NULL;
- }
return obj;
}
@@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
* flag to indicate when that is safe) it does not seem worth the memory
* savings. An example type that doesn't need the +1 is a subclass of
* tuple. See GH-100659 and GH-81381. */
- const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
+ size_t size = _PyObject_VAR_SIZE(type, nitems+1);
const size_t presize = _PyType_PreHeaderSize(type);
+ if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ assert(type->tp_itemsize == 0);
+ size += _PyInlineValuesSize(type);
+ }
char *alloc = _PyObject_MallocWithType(type, size + presize);
if (alloc == NULL) {
return PyErr_NoMemory();
@@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
else {
_PyObject_InitVar((PyVarObject *)obj, type, nitems);
}
+ if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ _PyObject_InitInlineValues(obj, type);
+ }
return obj;
}
@@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self)
if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
PyObject_ClearManagedDict(self);
}
+ else {
+ assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) ==
+ (type->tp_flags & Py_TPFLAGS_INLINE_VALUES));
+ }
}
else if (type->tp_dictoffset != base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self)
/* If we added a dict, DECREF it, or free inline values. */
if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- _PyObject_FreeInstanceAttributes(self);
- }
- else {
- Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr));
- }
- dorv_ptr->values = NULL;
+ PyObject_ClearManagedDict(self);
}
else if (type->tp_dictoffset && !base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context)
return func(descr, obj, value);
}
/* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */
- dictptr = _PyObject_GetDictPtr(obj);
- if (dictptr == NULL) {
- PyErr_SetString(PyExc_AttributeError,
- "This object has no __dict__");
- return -1;
- }
if (value != NULL && !PyDict_Check(value)) {
PyErr_Format(PyExc_TypeError,
"__dict__ must be set to a dictionary, "
"not a '%.200s'", Py_TYPE(value)->tp_name);
return -1;
}
- Py_XSETREF(*dictptr, Py_XNewRef(value));
+ if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ PyObject_ClearManagedDict(obj);
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value);
+ }
+ else {
+ dictptr = _PyObject_ComputedDictPointer(obj);
+ if (dictptr == NULL) {
+ PyErr_SetString(PyExc_AttributeError,
+ "This object has no __dict__");
+ return -1;
+ }
+ Py_CLEAR(*dictptr);
+ *dictptr = Py_XNewRef(value);
+ }
return 0;
}
@@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (obj == NULL) {
return NULL;
}
- if (_PyObject_InitializeDict(obj)) {
- Py_DECREF(obj);
- return NULL;
- }
return obj;
}
@@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char*
!same_slots_added(newbase, oldbase))) {
goto differs;
}
+ if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) !=
+ ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES)))
+ {
+ goto differs;
+ }
/* The above does not check for the preheader */
if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) ==
((newto->tp_flags & Py_TPFLAGS_PREHEADER)))
@@ -6137,14 +6144,18 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
if (compatible_for_assignment(oldto, newto, "__class__")) {
/* Changing the class will change the implicit dict keys,
* so we must materialize the dictionary first. */
- assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER));
- _PyObject_GetDictPtr(self);
- if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT &&
- _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self)))
- {
- /* Was unable to convert to dict */
- PyErr_NoMemory();
- return -1;
+ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict;
+ if (dict == NULL) {
+ dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self);
+ if (dict == NULL) {
+ return -1;
+ }
+ _PyObject_ManagedDictPointer(self)->dict = dict;
+ }
+ if (_PyDict_DetachFromObject(dict, self)) {
+ return -1;
+ }
}
if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
Py_INCREF(newto);
@@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type)
return -1;
}
}
+ if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) {
+ type->tp_flags |= Py_TPFLAGS_INLINE_VALUES;
+ }
return 0;
}
@@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type)
/* Historically, all static types were immutable. See bpo-43908 */
if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
+ /* Static types must be immortal */
+ _Py_SetImmortalUntracked((PyObject *)type);
}
int res;
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-04 09:50:35 +0000