diff options
Diffstat (limited to 'Include/internal')
-rw-r--r-- | Include/internal/mimalloc/mimalloc/internal.h | 4 | ||||
-rw-r--r-- | Include/internal/pycore_ceval.h | 3 | ||||
-rw-r--r-- | Include/internal/pycore_crossinterp.h | 37 | ||||
-rw-r--r-- | Include/internal/pycore_freelist_state.h | 2 | ||||
-rw-r--r-- | Include/internal/pycore_interp_structs.h | 3 | ||||
-rw-r--r-- | Include/internal/pycore_long.h | 6 | ||||
-rw-r--r-- | Include/internal/pycore_magic_number.h | 3 | ||||
-rw-r--r-- | Include/internal/pycore_opcode_metadata.h | 6 | ||||
-rw-r--r-- | Include/internal/pycore_stackref.h | 68 | ||||
-rw-r--r-- | Include/internal/pycore_uop_metadata.h | 6 |
10 files changed, 103 insertions, 35 deletions
diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 71b7ea702d6..a7daa3a40a4 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -634,10 +634,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #ifdef MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed(&block->next), keys); + next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next), keys); #else MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)mi_atomic_load_relaxed(&block->next); + next = (mi_block_t*)mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 092feeb40b0..239177deb4a 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -353,7 +353,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif -_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index); +PyAPI_FUNC(_PyStackRef) +_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr); #ifdef __cplusplus } diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 713ddc66ba7..81faffac194 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -303,10 +303,10 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; -PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(_PyXI_excinfo *) _PyXI_NewExcInfo(PyObject *exc); +PyAPI_FUNC(void) _PyXI_FreeExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); -PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); typedef enum error_code { @@ -322,19 +322,20 @@ typedef enum error_code { _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; +typedef struct xi_failure _PyXI_failure; -typedef struct _sharedexception { - // The originating interpreter. - PyInterpreterState *interp; - // The kind of error to propagate. - _PyXI_errcode code; - // The exception information to propagate, if applicable. - // This is populated only for some error codes, - // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. - _PyXI_excinfo uncaught; -} _PyXI_error; +PyAPI_FUNC(_PyXI_failure *) _PyXI_NewFailure(void); +PyAPI_FUNC(void) _PyXI_FreeFailure(_PyXI_failure *); +PyAPI_FUNC(_PyXI_errcode) _PyXI_GetFailureCode(_PyXI_failure *); +PyAPI_FUNC(int) _PyXI_InitFailure(_PyXI_failure *, _PyXI_errcode, PyObject *); +PyAPI_FUNC(void) _PyXI_InitFailureUTF8( + _PyXI_failure *, + _PyXI_errcode, + const char *); -PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); +PyAPI_FUNC(int) _PyXI_UnwrapNotShareableError( + PyThreadState *, + _PyXI_failure *); // A cross-interpreter session involves entering an interpreter @@ -366,19 +367,21 @@ PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Exit( _PyXI_session *, - _PyXI_errcode, + _PyXI_failure *, _PyXI_session_result *); PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( _PyXI_session *, - _PyXI_errcode *); + _PyXI_failure *); PyAPI_FUNC(int) _PyXI_Preserve( _PyXI_session *, const char *, PyObject *, - _PyXI_errcode *); -PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); + _PyXI_failure *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved( + _PyXI_session_result *, + const char *); /*************/ diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4828dfd948f..59beb92f3f7 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -16,6 +16,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_complexes_MAXFREELIST 100 # define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_ranges_MAXFREELIST 6 @@ -43,6 +44,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist complexes; struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f25f5847b3b..f1f427d99de 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -159,10 +159,11 @@ struct atexit_state { typedef struct { // Tagged pointer to next object in the list. // 0 means the object is not tracked - uintptr_t _gc_next; + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) _gc_next; // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. + // Those bits are made available by the struct's minimum alignment. uintptr_t _gc_prev; } PyGC_Head; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3196d1b8208..b8efba74bdc 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -112,9 +112,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); -PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index cd1fc873623..347d9762f26 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -280,6 +280,7 @@ Known values: Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) + Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.16 will start with 3700 @@ -293,7 +294,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3652 +#define PYC_MAGIC_NUMBER 3653 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 00e918cb8f0..59c85ddeb3d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1072,12 +1072,12 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index f2ecc30b053..10e7199269e 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -62,14 +62,15 @@ PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filenam extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; +static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; // Use the first 3 even numbers for None, True and False. // Odd numbers are reserved for (tagged) integers -#define PyStackRef_None ((_PyStackRef){ .index = 2 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 4 }) -#define PyStackRef_True ((_PyStackRef){ .index = 6 }) +#define PyStackRef_None ((_PyStackRef){ .index = 4 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 6 }) +#define PyStackRef_True ((_PyStackRef){ .index = 8 }) -#define INITIAL_STACKREF_INDEX 8 +#define INITIAL_STACKREF_INDEX 10 static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -77,6 +78,19 @@ PyStackRef_IsNull(_PyStackRef ref) return ref.index == 0; } +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.index == 2; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref); +} + static inline int PyStackRef_IsTrue(_PyStackRef ref) { @@ -104,6 +118,7 @@ PyStackRef_IsTaggedInt(_PyStackRef ref) static inline PyObject * _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); assert(!PyStackRef_IsTaggedInt(ref)); _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); @@ -155,6 +170,7 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (PyStackRef_IsNull(ref)) { return; } @@ -165,6 +181,7 @@ _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (PyStackRef_IsTaggedInt(ref)) { return ref; } @@ -241,9 +258,51 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else #define Py_INT_TAG 3 +#define Py_TAG_INVALID 2 #define Py_TAG_REFCNT 1 #define Py_TAG_BITS 3 +static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; + +/* Wrap a pointer in a stack ref. + * The resulting stack reference is not safe and should only be used + * in the interpreter to pass values from one uop to another. + * The GC should never see one of these stack refs. */ +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .bits = (uintptr_t)ptr }; +#endif +} + +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.bits & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.bits & ~Py_TAG_BITS); +#else + return (void *)(ref.bits); +#endif +} + +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.bits == Py_TAG_INVALID; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return ref.bits >= Py_INT_TAG; +} + static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { @@ -284,6 +343,7 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; + #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b08909e72c4..cd36023c25c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -85,9 +85,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, |