diff options
Diffstat (limited to 'Include/internal')
21 files changed, 176 insertions, 75 deletions
diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 4f77bd7bc52..a04169f7fb8 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -50,6 +50,32 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif +#if (MI_DEBUG) +#if defined(_MSC_VER) +#define mi_decl_noreturn __declspec(noreturn) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) +#define mi_decl_noreturn __attribute__((__noreturn__)) +#else +#define mi_decl_noreturn +#endif + +/* + * 'cold' attribute seems to have been fully supported since GCC 4.x. + * See https://github.com/gcc-mirror/gcc/commit/52bf96d2f299e9e6. + */ +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +#define mi_decl_cold __attribute__((cold)) +#else +#define mi_decl_cold +#endif + +#if (defined(__GNUC__) && defined(__THROW)) +#define mi_decl_throw __THROW +#else +#define mi_decl_throw +#endif +#endif + // ------------------------------------------------------ // Variants // ------------------------------------------------------ @@ -582,7 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +mi_decl_noreturn mi_decl_cold mi_decl_throw +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 3d8247df31c..092feeb40b0 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -353,6 +353,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif +_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 439989c60f6..8e1415f27b6 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -313,7 +313,7 @@ extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index aecc50be1e6..c18e04bf67a 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -95,6 +95,7 @@ typedef enum { enum _PyCompile_FBlockType { COMPILE_FBLOCK_WHILE_LOOP, COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_ASYNC_FOR_LOOP, COMPILE_FBLOCK_TRY_EXCEPT, COMPILE_FBLOCK_FINALLY_TRY, COMPILE_FBLOCK_FINALLY_END, diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 12729274138..713ddc66ba7 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -317,7 +317,9 @@ typedef enum error_code { _PyXI_ERR_ALREADY_RUNNING = -4, _PyXI_ERR_MAIN_NS_FAILURE = -5, _PyXI_ERR_APPLY_NS_FAILURE = -6, - _PyXI_ERR_NOT_SHAREABLE = -7, + _PyXI_ERR_PRESERVE_FAILURE = -7, + _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8, + _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; @@ -350,16 +352,33 @@ typedef struct xi_session _PyXI_session; PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); +typedef struct { + PyObject *preserved; + PyObject *excinfo; + _PyXI_errcode errcode; +} _PyXI_session_result; +PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); + PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, PyInterpreterState *interp, - PyObject *nsupdates); -PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); - -PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); - -PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); -PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); + PyObject *nsupdates, + _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit( + _PyXI_session *, + _PyXI_errcode, + _PyXI_session_result *); + +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( + _PyXI_session *, + _PyXI_errcode *); + +PyAPI_FUNC(int) _PyXI_Preserve( + _PyXI_session *, + const char *, + PyObject *, + _PyXI_errcode *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); /*************/ diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 1a265c59ff8..ce3fcb109f4 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -54,11 +54,13 @@ extern "C" { # define _Py_Debug_Free_Threaded 1 # define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc) # define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index) +# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) #else # define _Py_Debug_gilruntimestate_enabled 0 # define _Py_Debug_Free_Threaded 0 # define _Py_Debug_code_object_co_tlbc 0 # define _Py_Debug_interpreter_frame_tlbc_index 0 +# define _Py_Debug_interpreter_state_tlbc_generation 0 #endif @@ -89,6 +91,8 @@ typedef struct _Py_DebugOffsets { uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; + uint64_t code_object_generation; + uint64_t tlbc_generation; } interpreter_state; // Thread state offset; @@ -216,6 +220,11 @@ typedef struct _Py_DebugOffsets { uint64_t gi_frame_state; } gen_object; + struct _llist_node { + uint64_t next; + uint64_t prev; + } llist_node; + struct _debugger_support { uint64_t eval_breaker; uint64_t remote_debugger_support; @@ -251,6 +260,8 @@ typedef struct _Py_DebugOffsets { .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \ + .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \ }, \ .thread_state = { \ .size = sizeof(PyThreadState), \ @@ -347,6 +358,10 @@ typedef struct _Py_DebugOffsets { .gi_iframe = offsetof(PyGenObject, gi_iframe), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ + .llist_node = { \ + .next = offsetof(struct llist_node, next), \ + .prev = offsetof(struct llist_node, prev), \ + }, \ .debugger_support = { \ .eval_breaker = offsetof(PyThreadState, eval_breaker), \ .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d896e870630..e118b86db50 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -795,6 +795,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); @@ -891,6 +892,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index a06d7495bab..36f3d23d095 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -286,6 +286,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) @@ -382,6 +383,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c3e6c77405b..f25f5847b3b 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -677,8 +677,11 @@ struct _Py_interp_cached_objects { /* object.__reduce__ */ PyObject *objreduce; +#ifndef Py_GIL_DISABLED + /* resolve_slotdups() */ PyObject *type_slots_pname; pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; +#endif /* TypeVar and related types */ PyTypeObject *generic_type; @@ -726,6 +729,10 @@ typedef struct _PyIndexPool { // Next index to allocate if no free indices are available int32_t next_index; + + // Generation counter incremented on thread creation/destruction + // Used for TLBC cache invalidation in remote debugging + uint32_t tlbc_generation; } _PyIndexPool; typedef union _Py_unique_id_entry { @@ -843,6 +850,8 @@ struct _is { /* The per-interpreter GIL, which might not be used. */ struct _gil_runtime_state _gil; + uint64_t _code_object_generation; + /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 7484b05d7f2..32b60cc33a2 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -48,6 +48,9 @@ typedef enum _PyLockFlags { // Handle signals if interrupted while waiting on the lock. _PY_LOCK_HANDLE_SIGNALS = 2, + + // Fail if interrupted by a signal while waiting on the lock. + _PY_FAIL_IF_INTERRUPTED = 4, } _PyLockFlags; // Lock a mutex with an optional timeout and additional options. See diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 3fd56c346b9..cd1fc873623 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -279,6 +279,8 @@ Known values: Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) + Python 3.15a1 3652 (Virtual iterators) + Python 3.16 will start with 3700 @@ -291,7 +293,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3651 +#define PYC_MAGIC_NUMBER 3652 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index 614e9f93751..d90f42e9cd8 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -27,9 +27,8 @@ PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); // Export for 'zlib' shared extension PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, Py_ssize_t, Py_ssize_t); -#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) #define _PyArg_CheckPositional(funcname, nargs, min, max) \ - ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ + (((min) <= (nargs) && (nargs) <= (max)) \ || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) extern PyObject ** _Py_VaBuildStack( diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b7e162c8abc..50225623fe5 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -313,7 +313,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); + return ((type->tp_flags) & feature) != 0; } extern void _PyType_InitCache(PyInterpreterState *interp); @@ -897,6 +897,9 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, + PyObject *name, _PyStackRef *method); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -1007,6 +1010,8 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; +int _PyObject_SetDict(PyObject *obj, PyObject *value); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 39d6a912a54..00e918cb8f0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -205,15 +205,15 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 2; case FOR_ITER: - return 1; + return 2; case FOR_ITER_GEN: - return 1; + return 2; case FOR_ITER_LIST: - return 1; + return 2; case FOR_ITER_RANGE: - return 1; + return 2; case FOR_ITER_TUPLE: - return 1; + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -239,11 +239,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: - return 2; + return 3; case INSTRUMENTED_END_SEND: return 2; case INSTRUMENTED_FOR_ITER: - return 1; + return 2; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -257,7 +257,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_NOT_TAKEN: return 0; case INSTRUMENTED_POP_ITER: - return 1; + return 2; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 1; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -395,7 +395,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case POP_EXCEPT: return 1; case POP_ITER: - return 1; + return 2; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -688,15 +688,15 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 1; case FOR_ITER: - return 2; + return 3; case FOR_ITER_GEN: - return 1; - case FOR_ITER_LIST: return 2; + case FOR_ITER_LIST: + return 3; case FOR_ITER_RANGE: - return 2; + return 3; case FOR_ITER_TUPLE: - return 2; + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -704,7 +704,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_AWAITABLE: return 1; case GET_ITER: - return 1; + return 2; case GET_LEN: return 2; case GET_YIELD_FROM_ITER: @@ -722,11 +722,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 0; case INSTRUMENTED_END_FOR: - return 1; + return 2; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: - return 2; + return 3; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -1157,7 +1157,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1242,7 +1242,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, @@ -1453,7 +1453,7 @@ _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, - [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 83301d8aef7..d172cc1485d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -793,6 +793,7 @@ extern "C" { INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(any), \ INIT_ID(append), \ @@ -889,6 +890,7 @@ extern "C" { INIT_ID(data), \ INIT_ID(database), \ INIT_ID(day), \ + INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 40ec00c8119..f2ecc30b053 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -232,6 +232,9 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); +/* Increments a tagged int, but does not check for overflow */ +extern _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref); + extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); @@ -239,11 +242,12 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #define Py_INT_TAG 3 #define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { - return (i.bits & Py_INT_TAG) == Py_INT_TAG; + return (i.bits & Py_TAG_BITS) == Py_INT_TAG; } static inline _PyStackRef @@ -262,12 +266,21 @@ PyStackRef_UntagInt(_PyStackRef i) } +static inline _PyStackRef +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) +{ + assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int + assert((ref.bits & (~Py_TAG_BITS)) != (INT_MAX & (~Py_TAG_BITS))); // Isn't about to overflow + return (_PyStackRef){ .bits = ref.bits + 4 }; +} + +#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) + #ifdef Py_GIL_DISABLED #define Py_TAG_DEFERRED Py_TAG_REFCNT #define Py_TAG_PTR ((uintptr_t)0) -#define Py_TAG_BITS ((uintptr_t)1) static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; @@ -379,7 +392,7 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) do { \ _PyStackRef _close_tmp = (REF); \ assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferred(_close_tmp)) { \ + if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ } \ } while (0) @@ -395,7 +408,7 @@ static inline _PyStackRef PyStackRef_DUP(_PyStackRef stackref) { assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { + if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); @@ -442,7 +455,6 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ -#define Py_TAG_BITS 3 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" #endif @@ -678,7 +690,13 @@ PyStackRef_XCLOSE(_PyStackRef ref) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) +static inline PyTypeObject * +PyStackRef_TYPE(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return &PyLong_Type; + } + return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)); +} // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. @@ -686,42 +704,30 @@ PyStackRef_XCLOSE(_PyStackRef ref) // StackRef type checks -static inline bool -PyStackRef_GenCheck(_PyStackRef stackref) -{ - return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} +#define STACKREF_CHECK_FUNC(T) \ + static inline bool \ + PyStackRef_ ## T ## Check(_PyStackRef stackref) { \ + if (PyStackRef_IsTaggedInt(stackref)) { \ + return false; \ + } \ + return Py ## T ## _Check(PyStackRef_AsPyObjectBorrow(stackref)); \ + } -static inline bool -PyStackRef_BoolCheck(_PyStackRef stackref) -{ - return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} +STACKREF_CHECK_FUNC(Gen) +STACKREF_CHECK_FUNC(Bool) +STACKREF_CHECK_FUNC(ExceptionInstance) +STACKREF_CHECK_FUNC(Code) +STACKREF_CHECK_FUNC(Function) static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return true; + } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } -static inline bool -PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) -{ - return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_CodeCheck(_PyStackRef stackref) -{ - return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_FunctionCheck(_PyStackRef stackref) -{ - return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - static inline void _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 008a2da0d04..347b0a7a790 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -8,11 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -PyAPI_FUNC(int) _PySys_GetOptionalAttr(PyObject *, PyObject **); -PyAPI_FUNC(int) _PySys_GetOptionalAttrString(const char *, PyObject **); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttr(PyObject *); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttrString(const char *); - // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 1a4f89fd244..0ee7d555c56 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -134,7 +134,6 @@ extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); -extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); PyTypeObject *_PyType_LookupByVersion(unsigned int version); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index c0f5f2b17f6..0a9be4e41ac 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -932,6 +932,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1316,6 +1320,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(debug); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d08799487fd..2b845527cf2 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -272,6 +272,7 @@ extern "C" { #define _POP_CALL_TWO 489 #define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 490 #define _POP_EXCEPT POP_EXCEPT +#define _POP_ITER POP_ITER #define _POP_JUMP_IF_FALSE 491 #define _POP_JUMP_IF_TRUE 492 #define _POP_TOP POP_TOP diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5ebe124983b..b08909e72c4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -66,6 +66,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_TWO] = HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, + [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, @@ -205,7 +206,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, @@ -569,6 +570,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_CALL_TWO] = "_POP_CALL_TWO", [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_ITER] = "_POP_ITER", [_POP_TOP] = "_POP_TOP", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", @@ -730,6 +732,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _END_FOR: return 1; + case _POP_ITER: + return 2; case _END_SEND: return 2; case _UNARY_NEGATIVE: |