18 files changed, 93 insertions, 77 deletions
diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h
index 8ee03e82f74..63886fca28e 100644
--- a/Include/cpython/lock.h
+++ b/Include/cpython/lock.h
@@ -36,6 +36,9 @@ PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m);
 // exported function for unlocking the mutex
 PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m);
 
+// exported function for checking if the mutex is locked
+PyAPI_FUNC(int) PyMutex_IsLocked(PyMutex *m);
+
 // Locks the mutex.
 //
 // If the mutex is currently locked, the calling thread will be parked until
@@ -61,3 +64,11 @@ _PyMutex_Unlock(PyMutex *m)
     }
 }
 #define PyMutex_Unlock _PyMutex_Unlock
+
+// Checks if the mutex is currently locked.
+static inline int
+_PyMutex_IsLocked(PyMutex *m)
+{
+    return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0;
+}
+#define PyMutex_IsLocked _PyMutex_IsLocked
diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
index 86c502730f4..73e3bc44d6c 100644
--- a/Include/cpython/unicodeobject.h
+++ b/Include/cpython/unicodeobject.h
@@ -109,7 +109,7 @@ struct _PyUnicodeObject_state {
    immediately follow the structure. utf8_length can be found
    in the length field; the utf8 pointer is equal to the data pointer. */
 typedef struct {
-    /* There are 4 forms of Unicode strings:
+    /* There are 3 forms of Unicode strings:
 
        - compact ascii:
 
diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h
index a04169f7fb8..a17f637fe68 100644
--- a/Include/internal/mimalloc/mimalloc/types.h
+++ b/Include/internal/mimalloc/mimalloc/types.h
@@ -705,7 +705,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
 // Thread Local data
 // ------------------------------------------------------
 
-// A "span" is is an available range of slices. The span queues keep
+// A "span" is an available range of slices. The span queues keep
 // track of slice spans of at most the given `slice_count` (but more than the previous size class).
 typedef struct mi_span_queue_s {
   mi_slice_t* first;
diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
index e118b86db50..c461bc1786d 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -1136,6 +1136,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(offset_src));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(on_type_read));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(onceregistry));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_active_thread));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode));
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index 36f3d23d095..72c2051bd97 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -627,6 +627,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(offset_src)
         STRUCT_FOR_ID(on_type_read)
         STRUCT_FOR_ID(onceregistry)
+        STRUCT_FOR_ID(only_active_thread)
         STRUCT_FOR_ID(only_keys)
         STRUCT_FOR_ID(oparg)
         STRUCT_FOR_ID(opcode)
diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h
index 32b60cc33a2..585120108cf 100644
--- a/Include/internal/pycore_lock.h
+++ b/Include/internal/pycore_lock.h
@@ -25,13 +25,6 @@ PyMutex_LockFast(PyMutex *m)
     return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED);
 }
 
-// Checks if the mutex is currently locked.
-static inline int
-PyMutex_IsLocked(PyMutex *m)
-{
-    return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0;
-}
-
 // Re-initializes the mutex after a fork to the unlocked state.
 static inline void
 _PyMutex_at_fork_reinit(PyMutex *m)
@@ -51,6 +44,11 @@ typedef enum _PyLockFlags {
 
     // Fail if interrupted by a signal while waiting on the lock.
     _PY_FAIL_IF_INTERRUPTED = 4,
+
+    // Locking & unlocking this lock requires attached thread state.
+    // If locking returns PY_LOCK_FAILURE, a Python exception *may* be raised.
+    // (Intended for use with _PY_LOCK_HANDLE_SIGNALS and _PY_LOCK_DETACH.)
+    _PY_LOCK_PYTHONLOCK = 8,
 } _PyLockFlags;
 
 // Lock a mutex with an optional timeout and additional options. See
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 50807e68e9a..40f8ca68c00 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -614,7 +614,7 @@ static inline PyObject *
 _Py_XGetRef(PyObject **ptr)
 {
     for (;;) {
-        PyObject *value = _Py_atomic_load_ptr(ptr);
+        PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));
         if (value == NULL) {
             return value;
         }
@@ -629,7 +629,7 @@ _Py_XGetRef(PyObject **ptr)
 static inline PyObject *
 _Py_TryXGetRef(PyObject **ptr)
 {
-    PyObject *value = _Py_atomic_load_ptr(ptr);
+    PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));
     if (value == NULL) {
         return value;
     }
@@ -918,7 +918,7 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *,
 extern unsigned int
 _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out);
 
-extern int _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj,
+PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj,
                                        PyObject *name, _PyStackRef *method);
 
 // Cache the provided init method in the specialization cache of type if the
@@ -1033,6 +1033,20 @@ enum _PyAnnotateFormat {
 
 int _PyObject_SetDict(PyObject *obj, PyObject *value);
 
+#ifndef Py_GIL_DISABLED
+static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)
+{
+    assert(!_Py_IsStaticImmortal(op));
+    op->ob_refcnt++;
+    _Py_INCREF_STAT_INC();
+#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)
+    if (!_Py_IsImmortal(op)) {
+        _Py_INCREF_IncRefTotal();
+    }
+#endif
+}
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index 576c2794782..8b7f12bf03d 100644
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -10,7 +10,7 @@ extern "C" {
 
 #include "pycore_typedefs.h"      // _PyInterpreterFrame
 #include "pycore_uop_ids.h"
-#include "pycore_stackref.h"
+#include "pycore_stackref.h"      // _PyStackRef
 #include <stdbool.h>
 
 
@@ -316,6 +316,9 @@ extern JitOptRef _Py_uop_sym_new_type(
     JitOptContext *ctx, PyTypeObject *typ);
 
 extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val);
+extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val);
+bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym);
+_PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym);
 extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx);
 extern bool _Py_uop_sym_has_type(JitOptRef sym);
 extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ);
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 633e5cf77db..ea3dfbd2eef 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -8,6 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_pythonrun.h"     // _PyOS_STACK_MARGIN_SHIFT
 #include "pycore_typedefs.h"      // _PyRuntimeState
 #include "pycore_tstate.h"
 
@@ -325,7 +326,7 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate)
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
     assert(_tstate->c_stack_hard_limit != 0);
     intptr_t here_addr = _Py_get_machine_stack_pointer();
-    return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, PYOS_STACK_MARGIN_SHIFT);
+    return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT);
 }
 
 #ifdef __cplusplus
diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h
index 7daed1326af..c2832098ddb 100644
--- a/Include/internal/pycore_pythonrun.h
+++ b/Include/internal/pycore_pythonrun.h
@@ -33,6 +33,28 @@ extern const char* _Py_SourceAsString(
     PyCompilerFlags *cf,
     PyObject **cmd_copy);
 
+
+/* Stack size, in "pointers". This must be large enough, so
+ * no two calls to check recursion depth are more than this far
+ * apart. In practice, that means it must be larger than the C
+ * stack consumption of PyEval_EvalDefault */
+#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER)
+#  define _PyOS_LOG2_STACK_MARGIN 12
+#elif defined(Py_DEBUG) && defined(WIN32)
+#  define _PyOS_LOG2_STACK_MARGIN 12
+#else
+#  define _PyOS_LOG2_STACK_MARGIN 11
+#endif
+#define _PyOS_STACK_MARGIN (1 << _PyOS_LOG2_STACK_MARGIN)
+#define _PyOS_STACK_MARGIN_BYTES (_PyOS_STACK_MARGIN * sizeof(void *))
+
+#if SIZEOF_VOID_P == 8
+#  define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 3)
+#else
+#  define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2)
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
index d172cc1485d..d378fcae26c 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -1134,6 +1134,7 @@ extern "C" {
     INIT_ID(offset_src), \
     INIT_ID(on_type_read), \
     INIT_ID(onceregistry), \
+    INIT_ID(only_active_thread), \
     INIT_ID(only_keys), \
     INIT_ID(oparg), \
     INIT_ID(opcode), \
diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index 10e7199269e..6bf82d8322f 100644
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -626,7 +626,7 @@ _PyStackRef_FromPyObjectNew(PyObject *obj)
     if (_Py_IsImmortal(obj)) {
         return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT};
     }
-    Py_INCREF_MORTAL(obj);
+    _Py_INCREF_MORTAL(obj);
     _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
     PyStackRef_CheckValid(ref);
     return ref;
@@ -637,7 +637,7 @@ static inline _PyStackRef
 _PyStackRef_FromPyObjectNewMortal(PyObject *obj)
 {
     assert(obj != NULL);
-    Py_INCREF_MORTAL(obj);
+    _Py_INCREF_MORTAL(obj);
     _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
     PyStackRef_CheckValid(ref);
     return ref;
@@ -654,14 +654,14 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj)
 /* WARNING: This macro evaluates its argument more than once */
 #ifdef _WIN32
 #define PyStackRef_DUP(REF) \
-    (PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
+    (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
 #else
 static inline _PyStackRef
 PyStackRef_DUP(_PyStackRef ref)
 {
     assert(!PyStackRef_IsNull(ref));
     if (PyStackRef_RefcountOnObject(ref)) {
-        Py_INCREF_MORTAL(BITS_TO_PTR(ref));
+        _Py_INCREF_MORTAL(BITS_TO_PTR(ref));
     }
     return ref;
 }
@@ -829,7 +829,7 @@ _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)
 static inline int
 _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
 {
-    PyObject *op = _Py_atomic_load_ptr_relaxed(src);
+    PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src));
     if (op == NULL) {
         *out = PyStackRef_NULL;
         return 1;
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
index 0a9be4e41ac..e516211f6c6 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -2296,6 +2296,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));
     assert(PyUnicode_GET_LENGTH(string) != 1);
+    string = &_Py_ID(only_active_thread);
+    _PyUnicode_InternStatic(interp, &string);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    assert(PyUnicode_GET_LENGTH(string) != 1);
     string = &_Py_ID(only_keys);
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));
diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index 52cbc2fffe4..ff7e800aa9b 100644
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -106,7 +106,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
     [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
     [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
-    [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG,
+    [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG,
     [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
     [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
     [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h
index 950aa0af290..4ed8928c0b9 100644
--- a/Include/internal/pycore_weakref.h
+++ b/Include/internal/pycore_weakref.h
@@ -29,6 +29,12 @@ extern "C" {
     PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH)
 #define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock)
 
+#define FT_CLEAR_WEAKREFS(obj, weakref_list)    \
+    do {                                        \
+        assert(Py_REFCNT(obj) == 0);            \
+        PyObject_ClearWeakRefs(obj);            \
+    } while (0)
+
 #else
 
 #define LOCK_WEAKREFS(obj)
@@ -37,6 +43,14 @@ extern "C" {
 #define LOCK_WEAKREFS_FOR_WR(wr)
 #define UNLOCK_WEAKREFS_FOR_WR(wr)
 
+#define FT_CLEAR_WEAKREFS(obj, weakref_list)        \
+    do {                                            \
+        assert(Py_REFCNT(obj) == 0);                \
+        if (weakref_list != NULL) {                 \
+            PyObject_ClearWeakRefs(obj);            \
+        }                                           \
+    } while (0)
+
 #endif
 
 static inline int _is_dead(PyObject *obj)
diff --git a/Include/pyport.h b/Include/pyport.h
index 0675294d5bc..89829373be2 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -667,25 +667,6 @@ extern "C" {
 #endif
 
 
-// _Py_NO_SANITIZE_UNDEFINED(): Disable Undefined Behavior sanitizer (UBsan)
-// on a function.
-//
-// Clang and GCC 9.0+ use __attribute__((no_sanitize("undefined"))).
-// GCC 4.9+ uses __attribute__((no_sanitize_undefined)).
-#if defined(__has_feature)
-#  if __has_feature(undefined_behavior_sanitizer)
-#    define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
-#  endif
-#endif
-#if !defined(_Py_NO_SANITIZE_UNDEFINED) && defined(__GNUC__) \
-    && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9))
-#  define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
-#endif
-#ifndef _Py_NO_SANITIZE_UNDEFINED
-#  define _Py_NO_SANITIZE_UNDEFINED
-#endif
-
-
 // _Py_NONSTRING: The nonstring variable attribute specifies that an object or
 // member declaration with type array of char, signed char, or unsigned char,
 // or pointer to such a type is intended to store character arrays that do not
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 716c4ab3c81..92b50aa807b 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -21,36 +21,15 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
 /* Stuff with no proper home (yet) */
 PyAPI_DATA(int) (*PyOS_InputHook)(void);
 
-/* Stack size, in "pointers". This must be large enough, so
- * no two calls to check recursion depth are more than this far
- * apart. In practice, that means it must be larger than the C
- * stack consumption of PyEval_EvalDefault */
-#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER)
-#  define PYOS_LOG2_STACK_MARGIN 12
-#elif defined(Py_DEBUG) && defined(WIN32)
-#  define PYOS_LOG2_STACK_MARGIN 12
-#else
-#  define PYOS_LOG2_STACK_MARGIN 11
-#endif
-#define PYOS_STACK_MARGIN (1 << PYOS_LOG2_STACK_MARGIN)
-#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *))
-
-#if SIZEOF_VOID_P == 8
-#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 3)
-#else
-#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 2)
-#endif
-
-
 #if defined(WIN32)
-#define USE_STACKCHECK
+#  define USE_STACKCHECK
 #endif
-
 #ifdef USE_STACKCHECK
 /* Check that we aren't overflowing our stack */
 PyAPI_FUNC(int) PyOS_CheckStack(void);
 #endif
 
+
 #ifndef Py_LIMITED_API
 #  define Py_CPYTHON_PYTHONRUN_H
 #  include "cpython/pythonrun.h"
diff --git a/Include/refcount.h b/Include/refcount.h
index 034c453f449..457972b6dcf 100644
--- a/Include/refcount.h
+++ b/Include/refcount.h
@@ -30,7 +30,7 @@ increase and decrease the objects reference count.
 
 In order to offer sufficient resilience to C extensions using the stable ABI
 compiled against 3.11 or earlier, we set the initial value near the
-middle of the range (2**31, 2**32). That way the the refcount can be
+middle of the range (2**31, 2**32). That way the refcount can be
 off by ~1 billion without affecting immortality.
 
 Reference count increases will use saturated arithmetic, taking advantage of
@@ -244,20 +244,6 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *);
 PyAPI_FUNC(void) _Py_IncRef(PyObject *);
 PyAPI_FUNC(void) _Py_DecRef(PyObject *);
 
-#ifndef Py_GIL_DISABLED
-static inline Py_ALWAYS_INLINE void Py_INCREF_MORTAL(PyObject *op)
-{
-    assert(!_Py_IsStaticImmortal(op));
-    op->ob_refcnt++;
-    _Py_INCREF_STAT_INC();
-#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)
-    if (!_Py_IsImmortal(op)) {
-        _Py_INCREF_IncRefTotal();
-    }
-#endif
-}
-#endif
-
 static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
 {
 #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))