gh-76785: Improved Subinterpreters Compatibility with 3.12 (2/2) (gh-126707)

These changes makes it easier to backport the _interpreters, _interpqueues, and _interpchannels modules to Python 3.12.

This involves the following:

* add the _PyXI_GET_STATE() and _PyXI_GET_GLOBAL_STATE() macros
* add _PyXIData_lookup_context_t and _PyXIData_GetLookupContext()
* add _Py_xi_state_init() and _Py_xi_state_fini()

1 files changed, 121 insertions, 42 deletions

diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index dfdb5f9d87a..fe7d75f6b72 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -9,10 +9,6 @@
 #include "pycore_pyerrors.h"      // _PyErr_Clear()
 
 
-#define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi)
-#define _PyXI_GET_STATE(interp) (&(interp)->xi)
-
-
 /**************/
 /* exceptions */
 /**************/
@@ -68,7 +64,7 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp,
 
 static void xid_lookup_init(_PyXIData_lookup_t *);
 static void xid_lookup_fini(_PyXIData_lookup_t *);
-static xidatafunc lookup_getdata(PyInterpreterState *, PyObject *);
+static xidatafunc lookup_getdata(_PyXIData_lookup_context_t *, PyObject *);
 #include "crossinterp_data_lookup.h"
 
 
@@ -202,9 +198,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *data)
 }
 
 static inline void
-_set_xid_lookup_failure(_PyXI_state_t *state, PyObject *obj, const char *msg)
+_set_xid_lookup_failure(dlcontext_t *ctx, PyObject *obj, const char *msg)
 {
-    PyObject *exctype = state->exceptions.PyExc_NotShareableError;
+    PyObject *exctype = ctx->PyExc_NotShareableError;
     assert(exctype != NULL);
     if (msg != NULL) {
         assert(obj == NULL);
@@ -221,14 +217,12 @@ _set_xid_lookup_failure(_PyXI_state_t *state, PyObject *obj, const char *msg)
 }
 
 int
-_PyObject_CheckXIData(PyObject *obj)
+_PyObject_CheckXIData(_PyXIData_lookup_context_t *ctx, PyObject *obj)
 {
-    PyInterpreterState *interp = PyInterpreterState_Get();
-    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
-    xidatafunc getdata = lookup_getdata(interp, obj);
+    xidatafunc getdata = lookup_getdata(ctx, obj);
     if (getdata == NULL) {
         if (!PyErr_Occurred()) {
-            _set_xid_lookup_failure(state, obj, NULL);
+            _set_xid_lookup_failure(ctx, obj, NULL);
         }
         return -1;
     }
@@ -236,11 +230,11 @@ _PyObject_CheckXIData(PyObject *obj)
 }
 
 int
-_PyObject_GetXIData(PyObject *obj, _PyXIData_t *data)
+_PyObject_GetXIData(_PyXIData_lookup_context_t *ctx,
+                    PyObject *obj, _PyXIData_t *data)
 {
     PyThreadState *tstate = PyThreadState_Get();
     PyInterpreterState *interp = tstate->interp;
-    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
 
     // Reset data before re-populating.
     *data = (_PyXIData_t){0};
@@ -248,11 +242,11 @@ _PyObject_GetXIData(PyObject *obj, _PyXIData_t *data)
 
     // Call the "getdata" func for the object.
     Py_INCREF(obj);
-    xidatafunc getdata = lookup_getdata(interp, obj);
+    xidatafunc getdata = lookup_getdata(ctx, obj);
     if (getdata == NULL) {
         Py_DECREF(obj);
         if (!PyErr_Occurred()) {
-            _set_xid_lookup_failure(state, obj, NULL);
+            _set_xid_lookup_failure(ctx, obj, NULL);
         }
         return -1;
     }
@@ -969,7 +963,8 @@ _PyXI_ClearExcInfo(_PyXI_excinfo *info)
 static int
 _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
 {
-    _PyXI_state_t *state;
+    dlcontext_t ctx;
+
     assert(!PyErr_Occurred());
     switch (code) {
     case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH;
@@ -1000,8 +995,10 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
                         "failed to apply namespace to __main__");
         break;
     case _PyXI_ERR_NOT_SHAREABLE:
-        state = _PyXI_GET_STATE(interp);
-        _set_xid_lookup_failure(state, NULL, NULL);
+        if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
+            return -1;
+        }
+        _set_xid_lookup_failure(&ctx, NULL, NULL);
         break;
     default:
 #ifdef Py_DEBUG
@@ -1063,8 +1060,11 @@ _PyXI_ApplyError(_PyXI_error *error)
     }
     else if (error->code == _PyXI_ERR_NOT_SHAREABLE) {
         // Propagate the exception directly.
-        _PyXI_state_t *state = _PyXI_GET_STATE(error->interp);
-        _set_xid_lookup_failure(state, NULL, error->uncaught.msg);
+        dlcontext_t ctx;
+        if (_PyXIData_GetLookupContext(error->interp, &ctx) < 0) {
+            return NULL;
+        }
+        _set_xid_lookup_failure(&ctx, NULL, error->uncaught.msg);
     }
     else {
         // Raise an exception corresponding to the code.
@@ -1151,7 +1151,12 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value)
         PyErr_NoMemory();
         return -1;
     }
-    if (_PyObject_GetXIData(value, item->data) != 0) {
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    dlcontext_t ctx;
+    if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
+        return -1;
+    }
+    if (_PyObject_GetXIData(&ctx, value, item->data) != 0) {
         PyMem_RawFree(item->data);
         item->data = NULL;
         // The caller may want to propagate PyExc_NotShareableError
@@ -1609,9 +1614,13 @@ _propagate_not_shareable_error(_PyXI_session *session)
         return;
     }
     PyInterpreterState *interp = PyInterpreterState_Get();
-    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
-    assert(state->exceptions.PyExc_NotShareableError != NULL);
-    if (PyErr_ExceptionMatches(state->exceptions.PyExc_NotShareableError)) {
+    dlcontext_t ctx;
+    if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
+        PyErr_FormatUnraisable(
+                "Exception ignored while propagating not shareable error");
+        return;
+    }
+    if (PyErr_ExceptionMatches(ctx.PyExc_NotShareableError)) {
         // We want to propagate the exception directly.
         session->_error_override = _PyXI_ERR_NOT_SHAREABLE;
         session->error_override = &session->_error_override;
@@ -1779,22 +1788,87 @@ _PyXI_Exit(_PyXI_session *session)
 /* runtime lifecycle */
 /*********************/
 
+int
+_Py_xi_global_state_init(_PyXI_global_state_t *state)
+{
+    assert(state != NULL);
+    xid_lookup_init(&state->data_lookup);
+    return 0;
+}
+
+void
+_Py_xi_global_state_fini(_PyXI_global_state_t *state)
+{
+    assert(state != NULL);
+    xid_lookup_fini(&state->data_lookup);
+}
+
+int
+_Py_xi_state_init(_PyXI_state_t *state, PyInterpreterState *interp)
+{
+    assert(state != NULL);
+    assert(interp == NULL || state == _PyXI_GET_STATE(interp));
+
+    xid_lookup_init(&state->data_lookup);
+
+    // Initialize exceptions.
+    if (interp != NULL) {
+        if (init_static_exctypes(&state->exceptions, interp) < 0) {
+            fini_heap_exctypes(&state->exceptions);
+            return -1;
+        }
+    }
+    if (init_heap_exctypes(&state->exceptions) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+void
+_Py_xi_state_fini(_PyXI_state_t *state, PyInterpreterState *interp)
+{
+    assert(state != NULL);
+    assert(interp == NULL || state == _PyXI_GET_STATE(interp));
+
+    fini_heap_exctypes(&state->exceptions);
+    if (interp != NULL) {
+        fini_static_exctypes(&state->exceptions, interp);
+    }
+
+    xid_lookup_fini(&state->data_lookup);
+}
+
+
 PyStatus
 _PyXI_Init(PyInterpreterState *interp)
 {
-    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
-
-    // Initialize the XID lookup state (e.g. registry).
     if (_Py_IsMainInterpreter(interp)) {
-        xid_lookup_init(&_PyXI_GET_GLOBAL_STATE(interp)->data_lookup);
+        _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp);
+        if (global_state == NULL) {
+            PyErr_PrintEx(0);
+            return _PyStatus_ERR(
+                    "failed to get global cross-interpreter state");
+        }
+        if (_Py_xi_global_state_init(global_state) < 0) {
+            PyErr_PrintEx(0);
+            return _PyStatus_ERR(
+                    "failed to initialize  global cross-interpreter state");
+        }
     }
-    xid_lookup_init(&state->data_lookup);
 
-    // Initialize exceptions.(heap types).
-    // See _PyXI_InitTypes() for the static types.
-    if (init_heap_exctypes(&_PyXI_GET_STATE(interp)->exceptions) < 0) {
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
+    if (state == NULL) {
+        PyErr_PrintEx(0);
+        return _PyStatus_ERR(
+                "failed to get interpreter's cross-interpreter state");
+    }
+    // The static types were already initialized in _PyXI_InitTypes(),
+    // so we pass in NULL here to avoid initializing them again.
+    if (_Py_xi_state_init(state, NULL) < 0) {
         PyErr_PrintEx(0);
-        return _PyStatus_ERR("failed to initialize exceptions");
+        return _PyStatus_ERR(
+                "failed to initialize interpreter's cross-interpreter state");
     }
 
     return _PyStatus_OK();
@@ -1807,15 +1881,19 @@ void
 _PyXI_Fini(PyInterpreterState *interp)
 {
     _PyXI_state_t *state = _PyXI_GET_STATE(interp);
+#ifndef NDEBUG
+    if (state == NULL) {
+        PyErr_PrintEx(0);
+        return;
+    }
+#endif
+    // The static types will be finalized soon in _PyXI_FiniTypes(),
+    // so we pass in NULL here to avoid finalizing them right now.
+    _Py_xi_state_fini(state, NULL);
 
-    // Finalize exceptions (heap types).
-    // See _PyXI_FiniTypes() for the static types.
-    fini_heap_exctypes(&_PyXI_GET_STATE(interp)->exceptions);
-
-    // Finalize the XID lookup state (e.g. registry).
-    xid_lookup_fini(&state->data_lookup);
     if (_Py_IsMainInterpreter(interp)) {
-        xid_lookup_fini(&_PyXI_GET_GLOBAL_STATE(interp)->data_lookup);
+        _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp);
+        _Py_xi_global_state_fini(global_state);
     }
 }
 
@@ -1824,7 +1902,8 @@ _PyXI_InitTypes(PyInterpreterState *interp)
 {
     if (init_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp) < 0) {
         PyErr_PrintEx(0);
-        return _PyStatus_ERR("failed to initialize an exception type");
+        return _PyStatus_ERR(
+                "failed to initialize the cross-interpreter exception types");
     }
     // We would initialize heap types here too but that leads to ref leaks.
     // Instead, we intialize them in _PyXI_Init().