diff options
34 files changed, 1222 insertions, 614 deletions
diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst new file mode 100644 index 00000000000..4c8212f2f5e --- /dev/null +++ b/Doc/c-api/extension-modules.rst @@ -0,0 +1,247 @@ +.. highlight:: c + +.. _extension-modules: + +Defining extension modules +-------------------------- + +A C extension for CPython is a shared library (for example, a ``.so`` file +on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process +(for example, it is compiled with compatible compiler settings), and which +exports an :ref:`initialization function <extension-export-hook>`. + +To be importable by default (that is, by +:py:class:`importlib.machinery.ExtensionFileLoader`), +the shared library must be available on :py:attr:`sys.path`, +and must be named after the module name plus an extension listed in +:py:attr:`importlib.machinery.EXTENSION_SUFFIXES`. + +.. note:: + + Building, packaging and distributing extension modules is best done with + third-party tools, and is out of scope of this document. + One suitable tool is Setuptools, whose documentation can be found at + https://setuptools.pypa.io/en/latest/setuptools.html. + +Normally, the initialization function returns a module definition initialized +using :c:func:`PyModuleDef_Init`. +This allows splitting the creation process into several phases: + +- Before any substantial code is executed, Python can determine which + capabilities the module supports, and it can adjust the environment or + refuse loading an incompatible extension. +- By default, Python itself creates the module object -- that is, it does + the equivalent of :py:meth:`object.__new__` for classes. + It also sets initial attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`. +- Afterwards, the module object is initialized using extension-specific + code -- the equivalent of :py:meth:`~object.__init__` on classes. + +This is called *multi-phase initialization* to distinguish it from the legacy +(but still supported) *single-phase initialization* scheme, +where the initialization function returns a fully constructed module. +See the :ref:`single-phase-initialization section below <single-phase-initialization>` +for details. + +.. versionchanged:: 3.5 + + Added support for multi-phase initialization (:pep:`489`). + + +Multiple module instances +......................... + +By default, extension modules are not singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created, and typically populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters <sub-interpreter-support>` +or after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state owned by the module, including references to Python objects, +should be specific to a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization <isolating-extensions-optout>`. + +All modules are expected to support +:ref:`sub-interpreters <sub-interpreter-support>`, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. + + +.. _extension-export-hook: + +Initialization function +....................... + +The initialization function defined by an extension module has the +following signature: + +.. c:function:: PyObject* PyInit_modulename(void) + +Its name should be :samp:`PyInit_{<name>}`, with ``<name>`` replaced by the +name of the module. + +For modules with ASCII-only names, the function must instead be named +:samp:`PyInit_{<name>}`, with ``<name>`` replaced by the name of the module. +When using :ref:`multi-phase-initialization`, non-ASCII module names +are allowed. In this case, the initialization function name is +:samp:`PyInitU_{<name>}`, with ``<name>`` encoded using Python's +*punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def initfunc_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyInit' + suffix + +It is recommended to define the initialization function using a helper macro: + +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module initialization function. + This macro: + + * specifies the :c:expr:`PyObject*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + +It is possible to export multiple modules from a single shared library by +defining multiple initialization functions. However, importing them requires +using symbolic links or a custom importer, because by default only the +function corresponding to the filename is found. +See the `Multiple modules in one library <https://peps.python.org/pep-0489/#multiple-modules-in-one-library>`__ +section in :pep:`489` for details. + +The initialization function is typically the only non-\ ``static`` +item defined in the module's C source. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +Normally, the :ref:`initialization function <extension-export-hook>` +(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with +non-``NULL`` :c:member:`~PyModuleDef.m_slots`. +Before it is returned, the ``PyModuleDef`` instance must be initialized +using the following function: + + +.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + + Ensure a module definition is a properly initialized Python object that + correctly reports its type and a reference count. + + Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + + Calling this function is required for :ref:`multi-phase-initialization`. + It should not be used in other contexts. + + Note that Python assumes that ``PyModuleDef`` structures are statically + allocated. + This function may return either a new reference or a borrowed one; + this reference must not be released. + + .. versionadded:: 3.5 + + +.. _single-phase-initialization: + +Legacy single-phase initialization +.................................. + +.. attention:: + Single-phase initialization is a legacy mechanism to initialize extension + modules, with known drawbacks and design flaws. Extension module authors + are encouraged to use multi-phase initialization instead. + +In single-phase initialization, the +:ref:`initialization function <extension-export-hook>` (``PyInit_modulename``) +should create, populate and return a module object. +This is typically done using :c:func:`PyModule_Create` and functions like +:c:func:`PyModule_AddObjectRef`. + +Single-phase initialization differs from the :ref:`default <multi-phase-initialization>` +in the following ways: + +* Single-phase modules are, or rather *contain*, “singletons”. + + When the module is first initialized, Python saves the contents of + the module's ``__dict__`` (that is, typically, the module's functions and + types). + + For subsequent imports, Python does not call the initialization function + again. + Instead, it creates a new module object with a new ``__dict__``, and copies + the saved contents to it. + For example, given a single-phase module ``_testsinglephase`` + [#testsinglephase]_ that defines a function ``sum`` and an exception class + ``error``: + + .. code-block:: python + + >>> import sys + >>> import _testsinglephase as one + >>> del sys.modules['_testsinglephase'] + >>> import _testsinglephase as two + >>> one is two + False + >>> one.__dict__ is two.__dict__ + False + >>> one.sum is two.sum + True + >>> one.error is two.error + True + + The exact behavior should be considered a CPython implementation detail. + +* To work around the fact that ``PyInit_modulename`` does not take a *spec* + argument, some state of the import machinery is saved and applied to the + first suitable module created during the ``PyInit_modulename`` call. + Specifically, when a sub-module is imported, this mechanism prepends the + parent package name to the name of the module. + + A single-phase ``PyInit_modulename`` function should create “its” module + object as soon as possible, before any other module objects can be created. + +* Non-ASCII module names (``PyInitU_modulename``) are not supported. + +* Single-phase modules support module lookup functions like + :c:func:`PyState_FindModule`. + +.. [#testsinglephase] ``_testsinglephase`` is an internal module used \ + in CPython's self-test suite; your installation may or may not \ + include it. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index ba56b03c6ac..e9df2a304d9 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -17,6 +17,7 @@ document the API functions in detail. veryhigh.rst refcounting.rst exceptions.rst + extension-modules.rst utilities.rst abstract.rst concrete.rst diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 41856922110..acce3dc215d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -111,33 +111,11 @@ Useful macros ============= Several useful macros are defined in the Python header files. Many are -defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). +defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, +:c:macro:`PyMODINIT_FUNC`). Others of a more general utility are defined here. This is not necessarily a complete listing. -.. c:macro:: PyMODINIT_FUNC - - Declare an extension module ``PyInit`` initialization function. The function - return type is :c:expr:`PyObject*`. The macro declares any special linkage - declarations required by the platform, and for C++ declares the function as - ``extern "C"``. - - The initialization function must be named :samp:`PyInit_{name}`, where - *name* is the name of the module, and should be the only non-\ ``static`` - item defined in the module file. Example:: - - static struct PyModuleDef spam_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - .. c:macro:: Py_ABS(x) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 710135dca89..c8edcecc5b4 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -127,25 +127,36 @@ Module Objects unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _initializing-modules: +.. _pymoduledef: -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ +Module definitions +------------------ -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. +The functions in the previous section work on any module object, including +modules imported from Python code. -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. +Modules defined using the C API typically use a *module definition*, +:c:type:`PyModuleDef` -- a statically allocated, constant “description" of +how a module should be created. + +The definition is usually used to define an extension's “main” module object +(see :ref:`extension-modules` for details). +It is also used to +:ref:`create extension modules dynamically <moduledef-dynamic>`. + +Unlike :c:func:`PyModule_New`, the definition allows management of +*module state* -- a piece of memory that is allocated and cleared together +with the module object. +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. .. c:type:: PyModuleDef The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. + a module object. + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module. .. c:member:: PyModuleDef_Base m_base @@ -170,13 +181,15 @@ or request "multi-phase initialization" by returning the definition struct itsel and freed when the module object is deallocated, after the :c:member:`~PyModuleDef.m_free` function has been called, if present. - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Setting it to a non-negative value means that the module can be re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. + for its state. + + Setting ``m_size`` to ``-1`` means that the module does not support + sub-interpreters, because it has global state. + Negative ``m_size`` is only allowed when using + :ref:`legacy single-phase initialization <single-phase-initialization>` + or when :ref:`creating modules dynamically <moduledef-dynamic>`. See :PEP:`3121` for more details. @@ -189,7 +202,7 @@ or request "multi-phase initialization" by returning the definition struct itsel An array of slot definitions for multi-phase initialization, terminated by a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be ``NULL``. + When using legacy single-phase initialization, *m_slots* must be ``NULL``. .. versionchanged:: 3.5 @@ -249,96 +262,9 @@ or request "multi-phase initialization" by returning the definition struct itsel .. versionchanged:: 3.9 No longer called before the module state is allocated. -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: - -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :c:macro:`PYTHON_API_VERSION`. - - -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - - Return ``NULL`` with an exception set on error. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`~object.__new__` and -:py:meth:`~object.__init__` methods of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons. -For example, if the :py:attr:`sys.modules` entry is removed and the module -is re-imported, a new module object is created, and typically populated with -fresh method and type objects. -The old module is subject to normal garbage collection. -This mirrors the behavior of pure-Python modules. - -Additional module instances may be created in -:ref:`sub-interpreters <sub-interpreter-support>` -or after after Python runtime reinitialization -(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). -In these cases, sharing Python objects between module instances would likely -cause crashes or undefined behavior. - -To avoid such issues, each instance of an extension module should -be *isolated*: changes to one instance should not implicitly affect the others, -and all state, including references to Python objects, should be specific to -a particular module instance. -See :ref:`isolating-extensions-howto` for more details and a practical guide. - -A simpler way to avoid these issues is -:ref:`raising an error on repeated initialization <isolating-extensions-optout>`. - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters <sub-interpreter-support>`, or otherwise explicitly -signal a lack of support. -This is usually achieved by isolation or blocking repeated initialization, -as above. -A module may also be limited to the main interpreter using -the :c:data:`Py_mod_multiple_interpreters` slot. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: +Module slots +............ .. c:type:: PyModuleDef_Slot @@ -352,8 +278,6 @@ The *m_slots* member of the module definition must point to an array of .. versionadded:: 3.5 -The *m_slots* array must be terminated by a slot with id 0. - The available slot types are: .. c:macro:: Py_mod_create @@ -464,21 +388,48 @@ The available slot types are: .. versionadded:: 3.13 -See :PEP:`489` for more details on multi-phase initialization. -Low-level module creation functions -................................... +.. _moduledef-dynamic: + +Creating extension modules dynamically +-------------------------------------- + +The following functions may be used to create a module outside of an +extension's :ref:`initialization function <extension-export-hook>`. +They are also used in +:ref:`single-phase initialization <single-phase-initialization>`. + +.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) + + Create a new module object, given the definition in *def*. + This is a macro that calls :c:func:`PyModule_Create2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API <limited-c-api>`. + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) + + Create a new module object, given the definition in *def*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + Return ``NULL`` with an exception set on error. + + This function does not support slots. + The :c:member:`~PyModuleDef.m_slots` member of *def* must be ``NULL``. + -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. + This macro calls :c:func:`PyModule_FromDefAndSpec2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API <limited-c-api>`. .. versionadded:: 3.5 @@ -491,6 +442,10 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and Return ``NULL`` with an exception set on error. + Note that this does not process execution slots (:c:data:`Py_mod_exec`). + Both ``PyModule_FromDefAndSpec`` and ``PyModule_ExecDef`` must be called + to fully initialize a module. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -504,35 +459,29 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. versionadded:: 3.5 -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) +.. c:macro:: PYTHON_API_VERSION - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + The C API version. Defined for backwards compatibility. - .. versionadded:: 3.5 + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) +.. c:macro:: PYTHON_ABI_VERSION - Add the functions from the ``NULL`` terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + Defined as ``3`` for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. - .. versionadded:: 3.5 Support functions -................. +----------------- -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: +The following functions are provided to help initialize a module +state. +They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +the initialization function for legacy :ref:`single-phase initialization <single-phase-initialization>`, +or code that creates modules dynamically. .. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) @@ -681,12 +630,39 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) + + Add the functions from the ``NULL`` terminated *functions* array to *module*. + Refer to the :c:type:`PyMethodDef` documentation for details on individual + entries (due to the lack of a shared module namespace, module level + "functions" implemented in C typically receive the module as their first + parameter, making them similar to instance methods on Python classes). + + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Some module authors may prefer defining functions in multiple + :c:type:`PyMethodDef` arrays; in that case they should call this function + directly. + + .. versionadded:: 3.5 + +.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) + + Set the docstring for *module* to *docstring*. + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + + .. versionadded:: 3.5 + .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) Indicate that *module* does or does not support running without the global interpreter lock (GIL), using one of the values from :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization - function. If this function is not called during module initialization, the + function when using :ref:`single-phase-initialization`. + If this function is not called during module initialization, the import machinery assumes the module does not support running without the GIL. This function is only available in Python builds configured with :option:`--disable-gil`. @@ -695,10 +671,11 @@ state: .. versionadded:: 3.13 -Module lookup -^^^^^^^^^^^^^ +Module lookup (single-phase initialization) +........................................... -Single-phase initialization creates singleton modules that can be looked up +The legacy :ref:`single-phase initialization <single-phase-initialization>` +initialization scheme creates singleton modules that can be looked up in the context of the current interpreter. This allows the module object to be retrieved later with only a reference to the module definition. @@ -719,7 +696,8 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. - Python calls ``PyState_AddModule`` automatically after importing a module, + Python calls ``PyState_AddModule`` automatically after importing a module + that uses :ref:`single-phase initialization <single-phase-initialization>`, so it is unnecessary (but harmless) to call it from module initialization code. An explicit call is needed only if the module's own init code subsequently calls ``PyState_FindModule``. @@ -727,6 +705,9 @@ since multiple such modules can be created from a single definition. mechanisms (either by calling it directly, or by referring to its implementation for details of the required state updates). + If a module was attached previously using the same *def*, it is replaced + by the new *module*. + The caller must have an :term:`attached thread state`. Return ``-1`` with an exception set on error, ``0`` on success. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 59b31ccf7bc..f5f02f0a79c 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1489,9 +1489,6 @@ PyModule_SetDocString:int::: PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: -PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:: - PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index a58eb40d431..098dde39ea5 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -6,41 +6,10 @@ Building C and C++ Extensions ***************************** -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. +A C extension for CPython is a shared library (for example, a ``.so`` file on +Linux, ``.pyd`` on Windows), which exports an *initialization function*. -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using setuptools, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlight:: python - -For modules with ASCII-only names, the function must be named -:samp:`PyInit_{<name>}`, with ``<name>`` replaced by the name of the module. -When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -:samp:`PyInitU_{<name>}`, with ``<name>`` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. +See :ref:`extension-modules` for details. .. highlight:: c @@ -51,7 +20,11 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. Building C and C++ Extensions with setuptools ============================================= -Python 3.12 and newer no longer come with distutils. Please refer to the -``setuptools`` documentation at -https://setuptools.readthedocs.io/en/latest/setuptools.html -to learn more about how build and distribute C/C++ extensions with setuptools. + +Building, packaging and distributing extension modules is best done with +third-party tools, and is out of scope of this document. +One suitable tool is Setuptools, whose documentation can be found at +https://setuptools.pypa.io/en/latest/setuptools.html. + +The :mod:`distutils` module, which was included in the standard library +until Python 3.12, is now maintained as part of Setuptools. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index c8061fb1638..ecb1d4102ca 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -387,8 +387,8 @@ Floating point manipulation functions .. function:: issubnormal(x) Return ``True`` if *x* is a subnormal number, that is a finite - nonzero number with a magnitude smaller than the smallest positive normal - number, see :data:`sys.float_info.min`. Return ``False`` otherwise. + nonzero number with a magnitude smaller than :data:`sys.float_info.min`. + Return ``False`` otherwise. .. versionadded:: next diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index b75e5ceecf8..394c302fd35 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1841,6 +1841,14 @@ expression support in the :mod:`re` module). unless an encoding error actually occurs, :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. + For example:: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + <class 'bytes'> + >>> encoded_str_to_bytes + b'Python' + .. versionchanged:: 3.1 Added support for keyword arguments. @@ -1855,7 +1863,19 @@ expression support in the :mod:`re` module). Return ``True`` if the string ends with the specified *suffix*, otherwise return ``False``. *suffix* can also be a tuple of suffixes to look for. With optional *start*, test beginning at that position. With optional *end*, stop comparing - at that position. + at that position. Using *start* and *end* is equivalent to + ``str[start:end].endswith(suffix)``. For example:: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also :meth:`startswith` and :meth:`removesuffix`. .. method:: str.expandtabs(tabsize=8) @@ -1871,12 +1891,15 @@ expression support in the :mod:`re` module). (``\n``) or return (``\r``), it is copied and the current column is reset to zero. Any other character is copied unchanged and the current column is incremented by one regardless of how the character is represented when - printed. + printed. For example:: >>> '01\t012\t0123\t01234'.expandtabs() '01 012 0123 01234' >>> '01\t012\t0123\t01234'.expandtabs(4) '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 .. method:: str.find(sub[, start[, end]]) diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 747ee3ee0e1..92d58024e84 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -257,7 +257,7 @@ The :mod:`uuid` module defines the following functions: non-specified arguments are substituted for a pseudo-random integer of appropriate size. - By default, *a*, *b* and *c* are generated by a non-cryptographically + By default, *a*, *b* and *c* are not generated by a cryptographically secure pseudo-random number generator (CSPRNG). Use :func:`uuid4` when a UUID needs to be used in a security-sensitive context. diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 713ddc66ba7..81faffac194 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -303,10 +303,10 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; -PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(_PyXI_excinfo *) _PyXI_NewExcInfo(PyObject *exc); +PyAPI_FUNC(void) _PyXI_FreeExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); -PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); typedef enum error_code { @@ -322,19 +322,20 @@ typedef enum error_code { _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; +typedef struct xi_failure _PyXI_failure; -typedef struct _sharedexception { - // The originating interpreter. - PyInterpreterState *interp; - // The kind of error to propagate. - _PyXI_errcode code; - // The exception information to propagate, if applicable. - // This is populated only for some error codes, - // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. - _PyXI_excinfo uncaught; -} _PyXI_error; +PyAPI_FUNC(_PyXI_failure *) _PyXI_NewFailure(void); +PyAPI_FUNC(void) _PyXI_FreeFailure(_PyXI_failure *); +PyAPI_FUNC(_PyXI_errcode) _PyXI_GetFailureCode(_PyXI_failure *); +PyAPI_FUNC(int) _PyXI_InitFailure(_PyXI_failure *, _PyXI_errcode, PyObject *); +PyAPI_FUNC(void) _PyXI_InitFailureUTF8( + _PyXI_failure *, + _PyXI_errcode, + const char *); -PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); +PyAPI_FUNC(int) _PyXI_UnwrapNotShareableError( + PyThreadState *, + _PyXI_failure *); // A cross-interpreter session involves entering an interpreter @@ -366,19 +367,21 @@ PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Exit( _PyXI_session *, - _PyXI_errcode, + _PyXI_failure *, _PyXI_session_result *); PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( _PyXI_session *, - _PyXI_errcode *); + _PyXI_failure *); PyAPI_FUNC(int) _PyXI_Preserve( _PyXI_session *, const char *, PyObject *, - _PyXI_errcode *); -PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); + _PyXI_failure *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved( + _PyXI_session_result *, + const char *); /*************/ diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4828dfd948f..59beb92f3f7 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -16,6 +16,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_complexes_MAXFREELIST 100 # define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_ranges_MAXFREELIST 6 @@ -43,6 +44,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist complexes; struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 1e30956fe24..ba416e7fa6e 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -27,6 +27,7 @@ charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('</[a-zA-Z]') piclose = re.compile('>') commentclose = re.compile(r'--\s*>') # Note: @@ -195,7 +196,7 @@ class HTMLParser(_markupbase.ParserBase): k = self.parse_pi(i) elif startswith("<!", i): k = self.parse_html_declaration(i) - elif (i + 1) < n: + elif (i + 1) < n or end: self.handle_data("<") k = i + 1 else: @@ -203,17 +204,35 @@ class HTMLParser(_markupbase.ParserBase): if k < 0: if not end: break - k = rawdata.find('>', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 - else: - k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("</", i): + if i + 2 == n: + self.handle_data("</") + elif endtagopen.match(rawdata, i): # </ + letter + pass + else: + # bogus comment + self.handle_comment(rawdata[i+2:]) + elif startswith("<!--", i): + j = n + for suffix in ("--!", "--", "-"): + if rawdata.endswith(suffix, i+4): + j -= len(suffix) + break + self.handle_comment(rawdata[i+4:j]) + elif startswith("<![CDATA[", i): + self.unknown_decl(rawdata[i+3:]) + elif rawdata[i:i+9].lower() == '<!doctype': + self.handle_decl(rawdata[i+2:]) + elif startswith("<!", i): + # bogus comment + self.handle_comment(rawdata[i+2:]) + elif startswith("<?", i): + self.handle_pi(rawdata[i+2:]) else: - self.handle_data(rawdata[i:k]) + raise AssertionError("we should not get here!") + k = n i = self.updatepos(i, k) elif startswith("&#", i): match = charref.match(rawdata, i) diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index 274beb65a6d..70c44da2ec6 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -57,6 +57,13 @@ def spam_with_globals_and_builtins(): print(res) +def spam_with_global_and_attr_same_name(): + try: + spam_minimal.spam_minimal + except AttributeError: + pass + + def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): return (a, b, c, d, e, f, args, kwargs) @@ -190,6 +197,7 @@ TOP_FUNCTIONS = [ spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, spam_full_args, spam_full_args_with_defaults, spam_args_attrs_and_builtins, @@ -258,6 +266,7 @@ STATELESS_CODE = [ script_with_globals, spam_full_args_with_defaults, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, spam_full, ] @@ -275,6 +284,7 @@ SCRIPT_FUNCTIONS = [ *PURE_SCRIPT_FUNCTIONS, script_with_globals, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, ] diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 9fc2b047bef..655f5a9be7f 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -701,6 +701,7 @@ class CodeTest(unittest.TestCase): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_with_global_and_attr_same_name: {}, defs.spam_full_args: { 'a': POSONLY, 'b': POSONLY, @@ -955,6 +956,10 @@ class CodeTest(unittest.TestCase): purelocals=5, globalvars=6, ), + defs.spam_with_global_and_attr_same_name: new_var_counts( + globalvars=2, + attrs=1, + ), defs.spam_full_args: new_var_counts( posonly=2, posorkw=2, diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 61fa24fab57..65a4bee72b9 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -5,6 +5,7 @@ import pprint import unittest from unittest.mock import patch +from test import support class EventCollector(html.parser.HTMLParser): @@ -430,28 +431,34 @@ text ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('endtag', 'a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -576,21 +583,50 @@ text for html, expected in data: self._run_check(html, expected) - def test_EOF_in_comments_or_decls(self): + def test_eof_in_comments(self): data = [ - ('<!', [('data', '<!')]), - ('<!-', [('data', '<!-')]), - ('<!--', [('data', '<!--')]), - ('<![', [('data', '<![')]), - ('<![CDATA[', [('data', '<![CDATA[')]), - ('<![CDATA[x', [('data', '<![CDATA[x')]), - ('<!DOCTYPE', [('data', '<!DOCTYPE')]), - ('<!DOCTYPE HTML', [('data', '<!DOCTYPE HTML')]), + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), ] for html, expected in data: self._run_check(html, expected) + + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<![CDATA[', [('unknown decl', 'CDATA[')]), + ('<![CDATA[x', [('unknown decl', 'CDATA[x')]), + ('<![CDATA[x]', [('unknown decl', 'CDATA[x]')]), + ('<![CDATA[x]]', [('unknown decl', 'CDATA[x]]')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + def test_bogus_comments(self): - html = ('<! not really a comment >' + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' @@ -604,6 +640,7 @@ text '<![CDATA]]>' # required '[' after CDATA ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), @@ -684,6 +721,26 @@ text ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 249e0f3ba32..63998a86c45 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -5,7 +5,6 @@ BytesIO -- for bytes import unittest from test import support -from test.support import threading_helper import gc import io @@ -13,7 +12,6 @@ import _pyio as pyio import pickle import sys import weakref -import threading class IntLike: def __init__(self, num): @@ -725,22 +723,6 @@ class TextIOTestMixin: for newline in (None, "", "\n", "\r", "\r\n"): self.ioclass(newline=newline) - @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful under free-threading") - @threading_helper.requires_working_threading() - def test_concurrent_use(self): - memio = self.ioclass("") - - def use(): - memio.write("x" * 10) - memio.readlines() - - threads = [threading.Thread(target=use) for _ in range(8)] - with threading_helper.catch_threading_exception() as cm: - with threading_helper.start_threads(threads): - pass - - self.assertIsNone(cm.exc_value) - class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin, TextIOTestMixin, unittest.TestCase): @@ -908,7 +890,6 @@ class CStringIOTest(PyStringIOTest): self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) - class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst b/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst new file mode 100644 index 00000000000..3eb13cefbe6 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst @@ -0,0 +1 @@ +Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` diff --git a/Misc/NEWS.d/next/Library/2025-06-11-19-05-49.gh-issue-135410.E89Boi.rst b/Misc/NEWS.d/next/Library/2025-06-11-19-05-49.gh-issue-135410.E89Boi.rst deleted file mode 100644 index a5917fba3f7..00000000000 --- a/Misc/NEWS.d/next/Library/2025-06-11-19-05-49.gh-issue-135410.E89Boi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash when iterating over :class:`io.StringIO` on the :term:`free -threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst b/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst new file mode 100644 index 00000000000..cf9aa8dbdf2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst @@ -0,0 +1,4 @@ +Fix quadratic complexity in processing specially crafted input in +:class:`html.parser.HTMLParser`. End-of-file errors are now handled according +to the HTML5 specs -- comments and declarations are automatically closed, +tags are ignored. diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 037e9544543..fdfb3e6dd34 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -80,21 +80,11 @@ is_notshareable_raised(PyThreadState *tstate) } static void -unwrap_not_shareable(PyThreadState *tstate) +unwrap_not_shareable(PyThreadState *tstate, _PyXI_failure *failure) { - if (!is_notshareable_raised(tstate)) { - return; - } - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *cause = PyException_GetCause(exc); - if (cause != NULL) { - Py_DECREF(exc); - exc = cause; + if (_PyXI_UnwrapNotShareableError(tstate, failure) < 0) { + _PyErr_Clear(tstate); } - else { - assert(PyException_GetContext(exc) == NULL); - } - _PyErr_SetRaisedException(tstate, exc); } @@ -532,13 +522,30 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, return 0; } +static void +wrap_notshareable(PyThreadState *tstate, const char *label) +{ + if (!is_notshareable_raised(tstate)) { + return; + } + assert(label != NULL && strlen(label) > 0); + PyObject *cause = _PyErr_GetRaisedException(tstate); + _PyXIData_FormatNotShareableError(tstate, "%s not shareable", label); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); + _PyErr_SetRaisedException(tstate, exc); +} + static int _interp_call_unpack(struct interp_call *call, PyObject **p_func, PyObject **p_args, PyObject **p_kwargs) { + PyThreadState *tstate = PyThreadState_Get(); + // Unpack the func. PyObject *func = _PyXIData_NewObject(call->func); if (func == NULL) { + wrap_notshareable(tstate, "func"); return -1; } // Unpack the args. @@ -553,6 +560,7 @@ _interp_call_unpack(struct interp_call *call, else { args = _PyXIData_NewObject(call->args); if (args == NULL) { + wrap_notshareable(tstate, "args"); Py_DECREF(func); return -1; } @@ -563,6 +571,7 @@ _interp_call_unpack(struct interp_call *call, if (call->kwargs != NULL) { kwargs = _PyXIData_NewObject(call->kwargs); if (kwargs == NULL) { + wrap_notshareable(tstate, "kwargs"); Py_DECREF(func); Py_DECREF(args); return -1; @@ -577,7 +586,7 @@ _interp_call_unpack(struct interp_call *call, static int _make_call(struct interp_call *call, - PyObject **p_result, _PyXI_errcode *p_errcode) + PyObject **p_result, _PyXI_failure *failure) { assert(call != NULL && call->func != NULL); PyThreadState *tstate = _PyThreadState_GET(); @@ -588,12 +597,10 @@ _make_call(struct interp_call *call, assert(func == NULL); assert(args == NULL); assert(kwargs == NULL); - *p_errcode = is_notshareable_raised(tstate) - ? _PyXI_ERR_NOT_SHAREABLE - : _PyXI_ERR_OTHER; + _PyXI_InitFailure(failure, _PyXI_ERR_OTHER, NULL); + unwrap_not_shareable(tstate, failure); return -1; } - *p_errcode = _PyXI_ERR_NO_ERROR; // Make the call. PyObject *resobj = PyObject_Call(func, args, kwargs); @@ -608,17 +615,17 @@ _make_call(struct interp_call *call, } static int -_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_errcode *p_errcode) +_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_failure *failure) { PyObject *code = _PyXIData_NewObject(script); if (code == NULL) { - *p_errcode = _PyXI_ERR_NOT_SHAREABLE; + _PyXI_InitFailure(failure, _PyXI_ERR_NOT_SHAREABLE, NULL); return -1; } PyObject *result = PyEval_EvalCode(code, ns, ns); Py_DECREF(code); if (result == NULL) { - *p_errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + _PyXI_InitFailure(failure, _PyXI_ERR_UNCAUGHT_EXCEPTION, NULL); return -1; } assert(result == Py_None); @@ -644,8 +651,14 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, PyObject *shareables, struct run_result *runres) { assert(!_PyErr_Occurred(tstate)); + int res = -1; + _PyXI_failure *failure = _PyXI_NewFailure(); + if (failure == NULL) { + return -1; + } _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { + _PyXI_FreeFailure(failure); return -1; } _PyXI_session_result result = {0}; @@ -655,43 +668,44 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, // If an error occured at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); assert(result.excinfo == NULL); return -1; } // Run in the interpreter. - int res = -1; - _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; if (script != NULL) { assert(call == NULL); - PyObject *mainns = _PyXI_GetMainNamespace(session, &errcode); + PyObject *mainns = _PyXI_GetMainNamespace(session, failure); if (mainns == NULL) { goto finally; } - res = _run_script(script, mainns, &errcode); + res = _run_script(script, mainns, failure); } else { assert(call != NULL); PyObject *resobj; - res = _make_call(call, &resobj, &errcode); + res = _make_call(call, &resobj, failure); if (res == 0) { - res = _PyXI_Preserve(session, "resobj", resobj, &errcode); + res = _PyXI_Preserve(session, "resobj", resobj, failure); Py_DECREF(resobj); if (res < 0) { goto finally; } } } - int exitres; finally: // Clean up and switch back. - exitres = _PyXI_Exit(session, errcode, &result); + (void)res; + int exitres = _PyXI_Exit(session, failure, &result); assert(res == 0 || exitres != 0); _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); res = exitres; if (_PyErr_Occurred(tstate)) { + // It's a directly propagated exception. assert(res < 0); } else if (res < 0) { @@ -1064,7 +1078,7 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) // Clean up and switch back. assert(!PyErr_Occurred()); - int res = _PyXI_Exit(session, _PyXI_ERR_NO_ERROR, NULL); + int res = _PyXI_Exit(session, NULL, NULL); _PyXI_FreeSession(session); assert(res == 0); if (res < 0) { @@ -1124,7 +1138,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) // global variables. They will be resolved against __main__. _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + unwrap_not_shareable(tstate, NULL); return NULL; } @@ -1188,7 +1202,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { - unwrap_not_shareable(tstate); + unwrap_not_shareable(tstate, NULL); return NULL; } @@ -1251,7 +1265,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + unwrap_not_shareable(tstate, NULL); return NULL; } @@ -1542,16 +1556,16 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *captured = NULL; - _PyXI_excinfo info = {0}; - if (_PyXI_InitExcInfo(&info, exc) < 0) { + _PyXI_excinfo *info = _PyXI_NewExcInfo(exc); + if (info == NULL) { goto finally; } - captured = _PyXI_ExcInfoAsObject(&info); + captured = _PyXI_ExcInfoAsObject(info); if (captured == NULL) { goto finally; } - PyObject *formatted = _PyXI_FormatExcInfo(&info); + PyObject *formatted = _PyXI_FormatExcInfo(info); if (formatted == NULL) { Py_CLEAR(captured); goto finally; @@ -1564,7 +1578,7 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } finally: - _PyXI_ClearExcInfo(&info); + _PyXI_FreeExcInfo(info); if (exc != exc_arg) { if (PyErr_Occurred()) { PyErr_SetRaisedException(exc); diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 8482c268176..56913fafefb 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -404,7 +404,7 @@ _io_StringIO_readline_impl(stringio *self, Py_ssize_t size) } static PyObject * -stringio_iternext_lock_held(PyObject *op) +stringio_iternext(PyObject *op) { PyObject *line; stringio *self = stringio_CAST(op); @@ -441,16 +441,6 @@ stringio_iternext_lock_held(PyObject *op) return line; } -static PyObject * -stringio_iternext(PyObject *op) -{ - PyObject *res; - Py_BEGIN_CRITICAL_SECTION(op); - res = stringio_iternext_lock_held(op); - Py_END_CRITICAL_SECTION(); - return res; -} - /*[clinic input] @critical_section _io.StringIO.truncate diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ee869d991d9..34b50ef97d5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1714,7 +1714,7 @@ static int identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, PyObject *globalnames, PyObject *attrnames, PyObject *globalsns, PyObject *builtinsns, - struct co_unbound_counts *counts) + struct co_unbound_counts *counts, int *p_numdupes) { // This function is inspired by inspect.getclosurevars(). // It would be nicer if we had something similar to co_localspluskinds, @@ -1729,6 +1729,7 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, assert(builtinsns == NULL || PyDict_Check(builtinsns)); assert(counts == NULL || counts->total == 0); struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_ssize_t len = Py_SIZE(co); for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); @@ -1747,6 +1748,12 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(attrnames, name) < 0) { return -1; } + if (PySet_Contains(globalnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } else if (inst.op.code == LOAD_GLOBAL) { int oparg = GET_OPARG(co, i, inst.op.arg); @@ -1778,11 +1785,20 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(globalnames, name) < 0) { return -1; } + if (PySet_Contains(attrnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } } if (counts != NULL) { *counts = unbound; } + if (p_numdupes != NULL) { + *p_numdupes = numdupes; + } return 0; } @@ -1932,20 +1948,24 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, // Fill in unbound.globals and unbound.numattrs. struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_BEGIN_CRITICAL_SECTION(co); res = identify_unbound_names( tstate, co, globalnames, attrnames, globalsns, builtinsns, - &unbound); + &unbound, &numdupes); Py_END_CRITICAL_SECTION(); if (res < 0) { goto finally; } assert(unbound.numunknown == 0); - assert(unbound.total <= counts->unbound.total); + assert(unbound.total - numdupes <= counts->unbound.total); assert(counts->unbound.numunknown == counts->unbound.total); - unbound.numunknown = counts->unbound.total - unbound.total; - unbound.total = counts->unbound.total; + // There may be a name that is both a global and an attr. + int totalunbound = counts->unbound.total + numdupes; + unbound.numunknown = totalunbound - unbound.total; + unbound.total = totalunbound; counts->unbound = unbound; + counts->total += numdupes; res = 0; finally: diff --git a/Objects/complexobject.c b/Objects/complexobject.c index c2dd320ae73..b66ebe131ae 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1,4 +1,3 @@ - /* Complex object implementation */ /* Borrows heavily from floatobject.c */ @@ -9,6 +8,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() #include "pycore_floatobject.h" // _Py_convert_int_to_double() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() @@ -410,16 +410,32 @@ complex_subtype_from_c_complex(PyTypeObject *type, Py_complex cval) PyObject * PyComplex_FromCComplex(Py_complex cval) { - /* Inline PyObject_New */ - PyComplexObject *op = PyObject_Malloc(sizeof(PyComplexObject)); + PyComplexObject *op = _Py_FREELIST_POP(PyComplexObject, complexes); + if (op == NULL) { - return PyErr_NoMemory(); + /* Inline PyObject_New */ + op = PyObject_Malloc(sizeof(PyComplexObject)); + if (op == NULL) { + return PyErr_NoMemory(); + } + _PyObject_Init((PyObject*)op, &PyComplex_Type); } - _PyObject_Init((PyObject*)op, &PyComplex_Type); op->cval = cval; return (PyObject *) op; } +static void +complex_dealloc(PyObject *op) +{ + assert(PyComplex_Check(op)); + if (PyComplex_CheckExact(op)) { + _Py_FREELIST_FREE(complexes, op, PyObject_Free); + } + else { + Py_TYPE(op)->tp_free(op); + } +} + static PyObject * complex_subtype_from_doubles(PyTypeObject *type, double real, double imag) { @@ -1383,7 +1399,7 @@ PyTypeObject PyComplex_Type = { "complex", sizeof(PyComplexObject), 0, - 0, /* tp_dealloc */ + complex_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/object.c b/Objects/object.c index 9fe61ba7f15..eff3a986212 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -925,6 +925,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); + clear_freelist(&freelists->complexes, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } diff --git a/PC/layout/main.py b/PC/layout/main.py index 7324a135133..8543e7c56e1 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -247,9 +247,15 @@ def get_layout(ns): if ns.include_freethreaded: yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") - else: + elif (VER_MAJOR, VER_MINOR) > (3, 12): yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") + else: + # Older versions of venv expected the scripts to be named 'python' + # and they were renamed at this stage. We need to replicate that + # when packaging older versions. + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") if ns.include_tools: @@ -652,15 +658,6 @@ def main(): ns.doc_build = (Path.cwd() / ns.doc_build).resolve() if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() - if not ns.arch: - # TODO: Calculate arch from files in ns.build instead - if sys.winver.endswith("-arm64"): - ns.arch = "arm64" - elif sys.winver.endswith("-32"): - ns.arch = "win32" - else: - ns.arch = "amd64" - if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): @@ -668,6 +665,17 @@ def main(): configure_logger(ns) + if not ns.arch: + from .support.arch import calculate_from_build_dir + ns.arch = calculate_from_build_dir(ns.build) + + expect = f"{VER_MAJOR}.{VER_MINOR}.{VER_MICRO}{VER_SUFFIX}" + actual = check_patchlevel_version(ns.source) + if actual and actual != expect: + log_error(f"Inferred version {expect} does not match {actual} from patchlevel.h. " + "You should set %PYTHONINCLUDE% or %PYTHON_HEXVERSION% before launching.") + return 5 + log_info( """OPTIONS Source: {ns.source} diff --git a/PC/layout/support/arch.py b/PC/layout/support/arch.py new file mode 100644 index 00000000000..daf4efbc7ab --- /dev/null +++ b/PC/layout/support/arch.py @@ -0,0 +1,34 @@ +from struct import unpack +from .constants import * +from .logging import * + +def calculate_from_build_dir(root): + candidates = [ + root / PYTHON_DLL_NAME, + root / FREETHREADED_PYTHON_DLL_NAME, + *root.glob("*.dll"), + *root.glob("*.pyd"), + # Check EXE last because it's easier to have cross-platform EXE + *root.glob("*.exe"), + ] + + ARCHS = { + b"PE\0\0\x4c\x01": "win32", + b"PE\0\0\x64\x86": "amd64", + b"PE\0\0\x64\xAA": "arm64" + } + + first_exc = None + for pe in candidates: + try: + # Read the PE header to grab the machine type + with open(pe, "rb") as f: + f.seek(0x3C) + offset = int.from_bytes(f.read(4), "little") + f.seek(offset) + arch = ARCHS[f.read(6)] + except (FileNotFoundError, PermissionError, LookupError) as ex: + log_debug("Failed to open {}: {}", pe, ex) + continue + log_info("Inferred architecture {} from {}", arch, pe) + return arch diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index ae22aa16ebf..6b8c915e519 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -6,6 +6,8 @@ __author__ = "Steve Dower <steve.dower@python.org>" __version__ = "3.8" import os +import pathlib +import re import struct import sys @@ -13,9 +15,15 @@ import sys def _unpack_hexversion(): try: hexversion = int(os.getenv("PYTHON_HEXVERSION"), 16) + return struct.pack(">i", hexversion) except (TypeError, ValueError): - hexversion = sys.hexversion - return struct.pack(">i", hexversion) + pass + if os.getenv("PYTHONINCLUDE"): + try: + return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) + except OSError: + pass + return struct.pack(">i", sys.hexversion) def _get_suffix(field4): @@ -26,6 +34,39 @@ def _get_suffix(field4): return "" +def _read_patchlevel_version(sources): + if not sources.match("Include"): + sources /= "Include" + values = {} + with open(sources / "patchlevel.h", "r", encoding="utf-8") as f: + for line in f: + m = re.match(r'#\s*define\s+(PY_\S+?)\s+(\S+)', line.strip(), re.I) + if m and m.group(2): + v = m.group(2) + if v.startswith('"'): + v = v[1:-1] + else: + v = values.get(v, v) + if isinstance(v, str): + try: + v = int(v, 16 if v.startswith("0x") else 10) + except ValueError: + pass + values[m.group(1)] = v + return ( + values["PY_MAJOR_VERSION"], + values["PY_MINOR_VERSION"], + values["PY_MICRO_VERSION"], + values["PY_RELEASE_LEVEL"] << 4 | values["PY_RELEASE_SERIAL"], + ) + + +def check_patchlevel_version(sources): + got = _read_patchlevel_version(sources) + if got != (VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4): + return f"{got[0]}.{got[1]}.{got[2]}{_get_suffix(got[3])}" + + VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 5e73ab28f2b..39c7ea69890 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1324,6 +1324,12 @@ _excinfo_normalize_type(struct _excinfo_type *info, *p_module = module; } +static int +excinfo_is_set(_PyXI_excinfo *info) +{ + return info->type.name != NULL || info->msg != NULL; +} + static void _PyXI_excinfo_clear(_PyXI_excinfo *info) { @@ -1485,6 +1491,11 @@ _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) if (tbexc == NULL) { PyErr_Clear(); } + else { + PyErr_SetObject(exctype, tbexc); + Py_DECREF(tbexc); + return; + } } PyObject *formatted = _PyXI_excinfo_format(info); @@ -1630,13 +1641,17 @@ error: } -int -_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +_PyXI_excinfo * +_PyXI_NewExcInfo(PyObject *exc) { assert(!PyErr_Occurred()); if (exc == NULL || exc == Py_None) { PyErr_SetString(PyExc_ValueError, "missing exc"); - return -1; + return NULL; + } + _PyXI_excinfo *info = PyMem_RawCalloc(1, sizeof(_PyXI_excinfo)); + if (info == NULL) { + return NULL; } const char *failure; if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { @@ -1646,10 +1661,18 @@ _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) failure = _PyXI_excinfo_InitFromObject(info, exc); } if (failure != NULL) { - PyErr_SetString(PyExc_Exception, failure); - return -1; + PyMem_RawFree(info); + set_exc_with_cause(PyExc_Exception, failure); + return NULL; } - return 0; + return info; +} + +void +_PyXI_FreeExcInfo(_PyXI_excinfo *info) +{ + _PyXI_excinfo_clear(info); + PyMem_RawFree(info); } PyObject * @@ -1664,12 +1687,6 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info) return _PyXI_excinfo_AsObject(info); } -void -_PyXI_ClearExcInfo(_PyXI_excinfo *info) -{ - _PyXI_excinfo_clear(info); -} - /***************************/ /* short-term data sharing */ @@ -1727,70 +1744,267 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) return -1; } +/* basic failure info */ + +struct xi_failure { + // The kind of error to propagate. + _PyXI_errcode code; + // The propagated message. + const char *msg; + int msg_owned; +}; // _PyXI_failure + +#define XI_FAILURE_INIT (_PyXI_failure){ .code = _PyXI_ERR_NO_ERROR } + +static void +clear_xi_failure(_PyXI_failure *failure) +{ + if (failure->msg != NULL && failure->msg_owned) { + PyMem_RawFree((void*)failure->msg); + } + *failure = XI_FAILURE_INIT; +} + +static void +copy_xi_failure(_PyXI_failure *dest, _PyXI_failure *src) +{ + *dest = *src; + dest->msg_owned = 0; +} + +_PyXI_failure * +_PyXI_NewFailure(void) +{ + _PyXI_failure *failure = PyMem_RawMalloc(sizeof(_PyXI_failure)); + if (failure == NULL) { + PyErr_NoMemory(); + return NULL; + } + *failure = XI_FAILURE_INIT; + return failure; +} + +void +_PyXI_FreeFailure(_PyXI_failure *failure) +{ + clear_xi_failure(failure); + PyMem_RawFree(failure); +} + +_PyXI_errcode +_PyXI_GetFailureCode(_PyXI_failure *failure) +{ + if (failure == NULL) { + return _PyXI_ERR_NO_ERROR; + } + return failure->code; +} + +void +_PyXI_InitFailureUTF8(_PyXI_failure *failure, + _PyXI_errcode code, const char *msg) +{ + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 0, + }; +} + +int +_PyXI_InitFailure(_PyXI_failure *failure, _PyXI_errcode code, PyObject *obj) +{ + PyObject *msgobj = PyObject_Str(obj); + if (msgobj == NULL) { + return -1; + } + // This will leak if not paired with clear_xi_failure(). + // That happens automatically in _capture_current_exception(). + const char *msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (PyErr_Occurred()) { + return -1; + } + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 1, + }; + return 0; +} + /* shared exceptions */ -static const char * -_PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) +typedef struct { + // The originating interpreter. + PyInterpreterState *interp; + // The error to propagate, if different from the uncaught exception. + _PyXI_failure *override; + _PyXI_failure _override; + // The exception information to propagate, if applicable. + // This is populated only for some error codes, + // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. + _PyXI_excinfo uncaught; +} _PyXI_error; + +static void +xi_error_clear(_PyXI_error *err) { - if (error->interp == NULL) { - error->interp = PyInterpreterState_Get(); + err->interp = NULL; + if (err->override != NULL) { + clear_xi_failure(err->override); } + _PyXI_excinfo_clear(&err->uncaught); +} - const char *failure = NULL; - if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // There is an unhandled exception we need to propagate. - failure = _PyXI_excinfo_InitFromException(&error->uncaught, excobj); - if (failure != NULL) { - // We failed to initialize error->uncaught. - // XXX Print the excobj/traceback? Emit a warning? - // XXX Print the current exception/traceback? - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - error->code = _PyXI_ERR_NO_MEMORY; - } - else { - error->code = _PyXI_ERR_OTHER; - } - PyErr_Clear(); +static int +xi_error_is_set(_PyXI_error *error) +{ + if (error->override != NULL) { + assert(error->override->code != _PyXI_ERR_NO_ERROR); + assert(error->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION + || excinfo_is_set(&error->uncaught)); + return 1; + } + return excinfo_is_set(&error->uncaught); +} + +static int +xi_error_has_override(_PyXI_error *err) +{ + if (err->override == NULL) { + return 0; + } + return (err->override->code != _PyXI_ERR_NO_ERROR + && err->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); +} + +static PyObject * +xi_error_resolve_current_exc(PyThreadState *tstate, + _PyXI_failure *override) +{ + assert(override == NULL || override->code != _PyXI_ERR_NO_ERROR); + + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { + assert(override == NULL + || override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + } + else if (override == NULL) { + // This is equivalent to _PyXI_ERR_UNCAUGHT_EXCEPTION. + } + else if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We want to actually capture the current exception. + } + else if (exc != NULL) { + // It might make sense to do similarly for other codes. + if (override->code == _PyXI_ERR_ALREADY_RUNNING) { + // We don't need the exception info. + Py_CLEAR(exc); + } + // ...else we want to actually capture the current exception. + } + return exc; +} + +static void +xi_error_set_override(PyThreadState *tstate, _PyXI_error *err, + _PyXI_failure *override) +{ + assert(err->override == NULL); + assert(override != NULL); + assert(override->code != _PyXI_ERR_NO_ERROR); + // Use xi_error_set_exc() instead of setting _PyXI_ERR_UNCAUGHT_EXCEPTION.. + assert(override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + err->override = &err->_override; + // The caller still owns override->msg. + copy_xi_failure(&err->_override, override); + err->interp = tstate->interp; +} + +static void +xi_error_set_override_code(PyThreadState *tstate, _PyXI_error *err, + _PyXI_errcode code) +{ + _PyXI_failure override = XI_FAILURE_INIT; + override.code = code; + xi_error_set_override(tstate, err, &override); +} + +static const char * +xi_error_set_exc(PyThreadState *tstate, _PyXI_error *err, PyObject *exc) +{ + assert(!_PyErr_Occurred(tstate)); + assert(!xi_error_is_set(err)); + assert(err->override == NULL); + assert(err->interp == NULL); + assert(exc != NULL); + const char *failure = + _PyXI_excinfo_InitFromException(&err->uncaught, exc); + if (failure != NULL) { + // We failed to initialize err->uncaught. + // XXX Print the excobj/traceback? Emit a warning? + // XXX Print the current exception/traceback? + if (_PyErr_ExceptionMatches(tstate, PyExc_MemoryError)) { + xi_error_set_override_code(tstate, err, _PyXI_ERR_NO_MEMORY); } else { - error->code = code; + xi_error_set_override_code(tstate, err, _PyXI_ERR_OTHER); } - assert(error->code != _PyXI_ERR_NO_ERROR); - } - else { - // There is an error code we need to propagate. - assert(excobj == NULL); - assert(code != _PyXI_ERR_NO_ERROR); - error->code = code; - _PyXI_excinfo_clear(&error->uncaught); + PyErr_Clear(); } return failure; } -PyObject * -_PyXI_ApplyError(_PyXI_error *error) +static PyObject * +_PyXI_ApplyError(_PyXI_error *error, const char *failure) { PyThreadState *tstate = PyThreadState_Get(); - if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + + if (failure != NULL) { + xi_error_clear(error); + return NULL; + } + + _PyXI_errcode code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + if (error->override != NULL) { + code = error->override->code; + assert(code != _PyXI_ERR_NO_ERROR); + } + + if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { // We will raise an exception that proxies the propagated exception. return _PyXI_excinfo_AsObject(&error->uncaught); } - else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { + else if (code == _PyXI_ERR_NOT_SHAREABLE) { // Propagate the exception directly. assert(!_PyErr_Occurred(tstate)); - _set_xid_lookup_failure(tstate, NULL, error->uncaught.msg, NULL); + PyObject *cause = NULL; + if (excinfo_is_set(&error->uncaught)) { + // Maybe instead set a PyExc_ExceptionSnapshot as __cause__? + // That type doesn't exist currently + // but would look like interpreters.ExecutionFailed. + _PyXI_excinfo_Apply(&error->uncaught, PyExc_Exception); + cause = _PyErr_GetRaisedException(tstate); + } + const char *msg = error->override != NULL + ? error->override->msg + : error->uncaught.msg; + _set_xid_lookup_failure(tstate, NULL, msg, cause); + Py_XDECREF(cause); } else { // Raise an exception corresponding to the code. - assert(error->code != _PyXI_ERR_NO_ERROR); - (void)_PyXI_ApplyErrorCode(error->code, error->interp); - if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { + (void)_PyXI_ApplyErrorCode(code, error->interp); + assert(error->override == NULL || error->override->msg == NULL); + if (excinfo_is_set(&error->uncaught)) { // __context__ will be set to a proxy of the propagated exception. - PyObject *exc = PyErr_GetRaisedException(); + // (or use PyExc_ExceptionSnapshot like _PyXI_ERR_NOT_SHAREABLE?) + PyObject *exc = _PyErr_GetRaisedException(tstate); _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); - PyObject *exc2 = PyErr_GetRaisedException(); + PyObject *exc2 = _PyErr_GetRaisedException(tstate); PyException_SetContext(exc, exc2); - PyErr_SetRaisedException(exc); + _PyErr_SetRaisedException(tstate, exc); } } assert(PyErr_Occurred()); @@ -2164,20 +2378,22 @@ error: return NULL; } -static void _propagate_not_shareable_error(_PyXI_errcode *); +static void _propagate_not_shareable_error(PyThreadState *, + _PyXI_failure *); static int _fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, - xidata_fallback_t fallback, _PyXI_errcode *p_errcode) + xidata_fallback_t fallback, _PyXI_failure *p_err) { // All items are expected to be shareable. assert(_sharedns_check_counts(ns)); assert(ns->numnames == ns->maxitems); assert(ns->numvalues == 0); + PyThreadState *tstate = PyThreadState_Get(); for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { - if (p_errcode != NULL) { - _propagate_not_shareable_error(p_errcode); + if (p_err != NULL) { + _propagate_not_shareable_error(tstate, p_err); } // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { @@ -2244,18 +2460,6 @@ _apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) /* switched-interpreter sessions */ /*********************************/ -struct xi_session_error { - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *override; - // This is set if exit captured an exception to propagate. - _PyXI_error *info; - - // -- pre-allocated memory -- - _PyXI_error _info; - _PyXI_errcode _override; -}; - struct xi_session { #define SESSION_UNUSED 0 #define SESSION_ACTIVE 1 @@ -2288,8 +2492,6 @@ struct xi_session { // once the session exits. Do not access this directly; use // _PyXI_Preserve() and _PyXI_GetPreserved() instead; PyObject *_preserved; - - struct xi_session_error error; }; _PyXI_session * @@ -2317,26 +2519,6 @@ _session_is_active(_PyXI_session *session) return session->status == SESSION_ACTIVE; } -static int -_session_pop_error(_PyXI_session *session, struct xi_session_error *err) -{ - if (session->error.info == NULL) { - assert(session->error.override == NULL); - *err = (struct xi_session_error){0}; - return 0; - } - *err = session->error; - err->info = &err->_info; - if (err->override != NULL) { - err->override = &err->_override; - } - session->error = (struct xi_session_error){0}; - return 1; -} - -static int _ensure_main_ns(_PyXI_session *, _PyXI_errcode *); -static inline void _session_set_error(_PyXI_session *, _PyXI_errcode); - /* enter/exit a cross-interpreter session */ @@ -2351,10 +2533,6 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) // Set elsewhere and cleared in _exit_session(). assert(!session->running); assert(session->main_ns == NULL); - // Set elsewhere and cleared in _capture_current_exception(). - assert(session->error.override == NULL); - // Set elsewhere and cleared in _PyXI_Exit(). - assert(session->error.info == NULL); // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); @@ -2409,17 +2587,14 @@ _exit_session(_PyXI_session *session) assert(!session->own_init_tstate); } - assert(session->error.info == NULL); - assert(session->error.override == _PyXI_ERR_NO_ERROR); - *session = (_PyXI_session){0}; } static void -_propagate_not_shareable_error(_PyXI_errcode *p_errcode) +_propagate_not_shareable_error(PyThreadState *tstate, + _PyXI_failure *override) { - assert(p_errcode != NULL); - PyThreadState *tstate = PyThreadState_Get(); + assert(override != NULL); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { PyErr_FormatUnraisable( @@ -2428,15 +2603,24 @@ _propagate_not_shareable_error(_PyXI_errcode *p_errcode) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - *p_errcode = _PyXI_ERR_NOT_SHAREABLE; + *override = (_PyXI_failure){ + .code = _PyXI_ERR_NOT_SHAREABLE, + }; } } + +static int _ensure_main_ns(_PyXI_session *, _PyXI_failure *); +static const char * capture_session_error(_PyXI_session *, _PyXI_error *, + _PyXI_failure *); + int _PyXI_Enter(_PyXI_session *session, PyInterpreterState *interp, PyObject *nsupdates, _PyXI_session_result *result) { + PyThreadState *tstate = _PyThreadState_GET(); + // Convert the attrs for cross-interpreter use. _PyXI_namespace *sharedns = NULL; if (nsupdates != NULL) { @@ -2457,16 +2641,16 @@ _PyXI_Enter(_PyXI_session *session, } // For now we limit it to shareable objects. xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; - _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; - if (_fill_sharedns(sharedns, nsupdates, fallback, &errcode) < 0) { - assert(PyErr_Occurred()); - assert(session->error.info == NULL); - if (errcode == _PyXI_ERR_NO_ERROR) { - errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + _PyXI_failure _err = XI_FAILURE_INIT; + if (_fill_sharedns(sharedns, nsupdates, fallback, &_err) < 0) { + assert(_PyErr_Occurred(tstate)); + if (_err.code == _PyXI_ERR_NO_ERROR) { + _err.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; } _destroy_sharedns(sharedns); if (result != NULL) { - result->errcode = errcode; + assert(_err.msg == NULL); + result->errcode = _err.code; } return -1; } @@ -2475,52 +2659,53 @@ _PyXI_Enter(_PyXI_session *session, // Switch to the requested interpreter (if necessary). _enter_session(session, interp); - _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + _PyXI_failure override = XI_FAILURE_INIT; + override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + tstate = _PyThreadState_GET(); // Ensure this thread owns __main__. if (_PyInterpreterState_SetRunningMain(interp) < 0) { // In the case where we didn't switch interpreters, it would // be more efficient to leave the exception in place and return // immediately. However, life is simpler if we don't. - errcode = _PyXI_ERR_ALREADY_RUNNING; + override.code = _PyXI_ERR_ALREADY_RUNNING; goto error; } session->running = 1; // Apply the cross-interpreter data. if (sharedns != NULL) { - if (_ensure_main_ns(session, &errcode) < 0) { + if (_ensure_main_ns(session, &override) < 0) { goto error; } if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { - errcode = _PyXI_ERR_APPLY_NS_FAILURE; + override.code = _PyXI_ERR_APPLY_NS_FAILURE; goto error; } _destroy_sharedns(sharedns); } - errcode = _PyXI_ERR_NO_ERROR; - assert(!PyErr_Occurred()); + override.code = _PyXI_ERR_NO_ERROR; + assert(!_PyErr_Occurred(tstate)); return 0; error: // We want to propagate all exceptions here directly (best effort). - assert(errcode != _PyXI_ERR_NO_ERROR); - _session_set_error(session, errcode); - assert(!PyErr_Occurred()); + assert(override.code != _PyXI_ERR_NO_ERROR); + _PyXI_error err = {0}; + const char *failure = capture_session_error(session, &err, &override); // Exit the session. - struct xi_session_error err; - (void)_session_pop_error(session, &err); _exit_session(session); + tstate = _PyThreadState_GET(); if (sharedns != NULL) { _destroy_sharedns(sharedns); } // Apply the error from the other interpreter. - PyObject *excinfo = _PyXI_ApplyError(err.info); - _PyXI_excinfo_clear(&err.info->uncaught); + PyObject *excinfo = _PyXI_ApplyError(&err, failure); + xi_error_clear(&err); if (excinfo != NULL) { if (result != NULL) { result->excinfo = excinfo; @@ -2529,84 +2714,100 @@ error: #ifdef Py_DEBUG fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded"); #endif + Py_DECREF(excinfo); } } - assert(PyErr_Occurred()); + assert(_PyErr_Occurred(tstate)); return -1; } static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **, - _PyXI_errcode *); + _PyXI_failure *); static int _finish_preserved(_PyXI_namespace *, PyObject **); int -_PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode, +_PyXI_Exit(_PyXI_session *session, _PyXI_failure *override, _PyXI_session_result *result) { + PyThreadState *tstate = _PyThreadState_GET(); int res = 0; // Capture the raised exception, if any. - assert(session->error.info == NULL); - if (PyErr_Occurred()) { - _session_set_error(session, errcode); - assert(!PyErr_Occurred()); + _PyXI_error err = {0}; + const char *failure = NULL; + if (override != NULL && override->code == _PyXI_ERR_NO_ERROR) { + assert(override->msg == NULL); + override = NULL; + } + if (_PyErr_Occurred(tstate)) { + failure = capture_session_error(session, &err, override); } else { - assert(errcode == _PyXI_ERR_NO_ERROR); - assert(session->error.override == NULL); + assert(override == NULL); } // Capture the preserved namespace. _PyXI_namespace *preserved = NULL; PyObject *preservedobj = NULL; if (result != NULL) { - errcode = _PyXI_ERR_NO_ERROR; - if (_pop_preserved(session, &preserved, &preservedobj, &errcode) < 0) { - if (session->error.info != NULL) { + assert(!_PyErr_Occurred(tstate)); + _PyXI_failure _override = XI_FAILURE_INIT; + if (_pop_preserved( + session, &preserved, &preservedobj, &_override) < 0) + { + assert(preserved == NULL); + assert(preservedobj == NULL); + if (xi_error_is_set(&err)) { // XXX Chain the exception (i.e. set __context__)? PyErr_FormatUnraisable( "Exception ignored while capturing preserved objects"); + clear_xi_failure(&_override); } else { - _session_set_error(session, errcode); + if (_override.code == _PyXI_ERR_NO_ERROR) { + _override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } + failure = capture_session_error(session, &err, &_override); } } } // Exit the session. - struct xi_session_error err; - (void)_session_pop_error(session, &err); + assert(!_PyErr_Occurred(tstate)); _exit_session(session); + tstate = _PyThreadState_GET(); // Restore the preserved namespace. assert(preserved == NULL || preservedobj == NULL); if (_finish_preserved(preserved, &preservedobj) < 0) { assert(preservedobj == NULL); - if (err.info != NULL) { + if (xi_error_is_set(&err)) { // XXX Chain the exception (i.e. set __context__)? PyErr_FormatUnraisable( "Exception ignored while capturing preserved objects"); } else { - errcode = _PyXI_ERR_PRESERVE_FAILURE; - _propagate_not_shareable_error(&errcode); + xi_error_set_override_code( + tstate, &err, _PyXI_ERR_PRESERVE_FAILURE); + _propagate_not_shareable_error(tstate, err.override); } } if (result != NULL) { result->preserved = preservedobj; - result->errcode = errcode; + result->errcode = err.override != NULL + ? err.override->code + : _PyXI_ERR_NO_ERROR; } // Apply the error from the other interpreter, if any. - if (err.info != NULL) { + if (xi_error_is_set(&err)) { res = -1; - assert(!PyErr_Occurred()); - PyObject *excinfo = _PyXI_ApplyError(err.info); - _PyXI_excinfo_clear(&err.info->uncaught); + assert(!_PyErr_Occurred(tstate)); + PyObject *excinfo = _PyXI_ApplyError(&err, failure); if (excinfo == NULL) { - assert(PyErr_Occurred()); - if (result != NULL) { + assert(_PyErr_Occurred(tstate)); + if (result != NULL && !xi_error_has_override(&err)) { _PyXI_ClearResult(result); *result = (_PyXI_session_result){ .errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE, @@ -2620,7 +2821,9 @@ _PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode, #ifdef Py_DEBUG fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); #endif + Py_DECREF(excinfo); } + xi_error_clear(&err); } return res; } @@ -2628,106 +2831,72 @@ _PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode, /* in an active cross-interpreter session */ -static void -_capture_current_exception(_PyXI_session *session) +static const char * +capture_session_error(_PyXI_session *session, _PyXI_error *err, + _PyXI_failure *override) { - assert(session->error.info == NULL); - if (!PyErr_Occurred()) { - assert(session->error.override == NULL); - return; - } - - // Handle the exception override. - _PyXI_errcode *override = session->error.override; - session->error.override = NULL; - _PyXI_errcode errcode = override != NULL - ? *override - : _PyXI_ERR_UNCAUGHT_EXCEPTION; + assert(_session_is_active(session)); + assert(!xi_error_is_set(err)); + PyThreadState *tstate = session->init_tstate; - // Pop the exception object. - PyObject *excval = NULL; - if (errcode == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // We want to actually capture the current exception. - excval = PyErr_GetRaisedException(); - } - else if (errcode == _PyXI_ERR_ALREADY_RUNNING) { - // We don't need the exception info. - PyErr_Clear(); - } - else { - // We could do a variety of things here, depending on errcode. - // However, for now we simply capture the exception and save - // the errcode. - excval = PyErr_GetRaisedException(); + // Normalize the exception override. + if (override != NULL) { + if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + assert(override->msg == NULL); + override = NULL; + } + else { + assert(override->code != _PyXI_ERR_NO_ERROR); + } } - // Capture the exception. - _PyXI_error *err = &session->error._info; - *err = (_PyXI_error){ - .interp = session->init_tstate->interp, - }; - const char *failure; - if (excval == NULL) { - failure = _PyXI_InitError(err, NULL, errcode); - } - else { - failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); - Py_DECREF(excval); - if (failure == NULL && override != NULL) { - err->code = errcode; + // Handle the exception, if any. + const char *failure = NULL; + PyObject *exc = xi_error_resolve_current_exc(tstate, override); + if (exc != NULL) { + // There is an unhandled exception we need to preserve. + failure = xi_error_set_exc(tstate, err, exc); + Py_DECREF(exc); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable(failure); } } - // Handle capture failure. - if (failure != NULL) { - // XXX Make this error message more generic. - fprintf(stderr, - "RunFailedError: script raised an uncaught exception (%s)", - failure); - err = NULL; + // Handle the override. + if (override != NULL && failure == NULL) { + xi_error_set_override(tstate, err, override); } // Finished! - assert(!PyErr_Occurred()); - session->error.info = err; -} - -static inline void -_session_set_error(_PyXI_session *session, _PyXI_errcode errcode) -{ - assert(_session_is_active(session)); - assert(PyErr_Occurred()); - if (errcode == _PyXI_ERR_NO_ERROR) { - // We're a bit forgiving here. - errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; - } - if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) { - session->error._override = errcode; - session->error.override = &session->error._override; - } - _capture_current_exception(session); + assert(!_PyErr_Occurred(tstate)); + return failure; } static int -_ensure_main_ns(_PyXI_session *session, _PyXI_errcode *p_errcode) +_ensure_main_ns(_PyXI_session *session, _PyXI_failure *failure) { assert(_session_is_active(session)); + PyThreadState *tstate = session->init_tstate; if (session->main_ns != NULL) { return 0; } // Cache __main__.__dict__. - PyObject *main_mod = _Py_GetMainModule(session->init_tstate); + PyObject *main_mod = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(main_mod) < 0) { - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; } return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; } return -1; } @@ -2736,13 +2905,13 @@ _ensure_main_ns(_PyXI_session *session, _PyXI_errcode *p_errcode) } PyObject * -_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_errcode *p_errcode) +_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_failure *failure) { if (!_session_is_active(session)) { PyErr_SetString(PyExc_RuntimeError, "session not active"); return NULL; } - if (_ensure_main_ns(session, p_errcode) < 0) { + if (_ensure_main_ns(session, failure) < 0) { return NULL; } return session->main_ns; @@ -2752,9 +2921,12 @@ _PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_errcode *p_errcode) static int _pop_preserved(_PyXI_session *session, _PyXI_namespace **p_xidata, PyObject **p_obj, - _PyXI_errcode *p_errcode) + _PyXI_failure *p_failure) { + _PyXI_failure failure = XI_FAILURE_INIT; + _PyXI_namespace *xidata = NULL; assert(_PyThreadState_GET() == session->init_tstate); // active session + if (session->_preserved == NULL) { *p_xidata = NULL; *p_obj = NULL; @@ -2772,10 +2944,8 @@ _pop_preserved(_PyXI_session *session, // We did switch interpreters. Py_ssize_t len = PyDict_Size(session->_preserved); if (len < 0) { - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; - } - return -1; + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; } else if (len == 0) { *p_xidata = NULL; @@ -2783,29 +2953,31 @@ _pop_preserved(_PyXI_session *session, else { _PyXI_namespace *xidata = _create_sharedns(session->_preserved); if (xidata == NULL) { - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; - } - return -1; + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; } - _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; if (_fill_sharedns(xidata, session->_preserved, - _PyXIDATA_FULL_FALLBACK, &errcode) < 0) + _PyXIDATA_FULL_FALLBACK, &failure) < 0) { - assert(session->error.info == NULL); - if (errcode != _PyXI_ERR_NOT_SHAREABLE) { - errcode = _PyXI_ERR_PRESERVE_FAILURE; + if (failure.code != _PyXI_ERR_NOT_SHAREABLE) { + assert(failure.msg != NULL); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; } - if (p_errcode != NULL) { - *p_errcode = errcode; - } - _destroy_sharedns(xidata); - return -1; + goto error; } *p_xidata = xidata; } Py_CLEAR(session->_preserved); return 0; + +error: + if (p_failure != NULL) { + *p_failure = failure; + } + if (xidata != NULL) { + _destroy_sharedns(xidata); + } + return -1; } static int @@ -2835,8 +3007,9 @@ finally: int _PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, - _PyXI_errcode *p_errcode) + _PyXI_failure *p_failure) { + _PyXI_failure failure = XI_FAILURE_INIT; if (!_session_is_active(session)) { PyErr_SetString(PyExc_RuntimeError, "session not active"); return -1; @@ -2846,20 +3019,22 @@ _PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, if (session->_preserved == NULL) { set_exc_with_cause(PyExc_RuntimeError, "failed to initialize preserved objects"); - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; - } - return -1; + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; } } if (PyDict_SetItemString(session->_preserved, name, value) < 0) { set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); - if (p_errcode != NULL) { - *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; - } - return -1; + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; } return 0; + +error: + if (p_failure != NULL) { + *p_failure = failure; + } + return -1; } PyObject * diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index b16f38b847f..6d0b93eb82a 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -88,6 +88,33 @@ _PyXIData_FormatNotShareableError(PyThreadState *tstate, va_end(vargs); } +int +_PyXI_UnwrapNotShareableError(PyThreadState * tstate, _PyXI_failure *failure) +{ + PyObject *exctype = get_notshareableerror_type(tstate); + assert(exctype != NULL); + if (!_PyErr_ExceptionMatches(tstate, exctype)) { + return -1; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (failure != NULL) { + _PyXI_errcode code = _PyXI_ERR_NOT_SHAREABLE; + if (_PyXI_InitFailure(failure, code, exc) < 0) { + return -1; + } + } + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; + } + else { + assert(PyException_GetContext(exc) == NULL); + } + _PyErr_SetRaisedException(tstate, exc); + return 0; +} + _PyXIData_getdata_t _PyXIData_Lookup(PyThreadState *tstate, PyObject *obj) diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index 12cd61db1b6..98411adc5eb 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -7,13 +7,6 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) } PyObject *exc = _PyErr_GetRaisedException(tstate); assert(exc != NULL); - PyObject *ctx = PyException_GetContext(exc); - if (ctx == NULL) { - PyException_SetContext(exc, Py_NewRef(cause)); - } - else { - Py_DECREF(ctx); - } assert(PyException_GetCause(exc) == NULL); PyException_SetCause(exc, Py_NewRef(cause)); _PyErr_SetRaisedException(tstate, exc); diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index d0a1c081ffe..b383e39da19 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -10,6 +10,7 @@ import re import sys import tempfile import typing +import shlex import _llvm import _schema @@ -46,6 +47,7 @@ class _Target(typing.Generic[_S, _R]): stable: bool = False debug: bool = False verbose: bool = False + cflags: str = "" known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -62,6 +64,7 @@ class _Target(typing.Generic[_S, _R]): hasher = hashlib.sha256() hasher.update(self.triple.encode()) hasher.update(self.debug.to_bytes()) + hasher.update(self.cflags.encode()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes()) @@ -155,6 +158,8 @@ class _Target(typing.Generic[_S, _R]): f"{o}", f"{c}", *self.args, + # Allow user-provided CFLAGS to override any defaults + *shlex.split(self.cflags), ] await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 1afd0c76bad..a0733005929 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -39,11 +39,15 @@ if __name__ == "__main__": parser.add_argument( "-v", "--verbose", action="store_true", help="echo commands as they are run" ) + parser.add_argument( + "--cflags", help="additional flags to pass to the compiler", default="" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug target.force = args.force target.verbose = args.verbose + target.cflags = args.cflags target.pyconfig_dir = args.pyconfig_dir target.build( comment=comment, diff --git a/configure b/configure index 029bf527da4..fef9f2d7da9 100755 --- a/configure +++ b/configure @@ -10863,7 +10863,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir ." + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\"" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : diff --git a/configure.ac b/configure.ac index 371b2e8ed73..cc37a636c52 100644 --- a/configure.ac +++ b/configure.ac @@ -2776,7 +2776,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir ."]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\""]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) AS_VAR_IF([Py_DEBUG], [true], |