diff options
127 files changed, 2938 insertions, 1593 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 775d9c63260..63a28490043 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -281,9 +281,13 @@ Doc/howto/clinic.rst @erlend-aasland # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently -Lib/test/support/interpreters/ @ericsnowcurrently Modules/_interp*module.c @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently # Android **/*Android* @mhsmith @freakboy3742 diff --git a/.gitignore b/.gitignore index cdb0352e0a8..7aa6272cf8e 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ Tools/unicode/data/ /autom4te.cache /build/ /builddir/ +/compile_commands.json /config.cache /config.log /config.status diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c8e1b5c2461..885dbeb7530 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -982,6 +982,7 @@ the variables: .. index:: single: PyExc_BaseException (C var) + single: PyExc_BaseExceptionGroup (C var) single: PyExc_Exception (C var) single: PyExc_ArithmeticError (C var) single: PyExc_AssertionError (C var) @@ -1041,6 +1042,8 @@ the variables: +=========================================+=================================+==========+ | :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BaseExceptionGroup` | :exc:`BaseExceptionGroup` | [1]_ | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | @@ -1164,6 +1167,9 @@ the variables: .. versionadded:: 3.6 :c:data:`PyExc_ModuleNotFoundError`. +.. versionadded:: 3.11 + :c:data:`PyExc_BaseExceptionGroup`. + These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: @@ -1207,6 +1213,7 @@ the variables: single: PyExc_Warning (C var) single: PyExc_BytesWarning (C var) single: PyExc_DeprecationWarning (C var) + single: PyExc_EncodingWarning (C var) single: PyExc_FutureWarning (C var) single: PyExc_ImportWarning (C var) single: PyExc_PendingDeprecationWarning (C var) @@ -1225,6 +1232,8 @@ the variables: +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | +------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EncodingWarning` | :exc:`EncodingWarning` | | ++------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | @@ -1245,6 +1254,9 @@ the variables: .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. +.. versionadded:: 3.10 + :c:data:`PyExc_EncodingWarning`. + Notes: .. [3] diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 9c866438b48..3106bf9808f 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -492,17 +492,8 @@ Initializing and finalizing the interpreter strings other than those passed in (however, the contents of the strings pointed to by the argument list are not modified). - The return value will be ``0`` if the interpreter exits normally (i.e., - without an exception), ``1`` if the interpreter exits due to an exception, - or ``2`` if the argument list does not represent a valid Python command - line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will - drop into the interactive Python prompt, at which point a second otherwise - unhandled :exc:`SystemExit` will still exit the process, while any other - means of exiting will set the return value as described above. + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. In terms of the CPython runtime configuration APIs documented in the :ref:`runtime configuration <init-config>` section (and without accounting @@ -539,23 +530,18 @@ Initializing and finalizing the interpreter If :c:member:`PyConfig.inspect` is not set (the default), the return value will be ``0`` if the interpreter exits normally (that is, without raising - an exception), or ``1`` if the interpreter exits due to an exception. If an - otherwise unhandled :exc:`SystemExit` is raised, the function will immediately - exit the process instead of returning ``1``. + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option is used), rather than returning when the interpreter exits, execution will instead resume in an interactive Python prompt (REPL) using the ``__main__`` module's global namespace. If the interpreter exited with an exception, it is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: returning ``0`` - if the session terminates without raising an unhandled exception, exiting - immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for - any other unhandled exception. - - This function always finalizes the Python interpreter regardless of whether - it returns a value or immediately exits the process due to an unhandled - :exc:`SystemExit` exception. + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. See :ref:`Python Configuration <init-python-config>` for an example of a customized Python that always runs in isolated mode using diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 91046c0e6f1..af2bead3bb5 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -686,6 +686,26 @@ and :c:data:`PyType_Type` effectively act as defaults.) instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to free the object itself. + If you may call functions that may set the error indicator, you must use + :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` + to ensure you don't clobber a preexisting error indicator (the deallocation + could have occurred while processing a different error): + + .. code-block:: c + + static void + foo_dealloc(foo_object *self) + { + PyObject *et, *ev, *etb; + PyObject *exc = PyErr_GetRaisedException(); + ... + PyErr_SetRaisedException(exc); + } + + The dealloc handler itself must not raise an exception; if it hits an error + case it should call :c:func:`PyErr_FormatUnraisable` to log (and clear) an + unraisable exception. + No guarantees are made about when an object is destroyed, except: * Python will destroy an object immediately or some time after the final diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 07fdcfd9729..84fee05cb4c 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1827,7 +1827,7 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. - .. versionadded:: next + .. versionadded:: 3.14 .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) diff --git a/Doc/conf.py b/Doc/conf.py index 7fadad66cb3..b08f5452901 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -234,6 +234,7 @@ nitpick_ignore += [ ('c:data', 'PyExc_AssertionError'), ('c:data', 'PyExc_AttributeError'), ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BaseExceptionGroup'), ('c:data', 'PyExc_BlockingIOError'), ('c:data', 'PyExc_BrokenPipeError'), ('c:data', 'PyExc_BufferError'), @@ -287,6 +288,7 @@ nitpick_ignore += [ # C API: Standard Python warning classes ('c:data', 'PyExc_BytesWarning'), ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_EncodingWarning'), ('c:data', 'PyExc_FutureWarning'), ('c:data', 'PyExc_ImportWarning'), ('c:data', 'PyExc_PendingDeprecationWarning'), diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 39090e36ed9..b292d828841 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -251,7 +251,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 3) specifies the number of months per row. *css* is the name for the cascading style sheet to be used. :const:`None` can be passed if no style sheet should be used. *encoding* specifies the encoding to be used for the - output (defaulting to the system default encoding). + output (defaulting to ``'utf-8'``). .. method:: formatmonthname(theyear, themonth, withyear=True) diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 5be1a1106b0..18f9443cbfe 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -18,6 +18,7 @@ multitasking). Here's an overview: multiprocessing.shared_memory.rst concurrent.rst concurrent.futures.rst + concurrent.interpreters.rst subprocess.rst sched.rst queue.rst diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst new file mode 100644 index 00000000000..8860418e87a --- /dev/null +++ b/Doc/library/concurrent.interpreters.rst @@ -0,0 +1,198 @@ +:mod:`!concurrent.interpreters` --- Multiple interpreters in the same process +============================================================================= + +.. module:: concurrent.interpreters + :synopsis: Multiple interpreters in the same process + +.. moduleauthor:: Eric Snow <ericsnowcurrently@gmail.com> +.. sectionauthor:: Eric Snow <ericsnowcurrently@gmail.com> + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/concurrent/interpreters.py` + +-------------- + + +Introduction +------------ + +The :mod:`!concurrent.interpreters` module constructs higher-level +interfaces on top of the lower level :mod:`!_interpreters` module. + +.. XXX Add references to the upcoming HOWTO docs in the seealso block. + +.. seealso:: + + :ref:`isolating-extensions-howto` + how to update an extension module to support multiple interpreters + + :pep:`554` + + :pep:`734` + + :pep:`684` + +.. XXX Why do we disallow multiple interpreters on WASM? + +.. include:: ../includes/wasm-notavail.rst + + +Key details +----------- + +Before we dive into examples, there are a small number of details +to keep in mind about using multiple interpreters: + +* isolated, by default +* no implicit threads +* not all PyPI packages support use in multiple interpreters yet + +.. XXX Are there other relevant details to list? + +In the context of multiple interpreters, "isolated" means that +different interpreters do not share any state. In practice, there is some +process-global data they all share, but that is managed by the runtime. + + +Reference +--------- + +This module defines the following functions: + +.. function:: list_all() + + Return a :class:`list` of :class:`Interpreter` objects, + one for each existing interpreter. + +.. function:: get_current() + + Return an :class:`Interpreter` object for the currently running + interpreter. + +.. function:: get_main() + + Return an :class:`Interpreter` object for the main interpreter. + +.. function:: create() + + Initialize a new (idle) Python interpreter + and return a :class:`Interpreter` object for it. + + +Interpreter objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Interpreter(id) + + A single interpreter in the current process. + + Generally, :class:`Interpreter` shouldn't be called directly. + Instead, use :func:`create` or one of the other module functions. + + .. attribute:: id + + (read-only) + + The interpreter's ID. + + .. attribute:: whence + + (read-only) + + A string describing where the interpreter came from. + + .. method:: is_running() + + Return ``True`` if the interpreter is currently executing code + in its :mod:`!__main__` module and ``False`` otherwise. + + .. method:: close() + + Finalize and destroy the interpreter. + + .. method:: prepare_main(ns=None, **kwargs) + + Bind "shareable" objects in the interpreter's + :mod:`!__main__` module. + + .. method:: exec(code, /, dedent=True) + + Run the given source code in the interpreter (in the current thread). + + .. method:: call(callable, /, *args, **kwargs) + + Return the result of calling running the given function in the + interpreter (in the current thread). + + .. method:: call_in_thread(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in a new thread). + +Exceptions +^^^^^^^^^^ + +.. exception:: InterpreterError + + This exception, a subclass of :exc:`Exception`, is raised when + an interpreter-related error happens. + +.. exception:: InterpreterNotFoundError + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the targeted interpreter no longer exists. + +.. exception:: ExecutionFailed + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the running code raised an uncaught exception. + + .. attribute:: excinfo + + A basic snapshot of the exception raised in the other interpreter. + +.. XXX Document the excinfoattrs? + +.. exception:: NotShareableError + + This exception, a subclass of :exc:`TypeError`, is raised when + an object cannot be sent to another interpreter. + + +.. XXX Add functions for communicating between interpreters. + + +Basic usage +----------- + +Creating an interpreter and running code in it:: + + from concurrent import interpreters + + interp = interpreters.create() + + # Run in the current OS thread. + + interp.exec('print("spam!")') + + interp.exec("""if True: + print('spam!') + """) + + from textwrap import dedent + interp.exec(dedent(""" + print('spam!') + """)) + + def run(): + print('spam!') + + interp.call(run) + + # Run in new OS thread. + + t = interp.call_in_thread(run) + t.join() + + +.. XXX Explain about object "sharing". diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 8caea78bbb5..748c72c733b 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,6 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there is only one module in this package: +This package contains the following modules: * :mod:`concurrent.futures` -- Launching parallel tasks +* :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 5297be17bd7..2e513bff651 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -609,7 +609,7 @@ A slightly more advanced use of the reader --- catching and reporting errors:: for row in reader: print(row) except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) + sys.exit(f'file {filename}, line {reader.line_num}: {e}') And while the module doesn't directly support parsing strings, it can easily be done:: diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index f18c7cc9c02..299c8aa399c 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -121,8 +121,11 @@ Module contents :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how *eq* and *frozen* are set. + - *unsafe_hash*: If true, force ``dataclasses`` to create a + :meth:`~object.__hash__` method, even though it may not be safe to do so. + Otherwise, generate a :meth:`~object.__hash__` method according to how + *eq* and *frozen* are set. + The default value is ``False``. :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 0e9dc33ae21..f8c71005a53 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -548,7 +548,7 @@ mnemonic that the corresponding value is a callable. The ``filters`` member of ``handlers`` and ``loggers`` can take filter instances in addition to ids. -You can also specify a special key ``'.'`` whose value is a dictionary is a +You can also specify a special key ``'.'`` whose value is a mapping of attribute names to values. If found, the specified attributes will be set on the user-defined object before it is returned. Thus, with the following configuration:: diff --git a/Doc/library/python.rst b/Doc/library/python.rst index c2c231af7c3..c5c762e11b9 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -27,3 +27,8 @@ overview: inspect.rst annotationlib.rst site.rst + +.. seealso:: + + * See the :mod:`concurrent.interpreters` module, which similarly + exposes core runtime functionality. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 2d0f9a740c6..12761baf792 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1482,7 +1482,9 @@ Cursor objects :type parameters: :class:`dict` | :term:`sequence` :raises ProgrammingError: - If *sql* contains more than one SQL statement. + When *sql* contains more than one SQL statement. + When :ref:`named placeholders <sqlite3-placeholders>` are used + and *parameters* is a sequence instead of a :class:`dict`. If :attr:`~Connection.autocommit` is :data:`LEGACY_TRANSACTION_CONTROL`, @@ -1491,13 +1493,11 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders <sqlite3-placeholders>` are used and *parameters* is a sequence instead of a :class:`dict`. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. Use :meth:`executescript` to execute multiple SQL statements. @@ -1519,8 +1519,10 @@ Cursor objects :type parameters: :term:`iterable` :raises ProgrammingError: - If *sql* contains more than one SQL statement, - or is not a DML statement. + When *sql* contains more than one SQL statement + or is not a DML statement, + When :ref:`named placeholders <sqlite3-placeholders>` are used + and the items in *parameters* are sequences instead of :class:`dict`\s. Example: @@ -1544,14 +1546,12 @@ Cursor objects .. _RETURNING clauses: https://www.sqlite.org/lang_returning.html - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders <sqlite3-placeholders>` are used and the items in *parameters* are sequences instead of :class:`dict`\s. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. .. method:: executescript(sql_script, /) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f0b4b09ff10..b75e5ceecf8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1018,7 +1018,7 @@ operations have the same priority as the corresponding numeric operations. [3]_ | ``s * n`` or | equivalent to adding *s* to | (2)(7) | | ``n * s`` | itself *n* times | | +--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | +| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(9) | +--------------------------+--------------------------------+----------+ | ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | +--------------------------+--------------------------------+----------+ @@ -1150,6 +1150,9 @@ Notes: without copying any data and with the returned index being relative to the start of the sequence rather than the start of the slice. +(9) + An :exc:`IndexError` is raised if *i* is outside the sequence range. + .. _typesseq-immutable: diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 1f92b5df430..c228006d4c1 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -51,7 +51,7 @@ The token constants are: .. data:: NAME Token value that indicates an :ref:`identifier <identifiers>`. - Note that keywords are also initially tokenized an ``NAME`` tokens. + Note that keywords are also initially tokenized as ``NAME`` tokens. .. data:: NUMBER diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 8cce6b98cbc..747ee3ee0e1 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -257,6 +257,10 @@ The :mod:`uuid` module defines the following functions: non-specified arguments are substituted for a pseudo-random integer of appropriate size. + By default, *a*, *b* and *c* are generated by a non-cryptographically + secure pseudo-random number generator (CSPRNG). Use :func:`uuid4` when + a UUID needs to be used in a security-sensitive context. + .. versionadded:: 3.14 diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index b9cca4444c9..55c148801d8 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -8,15 +8,15 @@ used to generate the CPython parser (see :source:`Grammar/python.gram`). The version here omits details related to code generation and error recovery. -The notation is a mixture of `EBNF -<https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form>`_ -and `PEG <https://en.wikipedia.org/wiki/Parsing_expression_grammar>`_. -In particular, ``&`` followed by a symbol, token or parenthesized -group indicates a positive lookahead (i.e., is required to match but -not consumed), while ``!`` indicates a negative lookahead (i.e., is -required *not* to match). We use the ``|`` separator to mean PEG's -"ordered choice" (written as ``/`` in traditional PEG grammars). See -:pep:`617` for more details on the grammar's syntax. +The notation used here is the same as in the preceding docs, +and is described in the :ref:`notation <notation>` section, +except for a few extra complications: + +* ``&e``: a positive lookahead (that is, ``e`` is required to match but + not consumed) +* ``!e``: a negative lookahead (that is, ``e`` is required *not* to match) +* ``~`` ("cut"): commit to the current alternative and fail the rule + even if this fails to parse .. literalinclude:: ../../Grammar/python.gram :language: peg diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index b7b70e6be5a..444acac374a 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -90,44 +90,122 @@ Notation .. index:: BNF, grammar, syntax, notation -The descriptions of lexical analysis and syntax use a modified -`Backus–Naur form (BNF) <https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form>`_ grammar -notation. This uses the following style of definition: - -.. productionlist:: notation - name: `lc_letter` (`lc_letter` | "_")* - lc_letter: "a"..."z" - -The first line says that a ``name`` is an ``lc_letter`` followed by a sequence -of zero or more ``lc_letter``\ s and underscores. An ``lc_letter`` in turn is -any of the single characters ``'a'`` through ``'z'``. (This rule is actually -adhered to for the names defined in lexical and grammar rules in this document.) - -Each rule begins with a name (which is the name defined by the rule) and -``::=``. A vertical bar (``|``) is used to separate alternatives; it is the -least binding operator in this notation. A star (``*``) means zero or more -repetitions of the preceding item; likewise, a plus (``+``) means one or more -repetitions, and a phrase enclosed in square brackets (``[ ]``) means zero or -one occurrences (in other words, the enclosed phrase is optional). The ``*`` -and ``+`` operators bind as tightly as possible; parentheses are used for -grouping. Literal strings are enclosed in quotes. White space is only -meaningful to separate tokens. Rules are normally contained on a single line; -rules with many alternatives may be formatted alternatively with each line after -the first beginning with a vertical bar. - -.. index:: lexical definitions, ASCII - -In lexical definitions (as the example above), two more conventions are used: -Two literal characters separated by three dots mean a choice of any single -character in the given (inclusive) range of ASCII characters. A phrase between -angular brackets (``<...>``) gives an informal description of the symbol -defined; e.g., this could be used to describe the notion of 'control character' -if needed. - -Even though the notation used is almost the same, there is a big difference -between the meaning of lexical and syntactic definitions: a lexical definition -operates on the individual characters of the input source, while a syntax -definition operates on the stream of tokens generated by the lexical analysis. -All uses of BNF in the next chapter ("Lexical Analysis") are lexical -definitions; uses in subsequent chapters are syntactic definitions. - +The descriptions of lexical analysis and syntax use a grammar notation that +is a mixture of +`EBNF <https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form>`_ +and `PEG <https://en.wikipedia.org/wiki/Parsing_expression_grammar>`_. +For example: + +.. grammar-snippet:: + :group: notation + + name: `letter` (`letter` | `digit` | "_")* + letter: "a"..."z" | "A"..."Z" + digit: "0"..."9" + +In this example, the first line says that a ``name`` is a ``letter`` followed +by a sequence of zero or more ``letter``\ s, ``digit``\ s, and underscores. +A ``letter`` in turn is any of the single characters ``'a'`` through +``'z'`` and ``A`` through ``Z``; a ``digit`` is a single character from ``0`` +to ``9``. + +Each rule begins with a name (which identifies the rule that's being defined) +followed by a colon, ``:``. +The definition to the right of the colon uses the following syntax elements: + +* ``name``: A name refers to another rule. + Where possible, it is a link to the rule's definition. + + * ``TOKEN``: An uppercase name refers to a :term:`token`. + For the purposes of grammar definitions, tokens are the same as rules. + +* ``"text"``, ``'text'``: Text in single or double quotes must match literally + (without the quotes). The type of quote is chosen according to the meaning + of ``text``: + + * ``'if'``: A name in single quotes denotes a :ref:`keyword <keywords>`. + * ``"case"``: A name in double quotes denotes a + :ref:`soft-keyword <soft-keywords>`. + * ``'@'``: A non-letter symbol in single quotes denotes an + :py:data:`~token.OP` token, that is, a :ref:`delimiter <delimiters>` or + :ref:`operator <operators>`. + +* ``e1 e2``: Items separated only by whitespace denote a sequence. + Here, ``e1`` must be followed by ``e2``. +* ``e1 | e2``: A vertical bar is used to separate alternatives. + It denotes PEG's "ordered choice": if ``e1`` matches, ``e2`` is + not considered. + In traditional PEG grammars, this is written as a slash, ``/``, rather than + a vertical bar. + See :pep:`617` for more background and details. +* ``e*``: A star means zero or more repetitions of the preceding item. +* ``e+``: Likewise, a plus means one or more repetitions. +* ``[e]``: A phrase enclosed in square brackets means zero or + one occurrences. In other words, the enclosed phrase is optional. +* ``e?``: A question mark has exactly the same meaning as square brackets: + the preceding item is optional. +* ``(e)``: Parentheses are used for grouping. +* ``"a"..."z"``: Two literal characters separated by three dots mean a choice + of any single character in the given (inclusive) range of ASCII characters. + This notation is only used in + :ref:`lexical definitions <notation-lexical-vs-syntactic>`. +* ``<...>``: A phrase between angular brackets gives an informal description + of the matched symbol (for example, ``<any ASCII character except "\">``), + or an abbreviation that is defined in nearby text (for example, ``<Lu>``). + This notation is only used in + :ref:`lexical definitions <notation-lexical-vs-syntactic>`. + +The unary operators (``*``, ``+``, ``?``) bind as tightly as possible; +the vertical bar (``|``) binds most loosely. + +White space is only meaningful to separate tokens. + +Rules are normally contained on a single line, but rules that are too long +may be wrapped: + +.. grammar-snippet:: + :group: notation + + literal: stringliteral | bytesliteral + | integer | floatnumber | imagnumber + +Alternatively, rules may be formatted with the first line ending at the colon, +and each alternative beginning with a vertical bar on a new line. +For example: + + +.. grammar-snippet:: + :group: notation-alt + + literal: + | stringliteral + | bytesliteral + | integer + | floatnumber + | imagnumber + +This does *not* mean that there is an empty first alternative. + +.. index:: lexical definitions + +.. _notation-lexical-vs-syntactic: + +Lexical and Syntactic definitions +--------------------------------- + +There is some difference between *lexical* and *syntactic* analysis: +the :term:`lexical analyzer` operates on the individual characters of the +input source, while the *parser* (syntactic analyzer) operates on the stream +of :term:`tokens <token>` generated by the lexical analysis. +However, in some cases the exact boundary between the two phases is a +CPython implementation detail. + +The practical difference between the two is that in *lexical* definitions, +all whitespace is significant. +The lexical analyzer :ref:`discards <whitespace>` all whitespace that is not +converted to tokens like :data:`token.INDENT` or :data:`~token.NEWLINE`. +*Syntactic* definitions then use these tokens, rather than source characters. + +This documentation uses the same BNF grammar for both styles of definitions. +All uses of BNF in the next chapter (:ref:`lexical`) are lexical definitions; +uses in subsequent chapters are syntactic definitions. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index cdb35da7bc9..9e06e03991b 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -13,10 +13,9 @@ end a multi-line command. .. only:: html - You can toggle the display of prompts and output by clicking on ``>>>`` - in the upper-right corner of an example box. If you hide the prompts - and output for an example, then you can easily copy and paste the input - lines into your interpreter. + You can use the "Copy" button (it appears in the upper-right corner + when hovering over or tapping a code example), which strips prompts + and omits output, to copy and paste the input lines into your interpreter. .. index:: single: # (hash); comment diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 45e68aea5fb..ca330a32b33 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -83,6 +83,7 @@ and improvements in user-friendliness and correctness. .. PEP-sized items next. * :ref:`PEP 649 and 749: deferred evaluation of annotations <whatsnew314-pep649>` +* :ref:`PEP 734: Multiple Interpreters in the Stdlib <whatsnew314-pep734>` * :ref:`PEP 741: Python Configuration C API <whatsnew314-pep741>` * :ref:`PEP 750: Template strings <whatsnew314-pep750>` * :ref:`PEP 758: Allow except and except* expressions without parentheses <whatsnew314-pep758>` @@ -123,6 +124,101 @@ of Python. See :ref:`below <whatsnew314-refcount>` for details. New features ============ +.. _whatsnew314-pep734: + +PEP 734: Multiple Interpreters in the Stdlib +-------------------------------------------- + +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an "interpreter". +However, the feature had been available only through the C-API. + +That limitation is removed in the 3.14 release, +with the new :mod:`concurrent.interpreters` module. + +There are at least two notable reasons why using multiple interpreters +is worth considering: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software enables efficiency and +can simplify software, at a high level. At the same time, implementing +and maintaining all but the simplest concurrency is often a struggle +for the human brain. That especially applies to plain threads +(for example, :mod:`threading`), where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like CSP or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters like threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of the 3.12 release, interpreters +are now sufficiently isolated from one another to be used in parallel. +(See :pep:`684`.) This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a stdlib +module. Consequently, they currently have several notable limitations, +which will improve significantly now that the feature is finally +going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (we will be working next on extensive internal sharing between + interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many extension modules on PyPI are not compatible with multiple + interpreters yet (stdlib extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, we expect to slowly see +libraries on PyPI for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. Isolating a module has a lot of +overlap with what is required to support +:ref:`free-threading <whatsnew314-free-threaded-cpython>`, +so the ongoing work in the community in that area will help accelerate +support for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +<whatsnew314-concurrent-futures-interp-pool>`. + +.. seealso:: + :pep:`734`. + + .. _whatsnew314-pep750: PEP 750: Template strings @@ -1109,6 +1205,8 @@ calendar concurrent.futures ------------------ +.. _whatsnew314-concurrent-futures-interp-pool: + * Add :class:`~concurrent.futures.InterpreterPoolExecutor`, which exposes "subinterpreters" (multiple Python interpreters in the same process) to Python code. This is separate from the proposed API diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2f8335a895c..9f327cf904d 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -305,12 +305,6 @@ New features functions as replacements for :c:func:`PySys_GetObject`. (Contributed by Serhiy Storchaka in :gh:`108512`.) -* Add :c:func:`PyUnicodeWriter_WriteASCII` function to write an ASCII string - into a :c:type:`PyUnicodeWriter`. The function is faster than - :c:func:`PyUnicodeWriter_WriteUTF8`, but has an undefined behavior if the - input string contains non-ASCII characters. - (Contributed by Victor Stinner in :gh:`133968`.) - * Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of a string. See the documentation for caveats. (Contributed by Petr Viktorin in :gh:`131510`) diff --git a/Include/Python.h b/Include/Python.h index f34d581f0b4..64be8014589 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -59,14 +59,6 @@ # include <intrin.h> // __readgsqword() #endif -// Suppress known warnings in Python header files. -#if defined(_MSC_VER) -// Warning that alignas behaviour has changed. Doesn't affect us, because we -// never relied on the old behaviour. -#pragma warning(push) -#pragma warning(disable: 5274) -#endif - // Include Python header files #include "pyport.h" #include "pymacro.h" @@ -146,9 +138,4 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" -// Restore warning filter -#ifdef _MSC_VER -#pragma warning(pop) -#endif - #endif /* !Py_PYTHON_H */ diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 7c1aac9696d..86c502730f4 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -47,6 +47,63 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { /* --- Unicode Type ------------------------------------------------------- */ +struct _PyUnicodeObject_state { + /* If interned is non-zero, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + The possible values here are: + 0: Not Interned + 1: Interned + 2: Interned and Immortal + 3: Interned, Immortal, and Static + This categorization allows the runtime to determine the right + cleanup mechanism at runtime shutdown. */ +#ifdef Py_GIL_DISABLED + // Needs to be accessed atomically, so can't be a bit field. + unsigned char interned; +#else + unsigned int interned:2; +#endif + /* Character size: + + - PyUnicode_1BYTE_KIND (1): + + * character type = Py_UCS1 (8 bits, unsigned) + * all characters are in the range U+0000-U+00FF (latin1) + * if ascii is set, all characters are in the range U+0000-U+007F + (ASCII), otherwise at least one character is in the range + U+0080-U+00FF + + - PyUnicode_2BYTE_KIND (2): + + * character type = Py_UCS2 (16 bits, unsigned) + * all characters are in the range U+0000-U+FFFF (BMP) + * at least one character is in the range U+0100-U+FFFF + + - PyUnicode_4BYTE_KIND (4): + + * character type = Py_UCS4 (32 bits, unsigned) + * all characters are in the range U+0000-U+10FFFF + * at least one character is in the range U+10000-U+10FFFF + */ + unsigned int kind:3; + /* Compact is with respect to the allocation scheme. Compact unicode + objects only require one memory block while non-compact objects use + one block for the PyUnicodeObject struct and another for its data + buffer. */ + unsigned int compact:1; + /* The string only contains characters in the range U+0000-U+007F (ASCII) + and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is + set, use the PyASCIIObject structure. */ + unsigned int ascii:1; + /* The object is statically allocated. */ + unsigned int statically_allocated:1; +#ifndef Py_GIL_DISABLED + /* Historical: padding to ensure that PyUnicode_DATA() is always aligned to + 4 bytes (see issue gh-63736 on m68k) */ + unsigned int :24; +#endif +}; + /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject structure. state.ascii and state.compact are set, and the data immediately follow the structure. utf8_length can be found @@ -99,67 +156,8 @@ typedef struct { PyObject_HEAD Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ -#ifdef Py_GIL_DISABLED - /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k. - In the non-free-threaded build, we'll use explicit padding instead */ - _Py_ALIGN_AS(4) -#endif - struct { - /* If interned is non-zero, the two references from the - dictionary to this object are *not* counted in ob_refcnt. - The possible values here are: - 0: Not Interned - 1: Interned - 2: Interned and Immortal - 3: Interned, Immortal, and Static - This categorization allows the runtime to determine the right - cleanup mechanism at runtime shutdown. */ -#ifdef Py_GIL_DISABLED - // Needs to be accessed atomically, so can't be a bit field. - unsigned char interned; -#else - unsigned int interned:2; -#endif - /* Character size: - - - PyUnicode_1BYTE_KIND (1): - - * character type = Py_UCS1 (8 bits, unsigned) - * all characters are in the range U+0000-U+00FF (latin1) - * if ascii is set, all characters are in the range U+0000-U+007F - (ASCII), otherwise at least one character is in the range - U+0080-U+00FF - - - PyUnicode_2BYTE_KIND (2): - - * character type = Py_UCS2 (16 bits, unsigned) - * all characters are in the range U+0000-U+FFFF (BMP) - * at least one character is in the range U+0100-U+FFFF - - - PyUnicode_4BYTE_KIND (4): - - * character type = Py_UCS4 (32 bits, unsigned) - * all characters are in the range U+0000-U+10FFFF - * at least one character is in the range U+10000-U+10FFFF - */ - unsigned int kind:3; - /* Compact is with respect to the allocation scheme. Compact unicode - objects only require one memory block while non-compact objects use - one block for the PyUnicodeObject struct and another for its data - buffer. */ - unsigned int compact:1; - /* The string only contains characters in the range U+0000-U+007F (ASCII) - and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is - set, use the PyASCIIObject structure. */ - unsigned int ascii:1; - /* The object is statically allocated. */ - unsigned int statically_allocated:1; -#ifndef Py_GIL_DISABLED - /* Padding to ensure that PyUnicode_DATA() is always aligned to - 4 bytes (see issue gh-63736 on m68k) */ - unsigned int :24; -#endif - } state; + /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k. */ + _Py_ALIGNED_DEF(4, struct _PyUnicodeObject_state) state; } PyASCIIObject; /* Non-ASCII strings allocated through PyUnicode_New use the diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f25f5847b3b..f1f427d99de 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -159,10 +159,11 @@ struct atexit_state { typedef struct { // Tagged pointer to next object in the list. // 0 means the object is not tracked - uintptr_t _gc_next; + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) _gc_next; // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. + // Those bits are made available by the struct's minimum alignment. uintptr_t _gc_prev; } PyGC_Head; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 87914767252..10e7199269e 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -264,6 +264,32 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* Wrap a pointer in a stack ref. + * The resulting stack reference is not safe and should only be used + * in the interpreter to pass values from one uop to another. + * The GC should never see one of these stack refs. */ +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .bits = (uintptr_t)ptr }; +#endif +} + +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.bits & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.bits & ~Py_TAG_BITS); +#else + return (void *)(ref.bits); +#endif +} + static inline bool PyStackRef_IsError(_PyStackRef ref) { diff --git a/Include/object.h b/Include/object.h index 42aed614d4a..c75e9db0cbd 100644 --- a/Include/object.h +++ b/Include/object.h @@ -101,6 +101,12 @@ whose size is determined when the object is allocated. #define PyObject_VAR_HEAD PyVarObject ob_base; #define Py_INVALID_SIZE (Py_ssize_t)-1 +/* PyObjects are given a minimum alignment so that the least significant bits + * of an object pointer become available for other purposes. + * This must be an integer literal with the value (1 << _PyGC_PREV_SHIFT), number of bytes. + */ +#define _PyObject_MIN_ALIGNMENT 4 + /* Nothing is actually declared to be a PyObject, but every pointer to * a Python object can be cast to a PyObject*. This is inheritance built * by hand. Similarly every pointer to a variable-size Python object can, @@ -136,6 +142,7 @@ struct _object { #else Py_ssize_t ob_refcnt; #endif + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner; }; #ifdef _MSC_VER __pragma(warning(pop)) @@ -153,7 +160,7 @@ struct _object { // ob_tid stores the thread id (or zero). It is also used by the GC and the // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. - uintptr_t ob_tid; + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) ob_tid; uint16_t ob_flags; PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state diff --git a/Include/pymacro.h b/Include/pymacro.h index d410645034d..bfe660e8303 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -24,44 +24,66 @@ #endif -// _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword, -// We currently use alignas for free-threaded builds only; additional compat -// checking would be great before we add it to the default build. -// Standards/compiler support: +// _Py_ALIGNED_DEF(N, T): Define a variable/member with increased alignment +// +// `N`: the desired minimum alignment, an integer literal, number of bytes +// `T`: the type of the defined variable +// (or a type with at least the defined variable's alignment) +// +// May not be used on a struct definition. +// +// Standards/compiler support for `alignas` alternatives: // - `alignas` is a keyword in C23 and C++11. // - `_Alignas` is a keyword in C11 // - GCC & clang has __attribute__((aligned)) // (use that for older standards in pedantic mode) // - MSVC has __declspec(align) // - `_Alignas` is common C compiler extension -// Older compilers may name it differently; to allow compilation on such -// unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already +// Older compilers may name `alignas` differently; to allow compilation on such +// unsupported platforms, we don't redefine _Py_ALIGNED_DEF if it's already // defined. Note that defining it wrong (including defining it to nothing) will // cause ABI incompatibilities. -#ifdef Py_GIL_DISABLED -# ifndef _Py_ALIGN_AS -# ifdef __cplusplus -# if __cplusplus >= 201103L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__GNUC__) || defined(__clang__) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) alignas(V) -# endif -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define _Py_ALIGN_AS(V) _Alignas(V) -# elif (defined(__GNUC__) || defined(__clang__)) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) _Alignas(V) -# endif -# endif +// +// Behavior of `alignas` alternatives: +// - `alignas` & `_Alignas`: +// - Can be used multiple times; the greatest alignment applies. +// - It is an *error* if the combined effect of all `alignas` modifiers would +// decrease the alignment. +// - Takes types or numbers. +// - May not be used on a struct definition, unless also defining a variable. +// - `__declspec(align)`: +// - Has no effect if it would decrease alignment. +// - Only takes an integer literal. +// - May be used on struct or variable definitions. +// However, when defining both the struct and the variable at once, +// `declspec(aligned)` causes compiler warning 5274 and possible ABI +// incompatibility. +// - ` __attribute__((aligned))`: +// - Has no effect if it would decrease alignment. +// - Takes types or numbers +// - May be used on struct or variable definitions. +#ifndef _Py_ALIGNED_DEF +# ifdef __cplusplus +# if __cplusplus >= 201103L +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# elif defined(__GNUC__) || defined(__clang__) +# define _Py_ALIGNED_DEF(N, T) __attribute__((aligned(N))) T +# elif defined(_MSC_VER) +# define _Py_ALIGNED_DEF(N, T) __declspec(align(N)) T +# else +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# endif +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define _Py_ALIGNED_DEF(N, T) _Alignas(N) _Alignas(T) T +# elif (defined(__GNUC__) || defined(__clang__)) +# define _Py_ALIGNED_DEF(N, T) __attribute__((aligned(N))) T +# elif defined(_MSC_VER) +# define _Py_ALIGNED_DEF(N, T) __declspec(align(N)) T +# else +# define _Py_ALIGNED_DEF(N, T) _Alignas(N) _Alignas(T) T +# endif #endif /* Minimum value between x and y */ diff --git a/Lib/calendar.py b/Lib/calendar.py index 18f76d52ff8..3be1b50500e 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -565,7 +565,7 @@ class HTMLCalendar(Calendar): Return a formatted year as a complete HTML page. """ if encoding is None: - encoding = sys.getdefaultencoding() + encoding = 'utf-8' v = [] a = v.append a('<?xml version="1.0" encoding="%s"?>\n' % encoding) @@ -846,7 +846,7 @@ def main(args=None): parser.add_argument( "-e", "--encoding", default=None, - help="encoding to use for output" + help="encoding to use for output (default utf-8)" ) parser.add_argument( "-t", "--type", @@ -890,7 +890,7 @@ def main(args=None): cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: - encoding = sys.getdefaultencoding() + encoding = 'utf-8' optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index a2c4fbfd3fb..f12b4ac33cd 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -167,7 +167,7 @@ class WorkerContext(_thread.WorkerContext): except _interpqueues.QueueError: continue except ModuleNotFoundError: - # interpreters.queues doesn't exist, which means + # interpreters._queues doesn't exist, which means # QueueEmpty doesn't. Act as though it does. continue else: diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 76b7b2abe83..a14650bf5fa 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -755,6 +755,11 @@ class ProcessPoolExecutor(_base.Executor): self._executor_manager_thread_wakeup def _adjust_process_count(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index 6d1b0690805..0fd661249a2 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -9,6 +9,10 @@ from _interpreters import ( InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) +from ._queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) __all__ = [ @@ -20,21 +24,6 @@ __all__ = [ ] -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - _EXEC_FAILURE_STR = """ {superstr} diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py index 544e197ba4c..f47eb693ac8 100644 --- a/Lib/test/support/interpreters/_crossinterp.py +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -61,7 +61,7 @@ class UnboundItem: def __repr__(self): return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' +# return f'interpreters._queues.UNBOUND' UNBOUND = object.__new__(UnboundItem) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/concurrent/interpreters/_queues.py index 99987f2f692..99987f2f692 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/concurrent/interpreters/_queues.py diff --git a/Lib/email/message.py b/Lib/email/message.py index 87fcab68868..41fcc2b9778 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -564,7 +564,7 @@ class Message: msg.add_header('content-disposition', 'attachment', filename='bud.gif') msg.add_header('content-disposition', 'attachment', - filename=('utf-8', '', Fußballer.ppt')) + filename=('utf-8', '', 'Fußballer.ppt')) msg.add_header('content-disposition', 'attachment', filename='Fußballer.ppt')) """ diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 7eab74dc0db..3de1f0d24a1 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -417,8 +417,14 @@ def decode_params(params): for name, continuations in rfc2231_params.items(): value = [] extended = False - # Sort by number - continuations.sort() + # Sort by number, treating None as 0 if there is no 0, + # and ignore it if there is already a 0. + has_zero = any(x[0] == 0 for x in continuations) + if has_zero: + continuations = [x for x in continuations if x[0] is not None] + else: + continuations = [(x[0] or 0, x[1], x[2]) for x in continuations] + continuations.sort(key=lambda x: x[0]) # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a *, then the entire string, after diff --git a/Lib/locale.py b/Lib/locale.py index 2feb10e59c9..dfedc6386cb 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -883,6 +883,10 @@ del k, v # updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin' # removed 'univ' # removed 'universal' +# +# SS 2025-06-10: +# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist +# on all platforms. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -962,7 +966,6 @@ locale_alias = { 'c.ascii': 'C', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', - 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9d6ae3e4d00..9a3a26a8400 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1100,6 +1100,11 @@ class AbstractUnpickleTests: self.check_unpickling_error((pickle.UnpicklingError, OverflowError), dumped) + def test_large_binstring(self): + errmsg = 'BINSTRING pickle has negative byte count' + with self.assertRaisesRegex(pickle.UnpicklingError, errmsg): + self.loads(b'T\0\0\0\x80') + def test_get(self): pickled = b'((lp100000\ng100000\nt.' unpickled = self.loads(pickled) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/channels.py index 1724759b75a..b2de24d9d3e 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from . import _crossinterp +from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) -from ._crossinterp import ( +from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index 88eee03a3de..858d31a73cf 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -9,7 +9,7 @@ import unittest from test.support import import_helper, skip_if_sanitizer _channels = import_helper.import_module('_interpchannels') -from test.support.interpreters import _crossinterp +from concurrent.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 7ade4271b7a..bc39c86b8cf 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -417,7 +417,7 @@ class OutputTestCase(unittest.TestCase): self.check_htmlcalendar_encoding('utf-8', 'utf-8') def test_output_htmlcalendar_encoding_default(self): - self.check_htmlcalendar_encoding(None, sys.getdefaultencoding()) + self.check_htmlcalendar_encoding(None, 'utf-8') def test_yeardatescalendar(self): def shrink(cal): diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index a292ebcc7f4..bf22ef2a592 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1666,13 +1666,11 @@ class TestUopsOptimization(unittest.TestCase): self.assertIn("_CONTAINS_OP_DICT", uops) self.assertNotIn("_TO_BOOL_BOOL", uops) - def test_remove_guard_for_known_type_str(self): def f(n): for i in range(n): false = i == TIER2_THRESHOLD empty = "X"[:false] - empty += "" # Make JIT realize this is a string. if empty: return 1 return 0 @@ -1778,11 +1776,12 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_GUARD_TOS_UNICODE", uops) self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - def test_call_type_1(self): + def test_call_type_1_guards_removed(self): def testfunc(n): x = 0 for _ in range(n): - x += type(42) is int + foo = eval('42') + x += type(foo) is int return x res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) @@ -1793,6 +1792,25 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_GUARD_NOS_NULL", uops) self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) + def test_call_type_1_known_type(self): + def testfunc(n): + x = 0 + for _ in range(n): + x += type(42) is int + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the result of type(...) is known, _CALL_TYPE_1 is replaced with + # _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW which is optimized away in + # remove_unneeded_uops. + self.assertNotIn("_CALL_TYPE_1", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_call_type_1_result_is_const(self): def testfunc(n): x = 0 @@ -1806,7 +1824,6 @@ class TestUopsOptimization(unittest.TestCase): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_call_str_1(self): @@ -2230,6 +2247,49 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_remove_guard_for_slice_list(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + sliced = [1, 2, 3][:false] + if sliced: + return 1 + return 0 + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_TO_BOOL_LIST", uops) + self.assertNotIn("_GUARD_TOS_LIST", uops) + + def test_remove_guard_for_slice_tuple(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + a, b = (1, 2, 3)[: false + 2] + + _, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_GUARD_TOS_TUPLE", uops) + + def test_unary_invert_long_type(self): + def testfunc(n): + for _ in range(n): + a = 9397 + x = ~a + ~a + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + def global_identity(x): return x diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f6c62ae4b20..5fd5684e103 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -8,10 +8,10 @@ import unittest from concurrent.futures.interpreter import ( ExecutionFailed, BrokenInterpreterPool, ) +from concurrent.interpreters import _queues as queues import _interpreters from test import support import test.test_asyncio.utils as testasyncio_utils -from test.support.interpreters import queues from .executor import ExecutorTest, mul from .util import BaseTestCase, InterpreterPoolMixin, setup_module diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 7a4065afd46..99b315b47e2 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -330,6 +330,64 @@ class ProcessPoolShutdownTest(ExecutorShutdownTest): # shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @classmethod + def _failing_task_gh_132969(cls, n): + raise ValueError("failing task") + + @classmethod + def _good_task_gh_132969(cls, n): + time.sleep(0.1 * n) + return n + + def _run_test_issue_gh_132969(self, max_workers): + # max_workers=2 will repro exception + # max_workers=4 will repro exception and then hang + + # Repro conditions + # max_tasks_per_child=1 + # a task ends abnormally + # shutdown(wait=False) is called + start_method = self.get_context().get_start_method() + if (start_method == "fork" or + (start_method == "forkserver" and sys.platform.startswith("win"))): + self.skipTest(f"Skipping test for {start_method = }") + executor = futures.ProcessPoolExecutor( + max_workers=max_workers, + max_tasks_per_child=1, + mp_context=self.get_context()) + f1 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 1) + f2 = executor.submit(ProcessPoolShutdownTest._failing_task_gh_132969, 2) + f3 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 3) + result = 0 + try: + result += f1.result() + result += f2.result() + result += f3.result() + except ValueError: + # stop processing results upon first exception + pass + + # Ensure that the executor cleans up after called + # shutdown with wait=False + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + time.sleep(0.2) + executor_manager_thread.join() + return result + + def test_shutdown_gh_132969_case_1(self): + # gh-132969: test that exception "object of type 'NoneType' has no len()" + # is not raised when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(2) + self.assertEqual(result, 1) + + def test_shutdown_gh_132969_case_2(self): + # gh-132969: test that process does not hang and + # exception "object of type 'NoneType' has no len()" is not raised + # when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(4) + self.assertEqual(result, 1) + create_executor_tests(globals(), ProcessPoolShutdownTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 192c8eab26e..57e818b1c68 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -125,21 +125,22 @@ class CProfileTest(ProfileTest): """ gh-106152 generator.throw() should trigger a call in cProfile - In the any() call below, there should be two entries for the generator: - * one for the call to __next__ which gets a True and terminates any - * one when the generator is garbage collected which will effectively - do a throw. """ + + def gen(): + yield + pr = self.profilerclass() pr.enable() - any(a == 1 for a in (1, 2)) + g = gen() + try: + g.throw(SyntaxError) + except SyntaxError: + pass pr.disable() pr.create_stats() - for func, (cc, nc, _, _, _) in pr.stats.items(): - if func[2] == "<genexpr>": - self.assertEqual(cc, 1) - self.assertEqual(nc, 1) + self.assertTrue(any("throw" in func[2] for func in pr.stats.keys())), def test_bad_descriptor(self): # gh-132250 diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7b14305f997..8765d121fd0 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -389,6 +389,24 @@ class TestMessageAPI(TestEmailBase): msg = email.message_from_string("Content-Type: blarg; baz; boo\n") self.assertEqual(msg.get_param('baz'), '') + def test_continuation_sorting_part_order(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"ignored\"; " + "filename*0*=\"utf-8''foo%20\"; " + "filename*1*=\"bar.txt\"\n" + ) + filename = msg.get_filename() + self.assertEqual(filename, 'foo bar.txt') + + def test_sorting_no_continuations(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"bar.txt\"; " + ) + filename = msg.get_filename() + self.assertEqual(filename, 'bar.txt') + def test_missing_filename(self): msg = email.message_from_string("From: foo\n") self.assertEqual(msg.get_filename(), None) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py new file mode 100644 index 00000000000..f75fb264c8a --- /dev/null +++ b/Lib/test/test_free_threading/test_heapq.py @@ -0,0 +1,240 @@ +import unittest + +import heapq + +from enum import Enum +from threading import Thread, Barrier +from random import shuffle, randint + +from test.support import threading_helper +from test import test_heapq + + +NTHREADS = 10 +OBJECT_COUNT = 5_000 + + +class Heap(Enum): + MIN = 1 + MAX = 2 + + +@threading_helper.requires_working_threading() +class TestHeapq(unittest.TestCase): + def setUp(self): + self.test_heapq = test_heapq.TestHeapPython() + + def test_racing_heapify(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + + self.run_concurrently( + worker_func=heapq.heapify, args=(heap,), nthreads=NTHREADS + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappush(self): + heap = [] + + def heappush_func(heap): + for item in reversed(range(OBJECT_COUNT)): + heapq.heappush(heap, item) + + self.run_concurrently( + worker_func=heappush_func, args=(heap,), nthreads=NTHREADS + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_func(heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop(heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_ascending(local_list)) + + self.run_concurrently( + worker_func=heappop_func, + args=(heap, per_thread_pop_count), + nthreads=NTHREADS, + ) + self.assertEqual(len(heap), 0) + + def test_racing_heappushpop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_func(heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop(heap, item) + self.assertTrue(popped_item <= item) + + self.run_concurrently( + worker_func=heappushpop_func, + args=(heap, pushpop_items), + nthreads=NTHREADS, + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapreplace(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_func(heap, replace_items): + for item in replace_items: + heapq.heapreplace(heap, item) + + self.run_concurrently( + worker_func=heapreplace_func, + args=(heap, replace_items), + nthreads=NTHREADS, + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapify_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + + self.run_concurrently( + worker_func=heapq.heapify_max, args=(max_heap,), nthreads=NTHREADS + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappush_max(self): + max_heap = [] + + def heappush_max_func(max_heap): + for item in range(OBJECT_COUNT): + heapq.heappush_max(max_heap, item) + + self.run_concurrently( + worker_func=heappush_max_func, args=(max_heap,), nthreads=NTHREADS + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_max_func(max_heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop_max(max_heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_descending(local_list)) + + self.run_concurrently( + worker_func=heappop_max_func, + args=(max_heap, per_thread_pop_count), + nthreads=NTHREADS, + ) + self.assertEqual(len(max_heap), 0) + + def test_racing_heappushpop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_max_func(max_heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop_max(max_heap, item) + self.assertTrue(popped_item >= item) + + self.run_concurrently( + worker_func=heappushpop_max_func, + args=(max_heap, pushpop_items), + nthreads=NTHREADS, + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heapreplace_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_max_func(max_heap, replace_items): + for item in replace_items: + heapq.heapreplace_max(max_heap, item) + + self.run_concurrently( + worker_func=heapreplace_max_func, + args=(max_heap, replace_items), + nthreads=NTHREADS, + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + @staticmethod + def is_sorted_ascending(lst): + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + @staticmethod + def is_sorted_descending(lst): + """ + Check if the list is sorted in descending order (non-increasing). + """ + return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) + + @staticmethod + def create_heap(size, heap_kind): + """ + Create a min/max heap where elements are in the range (0, size - 1) and + shuffled before heapify. + """ + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + if heap_kind == Heap.MIN: + heapq.heapify(heap) + else: + heapq.heapify_max(heap) + + return heap + + @staticmethod + def create_random_list(a, b, size): + """ + Create a list of random numbers between a and b (inclusive). + """ + return [randint(-a, b) for _ in range(size)] + + def run_concurrently(self, worker_func, args, nthreads): + """ + Run the worker function concurrently in multiple threads. + """ + barrier = Barrier(nthreads) + + def wrapper_func(*args): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + worker_func(*args) + + with threading_helper.catch_threading_exception() as cm: + workers = ( + Thread(target=wrapper_func, args=args) for _ in range(nthreads) + ) + with threading_helper.start_threads(workers): + pass + + # Worker threads should not raise any exceptions + self.assertIsNone(cm.exc_value) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 37046d8e1c0..6411e4318b6 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -56,14 +56,14 @@ class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() inputs = [ - x := StackItem("x", None, "1"), - y := StackItem("y", None, "oparg"), - z := StackItem("z", None, "oparg*2"), + x := StackItem("x", "1"), + y := StackItem("y", "oparg"), + z := StackItem("z", "oparg*2"), ] outputs = [ - StackItem("x", None, "1"), - StackItem("b", None, "oparg*4"), - StackItem("c", None, "1"), + StackItem("x", "1"), + StackItem("b", "oparg*4"), + StackItem("c", "1"), ] null = CWriter.null() stack.pop(z, null) @@ -1103,32 +1103,6 @@ class TestGeneratedCases(unittest.TestCase): """ self.run_cases_test(input, output) - def test_pointer_to_stackref(self): - input = """ - inst(OP, (arg: _PyStackRef * -- out)) { - out = *arg; - DEAD(arg); - } - """ - output = """ - TARGET(OP) { - #if Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef *arg; - _PyStackRef out; - arg = (_PyStackRef *)stack_pointer[-1].bits; - out = *arg; - stack_pointer[-1] = out; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - def test_unused_cached_value(self): input = """ op(FIRST, (arg1 -- out)) { diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index b3c9ef8efba..1403cd145b6 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -13,11 +13,11 @@ from test.support import script_helper from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters from test.support import Py_GIL_DISABLED -from test.support import interpreters from test.support import force_not_colorized import test._crossinterp_definitions as defs -from test.support.interpreters import ( +from concurrent.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -133,7 +133,7 @@ class CreateTests(TestBase): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() print(interp.id) """)) @@ -196,7 +196,7 @@ class GetCurrentTests(TestBase): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -213,7 +213,7 @@ class GetCurrentTests(TestBase): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -225,7 +225,7 @@ class GetCurrentTests(TestBase): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -582,7 +582,7 @@ class TestInterpreterClose(TestBase): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -599,7 +599,7 @@ class TestInterpreterClose(TestBase): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -806,7 +806,7 @@ class TestInterpreterExec(TestBase): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - from test.support import interpreters + from concurrent import interpreters def script(): import spam @@ -827,7 +827,7 @@ class TestInterpreterExec(TestBase): ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -1281,7 +1281,7 @@ class TestInterpreterCall(TestBase): # no module indirection with self.subTest('no indirection'): text = run(f""" - from test.support import interpreters + from concurrent import interpreters def spam(): # This a global var... @@ -1301,7 +1301,7 @@ class TestInterpreterCall(TestBase): """) with self.subTest('indirect as func, direct interp'): text = run(f""" - from test.support import interpreters + from concurrent import interpreters import mymod def spam(): @@ -1317,7 +1317,7 @@ class TestInterpreterCall(TestBase): # indirect as func, indirect interp new_mod('mymod', f""" - from test.support import interpreters + from concurrent import interpreters def run(func): interp = interpreters.create() return interp.call(func) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 0c027b17cea..109ddf34453 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,8 +8,8 @@ import time from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -from test.support import interpreters -from test.support.interpreters import channels +from concurrent import interpreters +from test.support import channels from .utils import _run_output, TestBase @@ -171,7 +171,7 @@ class TestSendRecv(TestBase): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +244,7 @@ class TestSendRecv(TestBase): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -387,7 +387,7 @@ class TestSendRecv(TestBase): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -482,7 +482,7 @@ class TestSendRecv(TestBase): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) sch.send_nowait(1, unbounditems=channels.UNBOUND) sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) @@ -518,7 +518,7 @@ class TestSendRecv(TestBase): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() @@ -526,7 +526,7 @@ class TestSendRecv(TestBase): sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index ac24f6568ac..15537ac6cc8 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ class StartupTests(TestBase): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - from test.support import interpreters + from concurrent import interpreters orig = sys.path[0] @@ -170,7 +170,7 @@ class FinalizationTests(TestBase): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() raise Exception '''] diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 757373904d7..3e982d76e86 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,8 +7,8 @@ import unittest from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -from test.support import interpreters -from test.support.interpreters import queues, _crossinterp +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp from .utils import _run_output, TestBase as _TestBase @@ -126,7 +126,7 @@ class QueueTests(TestBase): interp = interpreters.create() interp.exec(dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); @@ -324,7 +324,7 @@ class TestQueueOps(TestBase): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): @@ -351,7 +351,7 @@ class TestQueueOps(TestBase): out = _run_output( interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -390,7 +390,7 @@ class TestQueueOps(TestBase): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' @@ -468,7 +468,7 @@ class TestQueueOps(TestBase): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) queue.put(1, unbounditems=queues.UNBOUND) queue.put(2, unbounditems=queues.UNBOUND_ERROR) @@ -504,14 +504,14 @@ class TestQueueOps(TestBase): queue.put(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() queue.put(2, unbounditems=queues.UNBOUND) queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index fae2f38cb55..e25e67a0d4f 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -6,7 +6,7 @@ from test.support import import_helper from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') -from test.support import interpreters +from concurrent import interpreters from .utils import TestBase diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index c25e0fb7475..ae09aa457b4 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -21,7 +21,7 @@ try: import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -from test.support import interpreters +from concurrent import interpreters try: diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 455d2af37ef..55b502e52ca 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -387,6 +387,10 @@ class NormalizeTest(unittest.TestCase): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 54910cd8054..31ebcb3b8b0 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -14,6 +14,15 @@ from test import support from fractions import Fraction from collections import abc, Counter + +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class TestBasicOps: # Superclass with tests common to all generators. # Subclasses must arrange for self.gen to retrieve the Random instance @@ -809,6 +818,9 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.gen.seed(1234567) self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + self.gen.seed(1234567) + self.assertEqual(self.gen.getrandbits(MyIndex(100)), + 97904845777343510404718956115) def test_getrandbits_2G_bits(self): size = 2**31 diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 7f0b0f36505..d993e28c4bb 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -3,6 +3,8 @@ import sqlite3 import sys import textwrap import unittest +import unittest.mock +import os from sqlite3.__main__ import main as cli from test.support.import_helper import import_module @@ -14,6 +16,7 @@ from test.support import ( captured_stdin, force_not_colorized_test_class, requires_subprocess, + verbose, ) @@ -253,7 +256,7 @@ class Completion(unittest.TestCase): def test_complete_no_match(self): input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" # Set NO_COLOR to disable coloring for self.PS1. - output = self.write_input(input_, env={"NO_COLOR": "1"}) + output = self.write_input(input_, env={**os.environ, "NO_COLOR": "1"}) lines = output.decode().splitlines() indices = ( i for i, line in enumerate(lines, 1) @@ -282,20 +285,30 @@ class Completion(unittest.TestCase): readline.parse_and_bind("set completion-query-items 0") readline.parse_and_bind("set page-completions off") readline.parse_and_bind("set completion-display-width 0") + readline.parse_and_bind("set show-all-if-ambiguous off") + readline.parse_and_bind("set show-all-if-unmodified off") main() """) input_ = b"\t\t.quit\n" - output = run_pty(script, input_, env={"NO_COLOR": "1"}) - lines = output.decode().splitlines() - indices = [ - i for i, line in enumerate(lines) - if line.startswith(self.PS1) - ] - self.assertEqual(len(indices), 2) - start, end = indices - candidates = [l.strip() for l in lines[start+1:end]] - self.assertEqual(candidates, sorted(SQLITE_KEYWORDS)) + output = run_pty(script, input_, env={**os.environ, "NO_COLOR": "1"}) + try: + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) + if line.startswith(self.PS1) + ] + self.assertEqual(len(indices), 2) + start, end = indices + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(candidates, sorted(SQLITE_KEYWORDS)) + except: + if verbose: + print(' PTY output: '.center(30, '-')) + print(output.decode(errors='replace')) + print(' end PTY output '.center(30, '-')) + raise + if __name__ == "__main__": diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 13aaba405e3..b09e524a756 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -382,6 +382,13 @@ SyntaxError: invalid syntax Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index bf415894903..39e62027f03 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -24,7 +24,7 @@ from test.support import import_helper from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None import textwrap diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 59b3a749d2f..125c2744698 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ from test import lock_tests from test import support try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 9011e0e1962..a117413301b 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2513,15 +2513,16 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - from test.support import interpreters + from concurrent import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.interpreters.channels # noqa: F401 + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = interpreters.channels.create() + rch, sch = self.create_channel() script = textwrap.dedent(""" import test.support @@ -2547,7 +2548,7 @@ class SubinterpreterTests(unittest.TestCase): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('from test.support import interpreters') + interp.exec('from concurrent import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py index bc95b4ebf4a..00a9eaf2f99 100644 --- a/Lib/test/test_zipfile/_path/_test_params.py +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -1,5 +1,5 @@ -import types import functools +import types from ._itertools import always_iterable diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index b505dd7c376..7c108fc6ab8 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -8,10 +8,8 @@ import zipfile from ._functools import compose from ._itertools import consume - from ._support import import_or_skip - big_o = import_or_skip('big_o') pytest = import_or_skip('pytest') diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 0afabc0c668..696134023a5 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -1,6 +1,6 @@ +import contextlib import io import itertools -import contextlib import pathlib import pickle import stat @@ -9,12 +9,11 @@ import unittest import zipfile import zipfile._path -from test.support.os_helper import temp_dir, FakePath +from test.support.os_helper import FakePath, temp_dir from ._functools import compose from ._itertools import Counter - -from ._test_params import parameterize, Invoked +from ._test_params import Invoked, parameterize class jaraco: @@ -193,10 +192,10 @@ class TestPath(unittest.TestCase): """EncodingWarning must blame the read_text and open calls.""" assert sys.flags.warn_default_encoding root = zipfile.Path(alpharep) - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").read_text() assert __file__ == wc.filename - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").open("r").close() assert __file__ == wc.filename @@ -365,6 +364,17 @@ class TestPath(unittest.TestCase): assert root.name == 'alpharep.zip' == root.filename.name @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + + @pass_alpharep def test_suffix(self, alpharep): """ The suffix of the root should be the suffix of the zipfile. diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py index 48c09b53717..7418391abad 100644 --- a/Lib/test/test_zipfile/_path/write-alpharep.py +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -1,4 +1,3 @@ from . import test_path - __name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/uuid.py b/Lib/uuid.py index 06f81a7c338..313f2fc46cb 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -656,18 +656,20 @@ def _windll_getnode(): def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 5ae16ec970d..faae4c84cae 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -7,19 +7,19 @@ https://github.com/python/importlib_metadata/wiki/Development-Methodology for more detail. """ +import functools import io -import posixpath -import zipfile import itertools -import contextlib import pathlib +import posixpath import re import stat import sys +import zipfile +from ._functools import save_method_args from .glob import Translator - __all__ = ['Path'] @@ -86,13 +86,12 @@ class InitializedState: Mix-in to save the initialization state for pickling. """ + @save_method_args def __init__(self, *args, **kwargs): - self.__args = args - self.__kwargs = kwargs super().__init__(*args, **kwargs) def __getstate__(self): - return self.__args, self.__kwargs + return self._saved___init__.args, self._saved___init__.kwargs def __setstate__(self, state): args, kwargs = state @@ -181,22 +180,27 @@ class FastLookup(CompleteDirs): """ def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super().namelist() - return self.__names + return self._namelist + + @functools.cached_property + def _namelist(self): + return super().namelist() def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super()._name_set() - return self.__lookup + return self._name_set_prop + + @functools.cached_property + def _name_set_prop(self): + return super()._name_set() def _extract_text_encoding(encoding=None, *args, **kwargs): # compute stack level so that the caller of the caller sees any warning. is_pypy = sys.implementation.name == 'pypy' - stack_level = 3 + is_pypy + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi return io.text_encoding(encoding, stack_level), args, kwargs @@ -351,7 +355,7 @@ class Path: return io.TextIOWrapper(stream, encoding, *args, **kwargs) def _base(self): - return pathlib.PurePosixPath(self.at or self.root.filename) + return pathlib.PurePosixPath(self.at) if self.at else self.filename @property def name(self): diff --git a/Lib/zipfile/_path/_functools.py b/Lib/zipfile/_path/_functools.py new file mode 100644 index 00000000000..7390be21873 --- /dev/null +++ b/Lib/zipfile/_path/_functools.py @@ -0,0 +1,20 @@ +import collections +import functools + + +# from jaraco.functools 4.0.2 +def save_method_args(method): + """ + Wrap a method such that when it is called, the args and kwargs are + saved on the method. + """ + args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') # noqa: PYI024 + + @functools.wraps(method) + def wrapper(self, /, *args, **kwargs): + attr_name = '_saved_' + method.__name__ + attr = args_and_kwargs(args, kwargs) + setattr(self, attr_name, attr) + return method(self, *args, **kwargs) + + return wrapper diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index d7fe45a4947..bd2839304b7 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,7 +1,6 @@ import os import re - _default_seps = os.sep + str(os.altsep) * bool(os.altsep) diff --git a/Makefile.pre.in b/Makefile.pre.in index b5703fbe6ae..66b34b779f2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2514,7 +2514,7 @@ XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ compression compression/_common compression/zstd \ - concurrent concurrent/futures \ + concurrent concurrent/futures concurrent/interpreters \ csv \ ctypes ctypes/macholib \ curses \ @@ -2573,7 +2573,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ - test/support/interpreters \ test/test_asyncio \ test/test_capi \ test/test_cext \ diff --git a/Misc/ACKS b/Misc/ACKS index 739af8d9e11..d4557a03eb5 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -658,6 +658,7 @@ Michael Goderbauer Karan Goel Jeroen Van Goey Christoph Gohlke +Daniel Golding Tim Golden Yonatan Goldschmidt Mark Gollahon diff --git a/Misc/NEWS.d/next/Build/2024-12-04-10-00-35.gh-issue-127545.t0THjE.rst b/Misc/NEWS.d/next/Build/2024-12-04-10-00-35.gh-issue-127545.t0THjE.rst new file mode 100644 index 00000000000..3667e2778b7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-04-10-00-35.gh-issue-127545.t0THjE.rst @@ -0,0 +1 @@ +Fix crash when building on Linux/m68k. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst new file mode 100644 index 00000000000..6ecbfb8d9cf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst @@ -0,0 +1 @@ +Make the JIT optimizer understand that slicing a string/list/tuple returns the same type. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst new file mode 100644 index 00000000000..1da76561469 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst @@ -0,0 +1,2 @@ +Add support to :pep:`768` remote debugging for Linux kernels which don't +have CONFIG_CROSS_MEMORY_ATTACH configured. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst new file mode 100644 index 00000000000..506eefdb21a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst @@ -0,0 +1 @@ +Make methods in :mod:`heapq` thread-safe on the :term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst new file mode 100644 index 00000000000..e4b5f610353 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst @@ -0,0 +1,2 @@ +Optimize away ``_CALL_TYPE_1`` in the JIT when the return type is known. +Patch by Tomas Roun diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-01-09-44.gh-issue-131798.1SuxO9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-01-09-44.gh-issue-131798.1SuxO9.rst new file mode 100644 index 00000000000..a3775262306 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-01-09-44.gh-issue-131798.1SuxO9.rst @@ -0,0 +1 @@ +Optimize ``_UNARY_INVERT`` in JIT-compiled code. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-23-57-37.gh-issue-130077.MHknDB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-23-57-37.gh-issue-130077.MHknDB.rst new file mode 100644 index 00000000000..a7d02426b6f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-09-23-57-37.gh-issue-130077.MHknDB.rst @@ -0,0 +1,2 @@ +Properly raise custom syntax errors when incorrect syntax containing names +that are prefixes of soft keywords is encountered. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Documentation/2021-09-15-13-07-25.bpo-45210.RtGk7i.rst b/Misc/NEWS.d/next/Documentation/2021-09-15-13-07-25.bpo-45210.RtGk7i.rst new file mode 100644 index 00000000000..ce3eba154ba --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-09-15-13-07-25.bpo-45210.RtGk7i.rst @@ -0,0 +1,2 @@ +Document that error indicator may be set in tp_dealloc, and how to avoid +clobbering it. diff --git a/Misc/NEWS.d/next/Library/2025-04-30-19-32-18.gh-issue-132969.EagQ3G.rst b/Misc/NEWS.d/next/Library/2025-04-30-19-32-18.gh-issue-132969.EagQ3G.rst new file mode 100644 index 00000000000..7364c425941 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-30-19-32-18.gh-issue-132969.EagQ3G.rst @@ -0,0 +1,7 @@ +Prevent the :class:`~concurrent.futures.ProcessPoolExecutor` executor thread, +which remains running when :meth:`shutdown(wait=False) +<concurrent.futures.Executor.shutdown>`, from +attempting to adjust the pool's worker processes after the object state has already been reset during shutdown. +A combination of conditions, including a worker process having terminated abormally, +resulted in an exception and a potential hang when the still-running executor thread +attempted to replace dead workers within the pool. diff --git a/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst b/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst new file mode 100644 index 00000000000..ecdde240b4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst @@ -0,0 +1,2 @@ +:mod:`email`: Fix :exc:`TypeError` in :func:`email.utils.decode_params` +when sorting :rfc:`2231` continuations that contain an unnumbered section. diff --git a/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst new file mode 100644 index 00000000000..2bda69bff52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-30-09-46-21.gh-issue-134939.Pu3nnm.rst @@ -0,0 +1 @@ +Add the :mod:`concurrent.interpreters` module. See :pep:`734`. diff --git a/Misc/NEWS.d/next/Library/2025-06-01-14-18-48.gh-issue-135004.cq3-fp.rst b/Misc/NEWS.d/next/Library/2025-06-01-14-18-48.gh-issue-135004.cq3-fp.rst new file mode 100644 index 00000000000..4c59b0f8e19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-01-14-18-48.gh-issue-135004.cq3-fp.rst @@ -0,0 +1,3 @@ +Rewrite and cleanup the internal :mod:`!_blake2` module. Some exception +messages were changed but their types were left untouched. Patch by Bénédikt +Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst b/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst new file mode 100644 index 00000000000..1f70358e64e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst @@ -0,0 +1,4 @@ +:mod:`uuid`: when the MAC address cannot be determined, the 48-bit node
+ID is now generated with a cryptographically-secure pseudo-random number
+generator (CSPRNG) as per :rfc:`RFC 9562, §6.10.3 <9562#section-6.10-3>`.
+This affects :func:`~uuid.uuid1` and :func:`~uuid.uuid6`.
diff --git a/Misc/NEWS.d/next/Library/2025-06-08-11-11-07.gh-issue-135234.wJCdh0.rst b/Misc/NEWS.d/next/Library/2025-06-08-11-11-07.gh-issue-135234.wJCdh0.rst new file mode 100644 index 00000000000..e1c11e46735 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-08-11-11-07.gh-issue-135234.wJCdh0.rst @@ -0,0 +1,3 @@ +:mod:`hashlib`: improve exception messages when an OpenSSL function failed. +When memory allocation fails on OpenSSL's side, a :exc:`MemoryError` is +raised instead of a :exc:`ValueError`. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-08-14-50-34.gh-issue-135276.ZLUhV1.rst b/Misc/NEWS.d/next/Library/2025-06-08-14-50-34.gh-issue-135276.ZLUhV1.rst new file mode 100644 index 00000000000..a8fbd48d08a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-08-14-50-34.gh-issue-135276.ZLUhV1.rst @@ -0,0 +1,6 @@ +Synchronized zipfile.Path with zipp 3.23, including improved performance of +:meth:`zipfile.Path.open` for non-reading modes, rely on +:func:`functools.cached_property` to cache values on the instance. Rely on +``save_method_args`` to save the initialization method arguments. Fixed +``.name``, ``.stem`` and other basename-based properties on Windows when +working with a zipfile on disk. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-00-42-30.gh-issue-135321.UHh9jT.rst b/Misc/NEWS.d/next/Library/2025-06-10-00-42-30.gh-issue-135321.UHh9jT.rst new file mode 100644 index 00000000000..9e63d8e28b7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-10-00-42-30.gh-issue-135321.UHh9jT.rst @@ -0,0 +1 @@ +Raise a correct exception for values greater than 0x7fffffff for the ``BINSTRING`` opcode in the C implementation of :mod:`pickle`. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-16-11-00.gh-issue-133967.P0c24q.rst b/Misc/NEWS.d/next/Library/2025-06-10-16-11-00.gh-issue-133967.P0c24q.rst new file mode 100644 index 00000000000..1976981727e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-10-16-11-00.gh-issue-133967.P0c24q.rst @@ -0,0 +1 @@ +Do not normalize :mod:`locale` name 'C.UTF-8' to 'en_US.UTF-8'. diff --git a/Misc/NEWS.d/next/Library/2025-06-12-18-15-31.gh-issue-135429.mch75_.rst b/Misc/NEWS.d/next/Library/2025-06-12-18-15-31.gh-issue-135429.mch75_.rst new file mode 100644 index 00000000000..b5213520a95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-12-18-15-31.gh-issue-135429.mch75_.rst @@ -0,0 +1 @@ +Fix the argument mismatch in ``_lsprof`` for ``PY_THROW`` event. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst new file mode 100644 index 00000000000..25599a865b7 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst @@ -0,0 +1,4 @@ +The cases generator no longer accepts type annotations on stack items. +Conversions to non-default types are now done explictly in bytecodes.c and +optimizer_bytecodes.c. This will simplify code generation for top-of-stack +caching and other future features. diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index eecf7a1c8a1..d7acfc6a06a 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -17,6 +17,7 @@ static const char PyCursesVersion[] = "2.1"; #include "Python.h" +#define CURSES_PANEL_MODULE #include "py_curses.h" #if defined(HAVE_NCURSESW_PANEL_H) @@ -28,10 +29,12 @@ static const char PyCursesVersion[] = "2.1"; #endif typedef struct { - PyObject *PyCursesError; + PyObject *error; PyTypeObject *PyCursesPanel_Type; } _curses_panel_state; +typedef struct PyCursesPanelObject PyCursesPanelObject; + static inline _curses_panel_state * get_curses_panel_state(PyObject *module) { @@ -40,11 +43,30 @@ get_curses_panel_state(PyObject *module) return (_curses_panel_state *)state; } +static inline _curses_panel_state * +get_curses_panel_state_by_panel(PyCursesPanelObject *panel) +{ + /* + * Note: 'state' may be NULL if Py_TYPE(panel) is not a heap + * type associated with this module, but the compiler would + * have likely already complained with an "invalid pointer + * type" at compile-time. + * + * To make it more robust, all functions recovering a module's + * state from an object should expect to return NULL with an + * exception set (in contrast to functions recovering a module's + * state from a module itself). + */ + void *state = PyType_GetModuleState(Py_TYPE(panel)); + assert(state != NULL); + return (_curses_panel_state *)state; +} + static int _curses_panel_clear(PyObject *mod) { _curses_panel_state *state = get_curses_panel_state(mod); - Py_CLEAR(state->PyCursesError); + Py_CLEAR(state->error); Py_CLEAR(state->PyCursesPanel_Type); return 0; } @@ -54,7 +76,7 @@ _curses_panel_traverse(PyObject *mod, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(mod)); _curses_panel_state *state = get_curses_panel_state(mod); - Py_VISIT(state->PyCursesError); + Py_VISIT(state->error); Py_VISIT(state->PyCursesPanel_Type); return 0; } @@ -65,28 +87,149 @@ _curses_panel_free(void *mod) (void)_curses_panel_clear((PyObject *)mod); } +/* Utility Error Procedures + * + * The naming and implementations are identical to those in _cursesmodule.c. + * Functions that are not yet needed (for instance, reporting an ERR value + * from a module-wide function, namely curses_panel_set_error()) are + * omitted and should only be added if needed. + */ + +static void +_curses_panel_format_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname, + const char *return_value, + const char *default_message) +{ + assert(!PyErr_Occurred()); + if (python_funcname == NULL && curses_funcname == NULL) { + PyErr_SetString(state->error, default_message); + } + else if (python_funcname == NULL) { + (void)PyErr_Format(state->error, CURSES_ERROR_FORMAT, + curses_funcname, return_value); + } + else { + assert(python_funcname != NULL); + (void)PyErr_Format(state->error, CURSES_ERROR_VERBOSE_FORMAT, + curses_funcname, python_funcname, return_value); + } +} + +/* + * Format a curses error for a function that returned ERR. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_ERR' + * message instead. + */ +static void +_curses_panel_set_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "ERR", catchall_ERR); +} + +/* + * Format a curses error for a function that returned NULL. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_NULL' + * message instead. + */ +static void +_curses_panel_set_null_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "NULL", catchall_NULL); +} + +/* Same as _curses_panel_set_null_error() for a module object. */ +static void +curses_panel_set_null_error(PyObject *module, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state(module); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_error() for a panel object. */ +static void +curses_panel_panel_set_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_null_error() for a panel object. */ +static void +curses_panel_panel_set_null_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* + * Indicate that a panel object couldn't be found. + * + * Use it for the following constructions: + * + * PROC caller_funcname: + * pan = called_funcname() + * find_po(panel) + * + * PROC caller_funcname: + * find_po(self->pan) +*/ +static void +curses_panel_notfound_error(const char *called_funcname, + const char *caller_funcname) +{ + assert(!(called_funcname == NULL && caller_funcname == NULL)); + if (caller_funcname == NULL) { + (void)PyErr_Format(PyExc_RuntimeError, + "%s(): cannot find panel object", + called_funcname); + } + else { + (void)PyErr_Format(PyExc_RuntimeError, + "%s() (called by %s()): cannot find panel object", + called_funcname, caller_funcname); + } +} + /* Utility Functions */ /* - * Check the return code from a curses function and return None - * or raise an exception as appropriate. + * Check the return code from a curses function, returning None + * on success and setting an exception on error. */ +/* + * Return None if 'code' is different from ERR (implementation-defined). + * Otherwise, set an exception using curses_panel_panel_set_error() and + * the remaining arguments, and return NULL. + */ static PyObject * -PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) +curses_panel_panel_check_err(PyCursesPanelObject *panel, int code, + const char *curses_funcname, + const char *python_funcname) { if (code != ERR) { Py_RETURN_NONE; } - else { - if (fname == NULL) { - PyErr_SetString(state->PyCursesError, catchall_ERR); - } - else { - PyErr_Format(state->PyCursesError, "%s() returned ERR", fname); - } - return NULL; - } + curses_panel_panel_set_error(panel, curses_funcname, python_funcname); + return NULL; } /***************************************************************************** @@ -95,7 +238,7 @@ PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) /* Definition of the panel object and panel type */ -typedef struct { +typedef struct PyCursesPanelObject { PyObject_HEAD PANEL *pan; PyCursesWindowObject *wo; /* for reference counts */ @@ -144,8 +287,11 @@ insert_lop(PyCursesPanelObject *po) return 0; } -/* Remove the panel object from lop */ -static void +/* Remove the panel object from lop. + * + * Return -1 on error but do NOT set an exception; otherwise return 0. + */ +static int remove_lop(PyCursesPanelObject *po) { list_of_panels *temp, *n; @@ -154,25 +300,23 @@ remove_lop(PyCursesPanelObject *po) if (temp->po == po) { lop = temp->next; PyMem_Free(temp); - return; + return 0; } while (temp->next == NULL || temp->next->po != po) { if (temp->next == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "remove_lop: can't find Panel Object"); - return; + return -1; } temp = temp->next; } n = temp->next->next; PyMem_Free(temp->next); temp->next = n; - return; + return 0; } /* Return the panel object that corresponds to pan */ static PyCursesPanelObject * -find_po(PANEL *pan) +find_po_impl(PANEL *pan) { list_of_panels *temp; for (temp = lop; temp->po->pan != pan; temp = temp->next) @@ -180,6 +324,17 @@ find_po(PANEL *pan) return temp->po; } +/* Same as find_po_impl() but with caller context information. */ +static PyCursesPanelObject * +find_po(PANEL *pan, const char *called_funcname, const char *caller_funcname) +{ + PyCursesPanelObject *res = find_po_impl(pan); + if (res == NULL) { + curses_panel_notfound_error(called_funcname, caller_funcname); + } + return res; +} + /*[clinic input] module _curses_panel class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" @@ -193,67 +348,59 @@ class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" /*[clinic input] _curses_panel.panel.bottom - cls: defining_class - Push the panel to the bottom of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=8ec7fbbc08554021 input=6b7d2c0578b5a1c4]*/ +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=7aa7d14d7e1d1ce6 input=b6c920c071b61e2e]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, bottom_panel(self->pan), "bottom"); + int rtn = bottom_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "bottom_panel", "bottom"); } /*[clinic input] _curses_panel.panel.hide - cls: defining_class - Hide the panel. This does not delete the object, it just makes the window on screen invisible. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc6ab7203cdc1450 input=1bfc741f473e6055]*/ +_curses_panel_panel_hide_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=a7bbbd523e1eab49 input=f6ab884e99386118]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, hide_panel(self->pan), "hide"); + int rtn = hide_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "hide_panel", "hide"); } /*[clinic input] _curses_panel.panel.show - cls: defining_class - Display the panel (which might have been hidden). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=dc3421de375f0409 input=8122e80151cb4379]*/ +_curses_panel_panel_show_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=6b4553ab45c97769 input=57b167bbefaa3755]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, show_panel(self->pan), "show"); + int rtn = show_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "show_panel", "show"); } /*[clinic input] _curses_panel.panel.top - cls: defining_class - Push panel to the top of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=10a072e511e873f7 input=1f372d597dda3379]*/ +_curses_panel_panel_top_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=0f5f2f8cdd2d1777 input=be33975ec3ca0e9a]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, top_panel(self->pan), "top"); + int rtn = top_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "top_panel", "top"); } /* Allocation and deallocation of Panel Objects */ @@ -287,13 +434,22 @@ PyCursesPanel_Dealloc(PyObject *self) tp = (PyObject *) Py_TYPE(po); obj = (PyObject *) panel_userptr(po->pan); if (obj) { - (void)set_panel_userptr(po->pan, NULL); Py_DECREF(obj); + if (set_panel_userptr(po->pan, NULL) == ERR) { + curses_panel_panel_set_error(po, "set_panel_userptr", "__del__"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } + } + if (del_panel(po->pan) == ERR && !PyErr_Occurred()) { + curses_panel_panel_set_error(po, "del_panel", "__del__"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); } - (void)del_panel(po->pan); if (po->wo != NULL) { Py_DECREF(po->wo); - remove_lop(po); + if (remove_lop(po) < 0) { + PyErr_SetString(PyExc_RuntimeError, "__del__: no panel object to delete"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } } PyObject_Free(po); Py_DECREF(tp); @@ -315,18 +471,11 @@ _curses_panel_panel_above_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_above(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on top of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "above"); + return Py_XNewRef(po); } /* panel_below(NULL) returns the top panel in the stack. To get @@ -345,18 +494,11 @@ _curses_panel_panel_below_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_below(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on the bottom of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "below"); + return Py_XNewRef(po); } /*[clinic input] @@ -378,7 +520,6 @@ _curses_panel_panel_hidden_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.move - cls: defining_class y: int x: int / @@ -387,12 +528,11 @@ Move the panel to the screen coordinates (y, x). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x) -/*[clinic end generated code: output=ce546c93e56867da input=60a0e7912ff99849]*/ +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x) +/*[clinic end generated code: output=d867535a89777415 input=e0b36b78acc03fba]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, move_panel(self->pan, y, x), "move_panel"); + int rtn = move_panel(self->pan, y, x); + return curses_panel_panel_check_err(self, rtn, "move_panel", "move"); } /*[clinic input] @@ -411,7 +551,6 @@ _curses_panel_panel_window_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.replace - cls: defining_class win: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") / @@ -420,22 +559,17 @@ Change the window associated with the panel to the window win. static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win) -/*[clinic end generated code: output=c71f95c212d58ae7 input=dbec7180ece41ff5]*/ +/*[clinic end generated code: output=2253a95f7b287255 input=4b1c4283987d9dfa]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - - PyCursesPanelObject *po = find_po(self->pan); + PyCursesPanelObject *po = find_po(self->pan, "replace", NULL); if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "replace_panel: can't find Panel Object"); return NULL; } int rtn = replace_panel(self->pan, win->win); if (rtn == ERR) { - PyErr_SetString(state->PyCursesError, "replace_panel() returned ERR"); + curses_panel_panel_set_error(self, "replace_panel", "replace"); return NULL; } Py_SETREF(po->wo, (PyCursesWindowObject*)Py_NewRef(win)); @@ -445,7 +579,6 @@ _curses_panel_panel_replace_impl(PyCursesPanelObject *self, /*[clinic input] _curses_panel.panel.set_userptr - cls: defining_class obj: object / @@ -454,8 +587,8 @@ Set the panel's user pointer to obj. static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj) -/*[clinic end generated code: output=db74f3db07b28080 input=e3fee2ff7b1b8e48]*/ + PyObject *obj) +/*[clinic end generated code: output=7fa1fd23f69db71e input=d2c6a9dbefabbf39]*/ { PyCursesInitialised; Py_INCREF(obj); @@ -464,34 +597,27 @@ _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, if (rc == ERR) { /* In case of an ncurses error, decref the new object again */ Py_DECREF(obj); + curses_panel_panel_set_error(self, "set_panel_userptr", "set_userptr"); + return NULL; } - else { - Py_XDECREF(oldobj); - } - - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, rc, "set_panel_userptr"); + Py_XDECREF(oldobj); + Py_RETURN_NONE; } /*[clinic input] _curses_panel.panel.userptr - cls: defining_class - Return the user pointer for the panel. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls) -/*[clinic end generated code: output=eea6e6f39ffc0179 input=f22ca4f115e30a80]*/ +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=e849c307b5dc9237 input=f78b7a47aef0fd50]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - PyCursesInitialised; PyObject *obj = (PyObject *) panel_userptr(self->pan); if (obj == NULL) { - PyErr_SetString(state->PyCursesError, "no userptr set"); + curses_panel_panel_set_null_error(self, "panel_userptr", "userptr"); return NULL; } @@ -552,18 +678,11 @@ _curses_panel_bottom_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_above(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "bottom_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -579,14 +698,13 @@ static PyObject * _curses_panel_new_panel_impl(PyObject *module, PyCursesWindowObject *win) /*[clinic end generated code: output=45e948e0176a9bd2 input=74d4754e0ebe4800]*/ { - _curses_panel_state *state = get_curses_panel_state(module); - PANEL *pan = new_panel(win->win); if (pan == NULL) { - PyErr_SetString(state->PyCursesError, catchall_NULL); + curses_panel_set_null_error(module, "new_panel", NULL); return NULL; } - return (PyObject *)PyCursesPanel_New(state, pan, win); + _curses_panel_state *state = get_curses_panel_state(module); + return PyCursesPanel_New(state, pan, win); } @@ -610,18 +728,11 @@ _curses_panel_top_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_below(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "top_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -673,10 +784,10 @@ _curses_panel_exec(PyObject *mod) } /* For exception _curses_panel.error */ - state->PyCursesError = PyErr_NewException( + state->error = PyErr_NewException( "_curses_panel.error", NULL, NULL); - if (PyModule_AddObjectRef(mod, "error", state->PyCursesError) < 0) { + if (PyModule_AddObjectRef(mod, "error", state->error) < 0) { return -1; } diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 331275076d7..ce9603d5db8 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -260,7 +260,7 @@ static PyModuleDef _hashlibmodule; typedef struct { PyTypeObject *HASH_type; // based on EVP_MD - PyTypeObject *HMACtype; + PyTypeObject *HMAC_type; #ifdef PY_OPENSSL_HAS_SHAKE PyTypeObject *HASHXOF_type; // based on EVP_MD #endif @@ -300,19 +300,20 @@ typedef struct { #include "clinic/_hashopenssl.c.h" /*[clinic input] module _hashlib -class _hashlib.HASH "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPtype" -class _hashlib.HASHXOF "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPXOFtype" -class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module))->HMACtype" +class _hashlib.HASH "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->HASH_type" +class _hashlib.HASHXOF "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->HASHXOF_type" +class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module))->HMAC_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4f6b8873ed13d1ff]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=eb805ce4b90b1b31]*/ /* LCOV_EXCL_START */ /* Set an exception of given type using the given OpenSSL error code. */ static void -set_ssl_exception_from_errcode(PyObject *exc, unsigned long errcode) +set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) { + assert(exc_type != NULL); assert(errcode != 0); /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ @@ -321,13 +322,29 @@ set_ssl_exception_from_errcode(PyObject *exc, unsigned long errcode) const char *reason = ERR_reason_error_string(errcode); if (lib && func) { - PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); + PyErr_Format(exc_type, "[%s: %s] %s", lib, func, reason); } else if (lib) { - PyErr_Format(exc, "[%s] %s", lib, reason); + PyErr_Format(exc_type, "[%s] %s", lib, reason); } else { - PyErr_SetString(exc, reason); + PyErr_SetString(exc_type, reason); + } +} + +/* + * Get an appropriate exception type for the given OpenSSL error code. + * + * The exception type depends on the error code reason. + */ +static PyObject * +get_smart_ssl_exception_type(unsigned long errcode, PyObject *default_exc_type) +{ + switch (ERR_GET_REASON(errcode)) { + case ERR_R_MALLOC_FAILURE: + return PyExc_MemoryError; + default: + return default_exc_type; } } @@ -335,80 +352,171 @@ set_ssl_exception_from_errcode(PyObject *exc, unsigned long errcode) * Set an exception of given type. * * By default, the exception's message is constructed by using the last SSL - * error that occurred. If no error occurred, the 'fallback_format' is used - * to create a C-style formatted fallback message. + * error that occurred. If no error occurred, the 'fallback_message' is used + * to create an exception message. */ static void -raise_ssl_error(PyObject *exc, const char *fallback_format, ...) +raise_ssl_error(PyObject *exc_type, const char *fallback_message) +{ + assert(fallback_message != NULL); + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + ERR_clear_error(); + set_ssl_exception_from_errcode(exc_type, errcode); + } + else { + PyErr_SetString(exc_type, fallback_message); + } +} + +/* Same as raise_ssl_error() but with a C-style formatted message. */ +static void +raise_ssl_error_f(PyObject *exc_type, const char *fallback_format, ...) { assert(fallback_format != NULL); unsigned long errcode = ERR_peek_last_error(); if (errcode) { ERR_clear_error(); - set_ssl_exception_from_errcode(exc, errcode); + set_ssl_exception_from_errcode(exc_type, errcode); } else { va_list vargs; va_start(vargs, fallback_format); - PyErr_FormatV(exc, fallback_format, vargs); + PyErr_FormatV(exc_type, fallback_format, vargs); + va_end(vargs); + } +} + +/* Same as raise_ssl_error_f() with smart exception types. */ +static void +raise_smart_ssl_error_f(PyObject *exc_type, const char *fallback_format, ...) +{ + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + ERR_clear_error(); + exc_type = get_smart_ssl_exception_type(errcode, exc_type); + set_ssl_exception_from_errcode(exc_type, errcode); + } + else { + va_list vargs; + va_start(vargs, fallback_format); + PyErr_FormatV(exc_type, fallback_format, vargs); va_end(vargs); } } /* - * Set an exception with a generic default message after an error occurred. - * - * It can also be used without previous calls to SSL built-in functions, - * in which case a generic error message is provided. + * Raise a ValueError with a default message after an error occurred. + * It can also be used without previous calls to SSL built-in functions. */ static inline void -notify_ssl_error_occurred(void) +notify_ssl_error_occurred(const char *message) { - raise_ssl_error(PyExc_ValueError, "no reason supplied"); + raise_ssl_error(PyExc_ValueError, message); } -/* LCOV_EXCL_STOP */ -static const char * -get_openssl_evp_md_utf8name(const EVP_MD *md) +/* Same as notify_ssl_error_occurred() for failed OpenSSL functions. */ +static inline void +notify_ssl_error_occurred_in(const char *funcname) { - assert(md != NULL); - int nid = EVP_MD_nid(md); - const char *name = NULL; - const py_hashentry_t *h; + raise_ssl_error_f(PyExc_ValueError, + "error in OpenSSL function %s()", funcname); +} - for (h = py_hashes; h->py_name != NULL; h++) { +/* Same as notify_ssl_error_occurred_in() with smart exception types. */ +static inline void +notify_smart_ssl_error_occurred_in(const char *funcname) +{ + raise_smart_ssl_error_f(PyExc_ValueError, + "error in OpenSSL function %s()", funcname); +} +/* LCOV_EXCL_STOP */ + +/* + * OpenSSL provides a way to go from NIDs to digest names for hash functions + * but lacks this granularity for MAC objects where it is not possible to get + * the underlying digest name (only the block size and digest size are allowed + * to be recovered). + * + * In addition, OpenSSL aliases pollute the list of known digest names + * as OpenSSL appears to have its own definition of alias. In particular, + * the resulting list still contains duplicate and alternate names for several + * algorithms. + * + * Therefore, digest names, whether they are used by hash functions or HMAC, + * are handled through EVP_MD objects or directly by using some NID. + */ + +/* Get a cached entry by OpenSSL NID. */ +static const py_hashentry_t * +get_hashentry_by_nid(int nid) +{ + for (const py_hashentry_t *h = py_hashes; h->py_name != NULL; h++) { if (h->ossl_nid == nid) { - name = h->py_name; - break; + return h; } } + return NULL; +} + +/* + * Convert the NID to a string via OBJ_nid2*() functions. + * + * If 'nid' cannot be resolved, set an exception and return NULL. + */ +static const char * +get_asn1_utf8name_by_nid(int nid) +{ + const char *name = OBJ_nid2ln(nid); if (name == NULL) { - /* Ignore aliased names and only use long, lowercase name. The aliases - * pollute the list and OpenSSL appears to have its own definition of - * alias as the resulting list still contains duplicate and alternate - * names for several algorithms. - */ - name = OBJ_nid2ln(nid); - if (name == NULL) - name = OBJ_nid2sn(nid); + // In OpenSSL 3.0 and later, OBJ_nid*() are thread-safe and may raise. + assert(ERR_peek_last_error() != 0); + if (ERR_GET_REASON(ERR_peek_last_error()) != OBJ_R_UNKNOWN_NID) { + goto error; + } + // fallback to short name and unconditionally propagate errors + name = OBJ_nid2sn(nid); + if (name == NULL) { + goto error; + } } return name; + +error: + raise_ssl_error_f(PyExc_ValueError, "cannot resolve NID %d", nid); + return NULL; } -static PyObject * -get_openssl_evp_md_name(const EVP_MD *md) +/* + * Convert the NID to an OpenSSL digest name. + * + * On error, set an exception and return NULL. + */ +static const char * +get_hashlib_utf8name_by_nid(int nid) +{ + const py_hashentry_t *e = get_hashentry_by_nid(nid); + return e ? e->py_name : get_asn1_utf8name_by_nid(nid); +} + +/* Same as get_hashlib_utf8name_by_nid() but using an EVP_MD object. */ +static const char * +get_hashlib_utf8name_by_evp_md(const EVP_MD *md) { - const char *name = get_openssl_evp_md_utf8name(md); - return PyUnicode_FromString(name); + assert(md != NULL); + return get_hashlib_utf8name_by_nid(EVP_MD_nid(md)); } -/* Get EVP_MD by HID and purpose */ +/* + * Get a new reference to an EVP_MD object described by name and purpose. + * + * If 'name' is an OpenSSL indexed name, the return value is cached. + */ static PY_EVP_MD * get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, Py_hash_type py_ht) { - PY_EVP_MD *digest = NULL; - PY_EVP_MD *other_digest = NULL; + PY_EVP_MD *digest = NULL, *other_digest = NULL; _hashlibstate *state = get_hashlib_state(module); py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( state->hashtable, (const void*)name @@ -442,15 +550,16 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, #endif } break; + default: + goto invalid_hash_type; } // if another thread same thing at same time make sure we got same ptr assert(other_digest == NULL || other_digest == digest); - if (digest != NULL) { - if (other_digest == NULL) { - PY_EVP_MD_up_ref(digest); - } + if (digest != NULL && other_digest == NULL) { + PY_EVP_MD_up_ref(digest); } - } else { + } + else { // Fall back for looking up an unindexed OpenSSL specific name. switch (py_ht) { case Py_ht_evp: @@ -461,55 +570,82 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, case Py_ht_evp_nosecurity: digest = PY_EVP_MD_fetch(name, "-fips"); break; + default: + goto invalid_hash_type; } } if (digest == NULL) { - raise_ssl_error(state->unsupported_digestmod_error, - "unsupported hash type %s", name); + raise_ssl_error_f(state->unsupported_digestmod_error, + "unsupported digest name: %s", name); return NULL; } return digest; + +invalid_hash_type: + assert(digest == NULL); + PyErr_Format(PyExc_SystemError, "unsupported hash type %d", py_ht); + return NULL; } -/* Get digest EVP_MD from object +/* + * Raise an exception indicating that 'digestmod' is not supported. + */ +static void +raise_unsupported_digestmod_error(PyObject *module, PyObject *digestmod) +{ + _hashlibstate *state = get_hashlib_state(module); + PyErr_Format(state->unsupported_digestmod_error, + "Unsupported digestmod %R", digestmod); +} + +/* + * Get a new reference to an EVP_MD described by 'digestmod' and purpose. + * + * On error, set an exception and return NULL. * - * * string - * * _hashopenssl builtin function + * Parameters * - * on error returns NULL with exception set. + * digestmod A digest name or a _hashopenssl builtin function + * py_ht The message digest purpose. */ static PY_EVP_MD * -get_openssl_evp_md(PyObject *module, PyObject *digestmod, - Py_hash_type py_ht) +get_openssl_evp_md(PyObject *module, PyObject *digestmod, Py_hash_type py_ht) { - PyObject *name_obj = NULL; const char *name; - if (PyUnicode_Check(digestmod)) { - name_obj = digestmod; - } else { - _hashlibstate *state = get_hashlib_state(module); - // borrowed ref - name_obj = PyDict_GetItemWithError(state->constructs, digestmod); + name = PyUnicode_AsUTF8(digestmod); + } + else { + PyObject *dict = get_hashlib_state(module)->constructs; + assert(dict != NULL); + PyObject *borrowed_ref = PyDict_GetItemWithError(dict, digestmod); + name = borrowed_ref == NULL ? NULL : PyUnicode_AsUTF8(borrowed_ref); } - if (name_obj == NULL) { + if (name == NULL) { if (!PyErr_Occurred()) { - _hashlibstate *state = get_hashlib_state(module); - PyErr_Format( - state->unsupported_digestmod_error, - "Unsupported digestmod %R", digestmod); + raise_unsupported_digestmod_error(module, digestmod); } return NULL; } + return get_openssl_evp_md_by_utf8name(module, name, py_ht); +} - name = PyUnicode_AsUTF8(name_obj); - if (name == NULL) { +// --- OpenSSL HASH wrappers -------------------------------------------------- + +/* Thin wrapper around EVP_MD_CTX_new() which sets an exception on failure. */ +static EVP_MD_CTX * +py_wrapper_EVP_MD_CTX_new(void) +{ + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + if (ctx == NULL) { + PyErr_NoMemory(); return NULL; } - - return get_openssl_evp_md_by_utf8name(module, name, py_ht); + return ctx; } +// --- HASH interface --------------------------------------------------------- + static HASHobject * new_hash_object(PyTypeObject *type) { @@ -519,10 +655,9 @@ new_hash_object(PyTypeObject *type) } HASHLIB_INIT_MUTEX(retval); - retval->ctx = EVP_MD_CTX_new(); + retval->ctx = py_wrapper_EVP_MD_CTX_new(); if (retval->ctx == NULL) { Py_DECREF(retval); - PyErr_NoMemory(); return NULL; } @@ -540,7 +675,7 @@ _hashlib_HASH_hash(HASHobject *self, const void *vp, Py_ssize_t len) else process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int); if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestUpdate)); return -1; } len -= process; @@ -568,7 +703,11 @@ _hashlib_HASH_copy_locked(HASHobject *self, EVP_MD_CTX *new_ctx_p) ENTER_HASHLIB(self); result = EVP_MD_CTX_copy(new_ctx_p, self->ctx); LEAVE_HASHLIB(self); - return result; + if (result == 0) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(EVP_MD_CTX_copy)); + return -1; + } + return 0; } /* External methods for a hash object */ @@ -588,14 +727,36 @@ _hashlib_HASH_copy_impl(HASHobject *self) if ((newobj = new_hash_object(Py_TYPE(self))) == NULL) return NULL; - if (!_hashlib_HASH_copy_locked(self, newobj->ctx)) { + if (_hashlib_HASH_copy_locked(self, newobj->ctx) < 0) { Py_DECREF(newobj); - notify_ssl_error_occurred(); return NULL; } return (PyObject *)newobj; } +static Py_ssize_t +_hashlib_HASH_digest_compute(HASHobject *self, unsigned char *digest) +{ + EVP_MD_CTX *ctx = py_wrapper_EVP_MD_CTX_new(); + if (ctx == NULL) { + return -1; + } + if (_hashlib_HASH_copy_locked(self, ctx) < 0) { + goto error; + } + Py_ssize_t digest_size = EVP_MD_CTX_size(ctx); + if (!EVP_DigestFinal(ctx, digest, NULL)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinal)); + goto error; + } + EVP_MD_CTX_free(ctx); + return digest_size; + +error: + EVP_MD_CTX_free(ctx); + return -1; +} + /*[clinic input] _hashlib.HASH.digest @@ -607,32 +768,8 @@ _hashlib_HASH_digest_impl(HASHobject *self) /*[clinic end generated code: output=3fc6f9671d712850 input=d8d528d6e50af0de]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; - EVP_MD_CTX *temp_ctx; - PyObject *retval; - unsigned int digest_size; - - temp_ctx = EVP_MD_CTX_new(); - if (temp_ctx == NULL) { - PyErr_NoMemory(); - return NULL; - } - - if (!_hashlib_HASH_copy_locked(self, temp_ctx)) { - goto error; - } - digest_size = EVP_MD_CTX_size(temp_ctx); - if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - goto error; - } - - retval = PyBytes_FromStringAndSize((const char *)digest, digest_size); - EVP_MD_CTX_free(temp_ctx); - return retval; - -error: - EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); - return NULL; + Py_ssize_t n = _hashlib_HASH_digest_compute(self, digest); + return n < 0 ? NULL : PyBytes_FromStringAndSize((const char *)digest, n); } /*[clinic input] @@ -646,32 +783,8 @@ _hashlib_HASH_hexdigest_impl(HASHobject *self) /*[clinic end generated code: output=1b8e60d9711e7f4d input=ae7553f78f8372d8]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; - EVP_MD_CTX *temp_ctx; - unsigned int digest_size; - - temp_ctx = EVP_MD_CTX_new(); - if (temp_ctx == NULL) { - PyErr_NoMemory(); - return NULL; - } - - /* Get the raw (binary) digest value */ - if (!_hashlib_HASH_copy_locked(self, temp_ctx)) { - goto error; - } - digest_size = EVP_MD_CTX_size(temp_ctx); - if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - goto error; - } - - EVP_MD_CTX_free(temp_ctx); - - return _Py_strhex((const char *)digest, (Py_ssize_t)digest_size); - -error: - EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); - return NULL; + Py_ssize_t n = _hashlib_HASH_digest_compute(self, digest); + return n < 0 ? NULL : _Py_strhex((const char *)digest, n); } /*[clinic input] @@ -742,10 +855,12 @@ _hashlib_HASH_get_name(PyObject *op, void *Py_UNUSED(closure)) HASHobject *self = HASHobject_CAST(op); const EVP_MD *md = EVP_MD_CTX_md(self->ctx); if (md == NULL) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred("missing EVP_MD for HASH context"); return NULL; } - return get_openssl_evp_md_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + return name == NULL ? NULL : PyUnicode_FromString(name); } static PyGetSetDef HASH_getsets[] = { @@ -829,20 +944,20 @@ _hashlib_HASHXOF_digest_impl(HASHobject *self, Py_ssize_t length) return NULL; } - temp_ctx = EVP_MD_CTX_new(); + temp_ctx = py_wrapper_EVP_MD_CTX_new(); if (temp_ctx == NULL) { Py_DECREF(retval); - PyErr_NoMemory(); return NULL; } - if (!_hashlib_HASH_copy_locked(self, temp_ctx)) { + if (_hashlib_HASH_copy_locked(self, temp_ctx) < 0) { goto error; } if (!EVP_DigestFinalXOF(temp_ctx, (unsigned char*)PyBytes_AS_STRING(retval), length)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinalXOF)); goto error; } @@ -852,7 +967,6 @@ _hashlib_HASHXOF_digest_impl(HASHobject *self, Py_ssize_t length) error: Py_DECREF(retval); EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); return NULL; } @@ -878,18 +992,18 @@ _hashlib_HASHXOF_hexdigest_impl(HASHobject *self, Py_ssize_t length) return NULL; } - temp_ctx = EVP_MD_CTX_new(); + temp_ctx = py_wrapper_EVP_MD_CTX_new(); if (temp_ctx == NULL) { PyMem_Free(digest); - PyErr_NoMemory(); return NULL; } /* Get the raw (binary) digest value */ - if (!_hashlib_HASH_copy_locked(self, temp_ctx)) { + if (_hashlib_HASH_copy_locked(self, temp_ctx) < 0) { goto error; } if (!EVP_DigestFinalXOF(temp_ctx, digest, length)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinalXOF)); goto error; } @@ -902,7 +1016,6 @@ _hashlib_HASHXOF_hexdigest_impl(HASHobject *self, Py_ssize_t length) error: PyMem_Free(digest); EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); return NULL; } @@ -1006,7 +1119,7 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, int result = EVP_DigestInit_ex(self->ctx, digest, NULL); if (!result) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestInit_ex)); Py_CLEAR(self); goto exit; } @@ -1415,7 +1528,7 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, if (!retval) { Py_CLEAR(key_obj); - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(PKCS5_PBKDF2_HMAC)); goto end; } @@ -1491,8 +1604,8 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, /* let OpenSSL validate the rest */ retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0); if (!retval) { - raise_ssl_error(PyExc_ValueError, - "Invalid parameter combination for n, r, p, maxmem."); + notify_ssl_error_occurred( + "Invalid parameter combination for n, r, p, maxmem."); return NULL; } @@ -1513,7 +1626,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, if (!retval) { Py_CLEAR(key_obj); - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_PBE_scrypt)); return NULL; } return key_obj; @@ -1570,7 +1683,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, PY_EVP_MD_free(evp); if (result == NULL) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC)); return NULL; } return PyBytes_FromStringAndSize((const char*)md, md_len); @@ -1579,6 +1692,18 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, /* OpenSSL-based HMAC implementation */ +/* Thin wrapper around HMAC_CTX_new() which sets an exception on failure. */ +static HMAC_CTX * +py_openssl_wrapper_HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = HMAC_CTX_new(); + if (ctx == NULL) { + PyErr_NoMemory(); + return NULL; + } + return ctx; +} + static int _hmac_update(HMACobject*, PyObject*); static const EVP_MD * @@ -1586,7 +1711,7 @@ _hashlib_hmac_get_md(HMACobject *self) { const EVP_MD *md = HMAC_CTX_get_md(self->ctx); if (md == NULL) { - raise_ssl_error(PyExc_ValueError, "missing EVP_MD for HMAC context"); + notify_ssl_error_occurred("missing EVP_MD for HMAC context"); } return md; } @@ -1628,22 +1753,21 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - ctx = HMAC_CTX_new(); + ctx = py_openssl_wrapper_HMAC_CTX_new(); if (ctx == NULL) { PY_EVP_MD_free(digest); - PyErr_NoMemory(); goto error; } r = HMAC_Init_ex(ctx, key->buf, (int)key->len, digest, NULL /* impl */); PY_EVP_MD_free(digest); if (r == 0) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Init_ex)); goto error; } _hashlibstate *state = get_hashlib_state(module); - self = PyObject_New(HMACobject, state->HMACtype); + self = PyObject_New(HMACobject, state->HMAC_type); if (self == NULL) { goto error; } @@ -1673,7 +1797,11 @@ locked_HMAC_CTX_copy(HMAC_CTX *new_ctx_p, HMACobject *self) ENTER_HASHLIB(self); result = HMAC_CTX_copy(new_ctx_p, self->ctx); LEAVE_HASHLIB(self); - return result; + if (result == 0) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(HMAC_CTX_copy)); + return -1; + } + return 0; } /* returning 0 means that an error occurred and an exception is set */ @@ -1687,7 +1815,7 @@ _hashlib_hmac_digest_size(HMACobject *self) unsigned int digest_size = EVP_MD_size(md); assert(digest_size <= EVP_MAX_MD_SIZE); if (digest_size == 0) { - raise_ssl_error(PyExc_ValueError, "invalid digest size"); + notify_ssl_error_occurred("invalid digest size"); } return digest_size; } @@ -1720,7 +1848,7 @@ _hmac_update(HMACobject *self, PyObject *obj) PyBuffer_Release(&view); if (r == 0) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Update)); return 0; } return 1; @@ -1738,13 +1866,12 @@ _hashlib_HMAC_copy_impl(HMACobject *self) { HMACobject *retval; - HMAC_CTX *ctx = HMAC_CTX_new(); + HMAC_CTX *ctx = py_openssl_wrapper_HMAC_CTX_new(); if (ctx == NULL) { - return PyErr_NoMemory(); + return NULL; } - if (!locked_HMAC_CTX_copy(ctx, self)) { + if (locked_HMAC_CTX_copy(ctx, self) < 0) { HMAC_CTX_free(ctx); - notify_ssl_error_occurred(); return NULL; } @@ -1775,20 +1902,15 @@ _hmac_dealloc(PyObject *op) static PyObject * _hmac_repr(PyObject *op) { + const char *digest_name; HMACobject *self = HMACobject_CAST(op); const EVP_MD *md = _hashlib_hmac_get_md(self); - if (md == NULL) { - return NULL; - } - PyObject *digest_name = get_openssl_evp_md_name(md); + digest_name = md == NULL ? NULL : get_hashlib_utf8name_by_evp_md(md); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *repr = PyUnicode_FromFormat( - "<%U HMAC object @ %p>", digest_name, self - ); - Py_DECREF(digest_name); - return repr; + return PyUnicode_FromFormat("<%s HMAC object @ %p>", digest_name, self); } /*[clinic input] @@ -1811,20 +1933,18 @@ _hashlib_HMAC_update_impl(HMACobject *self, PyObject *msg) static int _hmac_digest(HMACobject *self, unsigned char *buf, unsigned int len) { - HMAC_CTX *temp_ctx = HMAC_CTX_new(); + HMAC_CTX *temp_ctx = py_openssl_wrapper_HMAC_CTX_new(); if (temp_ctx == NULL) { - (void)PyErr_NoMemory(); return 0; } - if (!locked_HMAC_CTX_copy(temp_ctx, self)) { + if (locked_HMAC_CTX_copy(temp_ctx, self) < 0) { HMAC_CTX_free(temp_ctx); - notify_ssl_error_occurred(); return 0; } int r = HMAC_Final(temp_ctx, buf, &len); HMAC_CTX_free(temp_ctx); if (r == 0) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Final)); return 0; } return 1; @@ -1900,13 +2020,12 @@ _hashlib_hmac_get_name(PyObject *op, void *Py_UNUSED(closure)) if (md == NULL) { return NULL; } - PyObject *digest_name = get_openssl_evp_md_name(md); + const char *digest_name = get_hashlib_utf8name_by_evp_md(md); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *name = PyUnicode_FromFormat("hmac-%U", digest_name); - Py_DECREF(digest_name); - return name; + return PyUnicode_FromFormat("hmac-%s", digest_name); } static PyMethodDef HMAC_methods[] = { @@ -1982,7 +2101,9 @@ _openssl_hash_name_mapper(const EVP_MD *md, const char *from, return; } - py_name = get_openssl_evp_md_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + py_name = name == NULL ? NULL : PyUnicode_FromString(name); if (py_name == NULL) { state->error = 1; } else { @@ -2044,16 +2165,13 @@ _hashlib_get_fips_mode_impl(PyObject *module) #else ERR_clear_error(); int result = FIPS_mode(); - if (result == 0) { + if (result == 0 && ERR_peek_last_error()) { // "If the library was built without support of the FIPS Object Module, // then the function will return 0 with an error code of // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." // But 0 is also a valid result value. - unsigned long errcode = ERR_peek_last_error(); - if (errcode) { - notify_ssl_error_occurred(); - return -1; - } + notify_ssl_error_occurred_in(Py_STRINGIFY(FIPS_mode)); + return -1; } return result; #endif @@ -2204,7 +2322,7 @@ hashlib_traverse(PyObject *m, visitproc visit, void *arg) { _hashlibstate *state = get_hashlib_state(m); Py_VISIT(state->HASH_type); - Py_VISIT(state->HMACtype); + Py_VISIT(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE Py_VISIT(state->HASHXOF_type); #endif @@ -2218,7 +2336,7 @@ hashlib_clear(PyObject *m) { _hashlibstate *state = get_hashlib_state(m); Py_CLEAR(state->HASH_type); - Py_CLEAR(state->HMACtype); + Py_CLEAR(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE Py_CLEAR(state->HASHXOF_type); #endif @@ -2296,11 +2414,11 @@ hashlib_init_hmactype(PyObject *module) { _hashlibstate *state = get_hashlib_state(module); - state->HMACtype = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); - if (state->HMACtype == NULL) { + state->HMAC_type = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); + if (state->HMAC_type == NULL) { return -1; } - if (PyModule_AddType(module, state->HMACtype) < 0) { + if (PyModule_AddType(module, state->HMAC_type) < 0) { return -1; } return 0; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 095866eec7d..7784cdcd9ff 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -11,7 +11,7 @@ annotated by François Pinard, and converted to C by Raymond Hettinger. #endif #include "Python.h" -#include "pycore_list.h" // _PyList_ITEMS() +#include "pycore_list.h" // _PyList_ITEMS(), _PyList_AppendTakeRef() #include "clinic/_heapqmodule.c.h" @@ -117,6 +117,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush heap: object(subclass_of='&PyList_Type') @@ -128,13 +129,22 @@ Push item onto heap, maintaining the heap invariant. static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=912c094f47663935 input=7c69611f3698aceb]*/ +/*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { - if (PyList_Append(heap, item)) + if (item == NULL) { + PyErr_BadInternalCall(); return NULL; + } + + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { + return NULL; + } - if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) + if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) { return NULL; + } Py_RETURN_NONE; } @@ -171,6 +181,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heappop heap: object(subclass_of='&PyList_Type') @@ -181,7 +192,7 @@ Pop the smallest item off the heap, maintaining the heap invariant. static PyObject * _heapq_heappop_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=96dfe82d37d9af76 input=91487987a583c856]*/ +/*[clinic end generated code: output=96dfe82d37d9af76 input=ed396461b153dd51]*/ { return heappop_internal(heap, siftup); } @@ -207,6 +218,7 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec /*[clinic input] +@critical_section heap _heapq.heapreplace heap: object(subclass_of='&PyList_Type') @@ -226,12 +238,13 @@ this routine unless written as part of a conditional replacement: static PyObject * _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=82ea55be8fbe24b4 input=719202ac02ba10c8]*/ +/*[clinic end generated code: output=82ea55be8fbe24b4 input=9be1678b817ef1a9]*/ { return heapreplace_internal(heap, item, siftup); } /*[clinic input] +@critical_section heap _heapq.heappushpop heap: object(subclass_of='&PyList_Type') @@ -246,7 +259,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=5dc701f1eb4a4aa7]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ { PyObject *returnitem; int cmp; @@ -371,6 +384,7 @@ heapify_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heapify heap: object(subclass_of='&PyList_Type') @@ -381,7 +395,7 @@ Transform list into a heap, in-place, in O(len(heap)) time. static PyObject * _heapq_heapify_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=e63a636fcf83d6d0 input=53bb7a2166febb73]*/ +/*[clinic end generated code: output=e63a636fcf83d6d0 input=aaaaa028b9b6af08]*/ { return heapify_internal(heap, siftup); } @@ -481,6 +495,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush_max heap: object(subclass_of='&PyList_Type') @@ -492,9 +507,16 @@ Push item onto max heap, maintaining the heap invariant. static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=c869d5f9deb08277 input=4743d7db137b6e2b]*/ +/*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { - if (PyList_Append(heap, item)) { + if (item == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; } @@ -506,6 +528,7 @@ _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@critical_section heap _heapq.heappop_max heap: object(subclass_of='&PyList_Type') @@ -516,12 +539,13 @@ Maxheap variant of heappop. static PyObject * _heapq_heappop_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=2f051195ab404b77 input=e62b14016a5a26de]*/ +/*[clinic end generated code: output=2f051195ab404b77 input=5d70c997798aec64]*/ { return heappop_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapreplace_max heap: object(subclass_of='&PyList_Type') @@ -533,12 +557,13 @@ Maxheap variant of heapreplace. static PyObject * _heapq_heapreplace_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=8770778b5a9cbe9b input=21a3d28d757c881c]*/ +/*[clinic end generated code: output=8770778b5a9cbe9b input=fe70175356e4a649]*/ { return heapreplace_internal(heap, item, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapify_max heap: object(subclass_of='&PyList_Type') @@ -549,12 +574,13 @@ Maxheap variant of heapify. static PyObject * _heapq_heapify_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=8401af3856529807 input=edda4255728c431e]*/ +/*[clinic end generated code: output=8401af3856529807 input=4eee63231e7d1573]*/ { return heapify_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heappushpop_max heap: object(subclass_of='&PyList_Type') @@ -569,7 +595,7 @@ a separate call to heappop_max(). static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=525a843013cbd6c0]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=24d0defa6fd6df4a]*/ { PyObject *returnitem; int cmp; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index ea2e5f99dfa..ee5e2b005e0 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -220,6 +220,22 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) return 0; } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._channels"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.channels"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + /* module state *************************************************************/ @@ -2742,15 +2758,9 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channels"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); if (end == CHANNEL_SEND) { cls = state->send_channel_type; } diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 71d8fd8716c..e22709d5119 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -136,13 +136,10 @@ idarg_int64_converter(PyObject *arg, void *ptr) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._queues"); if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queues"); - if (highlevel == NULL) { - return -1; - } + return -1; } Py_DECREF(highlevel); return 0; @@ -299,7 +296,7 @@ add_QueueError(PyObject *mod) { module_state *state = get_module_state(mod); -#define PREFIX "test.support.interpreters." +#define PREFIX "concurrent.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index bbad5eb6903..d0074b2a0d1 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -632,6 +632,27 @@ _lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, } /*[clinic input] +_lsprof.Profiler._pythrow_callback + + code: object + instruction_offset: object + exception: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception) +/*[clinic end generated code: output=0a32988919dfb94c input=fd728fc2c074f5e6]*/ +{ + ptrace_enter_call((PyObject*)self, (void *)code, code); + + Py_RETURN_NONE; +} + +/*[clinic input] _lsprof.Profiler._pyreturn_callback code: object @@ -747,7 +768,7 @@ static const struct { } callback_table[] = { {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, - {PY_MONITORING_EVENT_PY_THROW, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_THROW, "_pythrow_callback"}, {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, @@ -1002,6 +1023,7 @@ static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_DISABLE_METHODDEF _LSPROF_PROFILER_CLEAR_METHODDEF _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF + _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 86d8b38620c..cf3ceb43fb3 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5543,17 +5543,16 @@ static int load_counted_binstring(PickleState *st, UnpicklerObject *self, int nbytes) { PyObject *obj; - Py_ssize_t size; + long size; char *s; if (_Unpickler_Read(self, st, &s, nbytes) < 0) return -1; - size = calc_binsize(s, nbytes); + size = calc_binint(s, nbytes); if (size < 0) { - PyErr_Format(st->UnpicklingError, - "BINSTRING exceeds system's maximum size of %zd bytes", - PY_SSIZE_T_MAX); + PyErr_SetString(st->UnpicklingError, + "BINSTRING pickle has negative byte count"); return -1; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 845c218e679..804cb4e4d1c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1788,9 +1788,9 @@ finally: /* To run some code in a sub-interpreter. -Generally you can use test.support.interpreters, +Generally you can use the interpreters module, but we keep this helper as a distinct implementation. -That's especially important for testing test.support.interpreters. +That's especially important for testing the interpreters module. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 07aa89f573f..2ce8c0cd3d7 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -13,7 +13,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#include "pyconfig.h" #include "Python.h" #include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() @@ -51,96 +50,19 @@ # undef HACL_CAN_COMPILE_SIMD256 #endif -// ECX -#define ECX_SSE3 (1 << 0) -#define ECX_SSSE3 (1 << 9) -#define ECX_SSE4_1 (1 << 19) -#define ECX_SSE4_2 (1 << 20) -#define ECX_AVX (1 << 28) - -// EBX -#define EBX_AVX2 (1 << 5) - -// EDX -#define EDX_SSE (1 << 25) -#define EDX_SSE2 (1 << 26) -#define EDX_CMOV (1 << 15) - -// zero-initialized by default -typedef struct { - bool sse, sse2, sse3, sse41, sse42, cmov, avx, avx2; - bool done; -} cpu_flags; - -void detect_cpu_features(cpu_flags *flags) { - if (!flags->done) { - int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; - int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; -#if defined(__x86_64__) && defined(__GNUC__) - __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); - __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); -#elif defined(_M_X64) - int info1[4] = { 0 }; - int info7[4] = { 0 }; - __cpuidex(info1, 1, 0); - __cpuidex(info7, 7, 0); - eax1 = info1[0]; - ebx1 = info1[1]; - ecx1 = info1[2]; - edx1 = info1[3]; - eax7 = info7[0]; - ebx7 = info7[1]; - ecx7 = info7[2]; - edx7 = info7[3]; -#endif - (void) eax1; (void) ebx1; (void) ecx1; (void) edx1; - (void) eax7; (void) ebx7; (void) ecx7; (void) edx7; - - - flags->avx = (ecx1 & ECX_AVX) != 0; - - flags->avx2 = (ebx7 & EBX_AVX2) != 0; - - flags->sse = (edx1 & EDX_SSE) != 0; - flags->sse2 = (edx1 & EDX_SSE2) != 0; - flags->cmov = (edx1 & EDX_CMOV) != 0; - - flags->sse3 = (ecx1 & ECX_SSE3) != 0; - /* ssse3 = (ecx1 & ECX_SSSE3) != 0; */ - flags->sse41 = (ecx1 & ECX_SSE4_1) != 0; - flags->sse42 = (ecx1 & ECX_SSE4_2) != 0; - - flags->done = true; - } -} - -#ifdef HACL_CAN_COMPILE_SIMD128 -static inline bool has_simd128(cpu_flags *flags) { - // For now this is Intel-only, could conceivably be #ifdef'd to something - // else. - return flags->sse && flags->sse2 && flags->sse3 && flags->sse41 && flags->sse42 && flags->cmov; -} -#endif - -#ifdef HACL_CAN_COMPILE_SIMD256 -static inline bool has_simd256(cpu_flags *flags) { - return flags->avx && flags->avx2; -} -#endif - // Small mismatch between the variable names Python defines as part of configure // at the ones HACL* expects to be set in order to enable those headers. #define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 #define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 -#include "_hacl/Hacl_Hash_Blake2b.h" #include "_hacl/Hacl_Hash_Blake2s.h" -#if HACL_CAN_COMPILE_SIMD256 -#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" -#endif +#include "_hacl/Hacl_Hash_Blake2b.h" #if HACL_CAN_COMPILE_SIMD128 #include "_hacl/Hacl_Hash_Blake2s_Simd128.h" #endif +#if HACL_CAN_COMPILE_SIMD256 +#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" +#endif // MODULE TYPE SLOTS @@ -148,16 +70,16 @@ static PyType_Spec blake2b_type_spec; static PyType_Spec blake2s_type_spec; PyDoc_STRVAR(blake2mod__doc__, -"_blake2b provides BLAKE2b for hashlib\n" -); + "_blake2 provides BLAKE2b and BLAKE2s for hashlib\n"); typedef struct { - PyTypeObject* blake2b_type; - PyTypeObject* blake2s_type; - cpu_flags flags; + PyTypeObject *blake2b_type; + PyTypeObject *blake2s_type; + bool can_run_simd128; + bool can_run_simd256; } Blake2State; -static inline Blake2State* +static inline Blake2State * blake2_get_state(PyObject *module) { void *state = _PyModule_GetState(module); @@ -166,7 +88,7 @@ blake2_get_state(PyObject *module) } #if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) -static inline Blake2State* +static inline Blake2State * blake2_get_state_from_type(PyTypeObject *module) { void *state = _PyType_GetModuleState(module); @@ -203,31 +125,107 @@ _blake2_free(void *module) (void)_blake2_clear((PyObject *)module); } -#define ADD_INT(d, name, value) do { \ - PyObject *x = PyLong_FromLong(value); \ - if (!x) \ - return -1; \ - if (PyDict_SetItemString(d, name, x) < 0) { \ - Py_DECREF(x); \ - return -1; \ - } \ - Py_DECREF(x); \ -} while(0) - -#define ADD_INT_CONST(NAME, VALUE) do { \ - if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ - return -1; \ - } \ -} while (0) +static void +blake2module_init_cpu_features(Blake2State *state) +{ + /* This must be kept in sync with hmacmodule_init_cpu_features() + * in hmacmodule.c */ + int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; + int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; +#if defined(__x86_64__) && defined(__GNUC__) + __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); + __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); +#elif defined(_M_X64) + int info1[4] = {0}; + __cpuidex(info1, 1, 0); + eax1 = info1[0], ebx1 = info1[1], ecx1 = info1[2], edx1 = info1[3]; + + int info7[4] = {0}; + __cpuidex(info7, 7, 0); + eax7 = info7[0], ebx7 = info7[1], ecx7 = info7[2], edx7 = info7[3]; +#endif + // fmt: off + (void)eax1; (void)ebx1; (void)ecx1; (void)edx1; + (void)eax7; (void)ebx7; (void)ecx7; (void)edx7; + // fmt: on + +#define EBX_AVX2 (1 << 5) +#define ECX_SSE3 (1 << 0) +#define ECX_SSSE3 (1 << 9) +#define ECX_SSE4_1 (1 << 19) +#define ECX_SSE4_2 (1 << 20) +#define ECX_AVX (1 << 28) +#define EDX_SSE (1 << 25) +#define EDX_SSE2 (1 << 26) +#define EDX_CMOV (1 << 15) + + bool avx = (ecx1 & ECX_AVX) != 0; + bool avx2 = (ebx7 & EBX_AVX2) != 0; + + bool sse = (edx1 & EDX_SSE) != 0; + bool sse2 = (edx1 & EDX_SSE2) != 0; + bool cmov = (edx1 & EDX_CMOV) != 0; + + bool sse3 = (ecx1 & ECX_SSE3) != 0; + bool sse41 = (ecx1 & ECX_SSE4_1) != 0; + bool sse42 = (ecx1 & ECX_SSE4_2) != 0; + +#undef EDX_CMOV +#undef EDX_SSE2 +#undef EDX_SSE +#undef ECX_AVX +#undef ECX_SSE4_2 +#undef ECX_SSE4_1 +#undef ECX_SSSE3 +#undef ECX_SSE3 +#undef EBX_AVX2 + +#if HACL_CAN_COMPILE_SIMD128 + // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection + state->can_run_simd128 = sse && sse2 && sse3 && sse41 && sse42 && cmov; +#else + // fmt: off + (void)sse; (void)sse2; (void)sse3; (void)sse41; (void)sse42; (void)cmov; + // fmt: on + state->can_run_simd128 = false; +#endif + +#if HACL_CAN_COMPILE_SIMD256 + // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection + state->can_run_simd256 = state->can_run_simd128 && avx && avx2; +#else + // fmt: off + (void)avx; (void)avx2; + // fmt: on + state->can_run_simd256 = false; +#endif +} static int blake2_exec(PyObject *m) { - Blake2State* st = blake2_get_state(m); - - // This is called at module initialization-time, and so appears to be as - // good a place as any to probe the CPU flags. - detect_cpu_features(&st->flags); + Blake2State *st = blake2_get_state(m); + blake2module_init_cpu_features(st); + +#define ADD_INT(DICT, NAME, VALUE) \ + do { \ + PyObject *x = PyLong_FromLong(VALUE); \ + if (x == NULL) { \ + return -1; \ + } \ + int rc = PyDict_SetItemString(DICT, NAME, x); \ + Py_DECREF(x); \ + if (rc < 0) { \ + return -1; \ + } \ + } while(0) + +#define ADD_INT_CONST(NAME, VALUE) \ + do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ + } while (0) ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE); @@ -237,7 +235,6 @@ blake2_exec(PyObject *m) if (st->blake2b_type == NULL) { return -1; } - /* BLAKE2b */ if (PyModule_AddType(m, st->blake2b_type) < 0) { return -1; } @@ -257,9 +254,9 @@ blake2_exec(PyObject *m) st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( m, &blake2s_type_spec, NULL); - if (NULL == st->blake2s_type) + if (st->blake2s_type == NULL) { return -1; - + } if (PyModule_AddType(m, st->blake2s_type) < 0) { return -1; } @@ -275,12 +272,11 @@ blake2_exec(PyObject *m) ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); +#undef ADD_INT_CONST +#undef ADD_INT return 0; } -#undef ADD_INT -#undef ADD_INT_CONST - static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -320,35 +316,39 @@ PyInit__blake2(void) // set. typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl; -static inline bool is_blake2b(blake2_impl impl) { - return impl == Blake2b || impl == Blake2b_256; +static inline bool +is_blake2b(blake2_impl impl) +{ + return impl == Blake2b || impl == Blake2b_256; } -static inline bool is_blake2s(blake2_impl impl) { - return !is_blake2b(impl); +static inline bool +is_blake2s(blake2_impl impl) +{ + return impl == Blake2s || impl == Blake2s_128; } -static inline blake2_impl type_to_impl(PyTypeObject *type) { +static inline blake2_impl +type_to_impl(PyTypeObject *type) +{ #if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) - Blake2State* st = blake2_get_state_from_type(type); + Blake2State *st = blake2_get_state_from_type(type); #endif if (!strcmp(type->tp_name, blake2b_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD256 - if (has_simd256(&st->flags)) - return Blake2b_256; - else -#endif +#if HACL_CAN_COMPILE_SIMD256 + return st->can_run_simd256 ? Blake2b_256 : Blake2b; +#else return Blake2b; - } else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD128 - if (has_simd128(&st->flags)) - return Blake2s_128; - else #endif + } + else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { +#if HACL_CAN_COMPILE_SIMD128 + return st->can_run_simd128 ? Blake2s_128 : Blake2s; +#else return Blake2s; - } else { - Py_UNREACHABLE(); +#endif } + Py_UNREACHABLE(); } typedef struct { @@ -356,10 +356,10 @@ typedef struct { union { Hacl_Hash_Blake2s_state_t *blake2s_state; Hacl_Hash_Blake2b_state_t *blake2b_state; -#ifdef HACL_CAN_COMPILE_SIMD128 +#if HACL_CAN_COMPILE_SIMD128 Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state; #endif -#ifdef HACL_CAN_COMPILE_SIMD256 +#if HACL_CAN_COMPILE_SIMD256 Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state; #endif }; @@ -425,39 +425,124 @@ static void update(Blake2Object *self, uint8_t *buf, Py_ssize_t len) { switch (self->impl) { - // These need to be ifdef'd out otherwise it's an unresolved symbol at - // link-time. -#ifdef HACL_CAN_COMPILE_SIMD256 + // blake2b_256_state and blake2s_128_state must be if'd since + // otherwise this results in an unresolved symbol at link-time. +#if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update,self->blake2b_256_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update, + self->blake2b_256_state, buf, len); return; #endif -#ifdef HACL_CAN_COMPILE_SIMD128 +#if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update,self->blake2s_128_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update, + self->blake2s_128_state, buf, len); return; #endif case Blake2b: - HACL_UPDATE(Hacl_Hash_Blake2b_update,self->blake2b_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2b_update, + self->blake2b_state, buf, len); return; case Blake2s: - HACL_UPDATE(Hacl_Hash_Blake2s_update,self->blake2s_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2s_update, + self->blake2s_state, buf, len); return; default: Py_UNREACHABLE(); } } -static PyObject * -py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) +#define BLAKE2_IMPLNAME(SELF) \ + (is_blake2b((SELF)->impl) ? "blake2b" : "blake2s") +#define GET_BLAKE2_CONST(SELF, NAME) \ + (is_blake2b((SELF)->impl) \ + ? HACL_HASH_BLAKE2B_ ## NAME \ + : HACL_HASH_BLAKE2S_ ## NAME) + +#define MAX_OUT_BYTES(SELF) GET_BLAKE2_CONST(SELF, OUT_BYTES) +#define MAX_SALT_LENGTH(SELF) GET_BLAKE2_CONST(SELF, SALT_BYTES) +#define MAX_KEY_BYTES(SELF) GET_BLAKE2_CONST(SELF, KEY_BYTES) +#define MAX_PERSONAL_BYTES(SELF) GET_BLAKE2_CONST(SELF, PERSONAL_BYTES) +static int +py_blake2_validate_params(Blake2Object *self, + int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size) +{ + /* Validate digest size. */ + if (digest_size <= 0 || (unsigned int)digest_size > MAX_OUT_BYTES(self)) { + PyErr_Format( + PyExc_ValueError, + "digest_size for %s must be between 1 and %d bytes, got %d", + BLAKE2_IMPLNAME(self), MAX_OUT_BYTES(self), digest_size + ); + goto error; + } + +#define CHECK_LENGTH(NAME, VALUE, MAX) \ + do { \ + if ((size_t)(VALUE) > (size_t)(MAX)) { \ + PyErr_Format(PyExc_ValueError, \ + "maximum %s length is %zu bytes, got %zd", \ + (NAME), (size_t)(MAX), (Py_ssize_t)(VALUE)); \ + goto error; \ + } \ + } while (0) + /* Validate key parameter. */ + if (key->obj && key->len) { + CHECK_LENGTH("key", key->len, MAX_KEY_BYTES(self)); + } + /* Validate salt parameter. */ + if (salt->obj && salt->len) { + CHECK_LENGTH("salt", salt->len, MAX_SALT_LENGTH(self)); + } + /* Validate personalization parameter. */ + if (person->obj && person->len) { + CHECK_LENGTH("person", person->len, MAX_PERSONAL_BYTES(self)); + } +#undef CHECK_LENGTH +#define CHECK_TREE(NAME, VALUE, MIN, MAX) \ + do { \ + if ((VALUE) < (MIN) || (size_t)(VALUE) > (size_t)(MAX)) { \ + PyErr_Format(PyExc_ValueError, \ + "'%s' must be between %zu and %zu", \ + (NAME), (size_t)(MIN), (size_t)(MAX)); \ + goto error; \ + } \ + } while (0) + /* Validate tree parameters. */ + CHECK_TREE("fanout", fanout, 0, 255); + CHECK_TREE("depth", depth, 1, 255); + CHECK_TREE("node_depth", node_depth, 0, 255); + CHECK_TREE("inner_size", inner_size, 0, MAX_OUT_BYTES(self)); +#undef CHECK_TREE + if (leaf_size > 0xFFFFFFFFU) { + /* maximum: 2**32 - 1 */ + PyErr_SetString(PyExc_OverflowError, "'leaf_size' is too large"); + goto error; + } + if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { + /* maximum: 2**48 - 1 */ + PyErr_SetString(PyExc_OverflowError, "'node_offset' is too large"); + goto error; + } + return 0; +error: + return -1; +} + + +static PyObject * +py_blake2_new(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) { Blake2Object *self = NULL; - Py_buffer buf; self = new_Blake2Object(type); if (self == NULL) { @@ -487,96 +572,31 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, default: Py_UNREACHABLE(); } - // Using Blake2b because we statically know that these are greater than the - // Blake2s sizes -- this avoids a VLA. - uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; - uint8_t personal_[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = { 0 }; - /* Validate digest size. */ - if (digest_size <= 0 || - (unsigned) digest_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) + // Unlike the state types, the parameters share a single (client-friendly) + // structure. + if (py_blake2_validate_params(self, + digest_size, + key, salt, person, + fanout, depth, leaf_size, + node_offset, node_depth, inner_size) < 0) { - PyErr_Format(PyExc_ValueError, - "digest_size for %s must be between 1 and %d bytes, here it is %d", - is_blake2b(self->impl) ? "Blake2b" : "Blake2s", - is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES, - digest_size); - goto error; - } - - /* Validate salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if ((size_t)salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)); - goto error; - } - memcpy(salt_, salt->buf, salt->len); - } - - /* Validate personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if ((size_t)person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)); - goto error; - } - memcpy(personal_, person->buf, person->len); - } - - /* Validate tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - - if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); goto error; } - if (inner_size < 0 || - (unsigned) inner_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)); - goto error; + // Using Blake2b because we statically know that these are greater than the + // Blake2s sizes -- this avoids a VLA. + uint8_t salt_buffer[HACL_HASH_BLAKE2B_SALT_BYTES] = {0}; + uint8_t personal_buffer[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = {0}; + if (salt->obj != NULL) { + assert(salt->buf != NULL); + memcpy(salt_buffer, salt->buf, salt->len); } - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if ((size_t)key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)); - goto error; - } + if (person->obj != NULL) { + assert(person->buf != NULL); + memcpy(personal_buffer, person->buf, person->len); } - // Unlike the state types, the parameters share a single (client-friendly) - // structure. - Hacl_Hash_Blake2b_blake2_params params = { .digest_length = digest_size, .key_length = (uint8_t)key->len, @@ -586,55 +606,46 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, .node_offset = node_offset, .node_depth = node_depth, .inner_length = inner_size, - .salt = salt_, - .personal = personal_ + .salt = salt_buffer, + .personal = personal_buffer }; +#define BLAKE2_MALLOC(TYPE, STATE) \ + do { \ + STATE = Hacl_Hash_ ## TYPE ## _malloc_with_params_and_key( \ + ¶ms, last_node, key->buf); \ + if (STATE == NULL) { \ + (void)PyErr_NoMemory(); \ + goto error; \ + } \ + } while (0) + switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: { - self->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(¶ms, last_node, key->buf); - if (self->blake2b_256_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2b_256: + BLAKE2_MALLOC(Blake2b_Simd256, self->blake2b_256_state); break; - } #endif #if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: { - self->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(¶ms, last_node, key->buf); - if (self->blake2s_128_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2s_128: + BLAKE2_MALLOC(Blake2s_Simd128, self->blake2s_128_state); break; - } #endif - case Blake2b: { - self->blake2b_state = Hacl_Hash_Blake2b_malloc_with_params_and_key(¶ms, last_node, key->buf); - if (self->blake2b_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2b: + BLAKE2_MALLOC(Blake2b, self->blake2b_state); break; - } - case Blake2s: { - self->blake2s_state = Hacl_Hash_Blake2s_malloc_with_params_and_key(¶ms, last_node, key->buf); - if (self->blake2s_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2s: + BLAKE2_MALLOC(Blake2s, self->blake2s_state); break; - } default: Py_UNREACHABLE(); } +#undef BLAKE2_MALLOC /* Process initial data if any. */ if (data != NULL) { + Py_buffer buf; GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { Py_BEGIN_ALLOW_THREADS update(self, buf.buf, buf.len); @@ -687,7 +698,9 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { return NULL; } - return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return py_blake2_new(type, data, digest_size, key, salt, person, + fanout, depth, leaf_size, node_offset, node_depth, + inner_size, last_node, usedforsecurity); } /*[clinic input] @@ -725,49 +738,44 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { return NULL; } - return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return py_blake2_new(type, data, digest_size, key, salt, person, + fanout, depth, leaf_size, node_offset, node_depth, + inner_size, last_node, usedforsecurity); } static int blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) { assert(cpy != NULL); +#define BLAKE2_COPY(TYPE, STATE_ATTR) \ + do { \ + cpy->STATE_ATTR = Hacl_Hash_ ## TYPE ## _copy(self->STATE_ATTR); \ + if (cpy->STATE_ATTR == NULL) { \ + goto error; \ + } \ + } while (0) + switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: { - cpy->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_copy(self->blake2b_256_state); - if (cpy->blake2b_256_state == NULL) { - goto error; - } + case Blake2b_256: + BLAKE2_COPY(Blake2b_Simd256, blake2b_256_state); break; - } #endif #if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: { - cpy->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_copy(self->blake2s_128_state); - if (cpy->blake2s_128_state == NULL) { - goto error; - } + case Blake2s_128: + BLAKE2_COPY(Blake2s_Simd128, blake2s_128_state); break; - } #endif - case Blake2b: { - cpy->blake2b_state = Hacl_Hash_Blake2b_copy(self->blake2b_state); - if (cpy->blake2b_state == NULL) { - goto error; - } + case Blake2b: + BLAKE2_COPY(Blake2b, blake2b_state); break; - } - case Blake2s: { - cpy->blake2s_state = Hacl_Hash_Blake2s_copy(self->blake2s_state); - if (cpy->blake2s_state == NULL) { - goto error; - } + case Blake2s: + BLAKE2_COPY(Blake2s, blake2s_state); break; - } default: Py_UNREACHABLE(); } +#undef BLAKE2_COPY cpy->impl = self->impl; return 0; @@ -829,7 +837,8 @@ _blake2_blake2b_update_impl(Blake2Object *self, PyObject *data) update(self, buf.buf, buf.len); PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS - } else { + } + else { update(self, buf.buf, buf.len); } @@ -838,40 +847,42 @@ _blake2_blake2b_update_impl(Blake2Object *self, PyObject *data) Py_RETURN_NONE; } -/*[clinic input] -_blake2.blake2b.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_digest_impl(Blake2Object *self) -/*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/ +static uint8_t +blake2_blake2b_compute_digest(Blake2Object *self, uint8_t *digest) { - uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; - - ENTER_HASHLIB(self); - uint8_t digest_length = 0; switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); - break; + return Hacl_Hash_Blake2b_Simd256_digest( + self->blake2b_256_state, digest); #endif #if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); - break; + return Hacl_Hash_Blake2s_Simd128_digest( + self->blake2s_128_state, digest); #endif case Blake2b: - digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); - break; + return Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); case Blake2s: - digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); - break; + return Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); default: Py_UNREACHABLE(); } +} + +/*[clinic input] +_blake2.blake2b.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_digest_impl(Blake2Object *self) +/*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/ +{ + uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + ENTER_HASHLIB(self); + digest_length = blake2_blake2b_compute_digest(self, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, digest_length); } @@ -886,30 +897,9 @@ static PyObject * _blake2_blake2b_hexdigest_impl(Blake2Object *self) /*[clinic end generated code: output=5ef54b138db6610a input=76930f6946351f56]*/ { - uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; - + uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES]; ENTER_HASHLIB(self); - uint8_t digest_length = 0; - switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: - digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); - break; -#endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: - digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); - break; -#endif - case Blake2b: - digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); - break; - case Blake2s: - digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); - break; - default: - Py_UNREACHABLE(); - } + digest_length = blake2_blake2b_compute_digest(self, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, digest_length); } @@ -928,43 +918,49 @@ static PyObject * py_blake2b_get_name(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); - return PyUnicode_FromString(is_blake2b(self->impl) ? "blake2b" : "blake2s"); + return PyUnicode_FromString(BLAKE2_IMPLNAME(self)); } - static PyObject * py_blake2b_get_block_size(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); - return PyLong_FromLong(is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_BLOCK_BYTES : HACL_HASH_BLAKE2S_BLOCK_BYTES); + return PyLong_FromLong(GET_BLAKE2_CONST(self, BLOCK_BYTES)); } - -static PyObject * -py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) +static Hacl_Hash_Blake2b_index +hacl_get_blake2_info(Blake2Object *self) { - Blake2Object *self = _Blake2Object_CAST(op); switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - return PyLong_FromLong(Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state).digest_length); + return Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state); #endif #if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - return PyLong_FromLong(Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state).digest_length); + return Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state); #endif case Blake2b: - return PyLong_FromLong(Hacl_Hash_Blake2b_info(self->blake2b_state).digest_length); + return Hacl_Hash_Blake2b_info(self->blake2b_state); case Blake2s: - return PyLong_FromLong(Hacl_Hash_Blake2s_info(self->blake2s_state).digest_length); + return Hacl_Hash_Blake2s_info(self->blake2s_state); default: Py_UNREACHABLE(); } } +static PyObject * +py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) +{ + Blake2Object *self = _Blake2Object_CAST(op); + Hacl_Hash_Blake2b_index info = hacl_get_blake2_info(self); + return PyLong_FromLong(info.digest_length); +} + + static PyGetSetDef py_blake2b_getsetters[] = { {"name", py_blake2b_get_name, NULL, NULL, NULL}, {"block_size", py_blake2b_get_block_size, NULL, NULL, NULL}, @@ -981,38 +977,35 @@ py_blake2_clear(PyObject *op) // initializes the HACL* internal state to NULL before allocating // it. If an error occurs in the constructor, we should only free // states that were allocated (i.e. that are not NULL). +#define BLAKE2_FREE(TYPE, STATE) \ + do { \ + if (STATE != NULL) { \ + Hacl_Hash_ ## TYPE ## _free(STATE); \ + STATE = NULL; \ + } \ + } while (0) + switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - if (self->blake2b_256_state != NULL) { - Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); - self->blake2b_256_state = NULL; - } + BLAKE2_FREE(Blake2b_Simd256, self->blake2b_256_state); break; #endif #if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - if (self->blake2s_128_state != NULL) { - Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); - self->blake2s_128_state = NULL; - } + BLAKE2_FREE(Blake2s_Simd128, self->blake2s_128_state); break; #endif case Blake2b: - if (self->blake2b_state != NULL) { - Hacl_Hash_Blake2b_free(self->blake2b_state); - self->blake2b_state = NULL; - } + BLAKE2_FREE(Blake2b, self->blake2b_state); break; case Blake2s: - if (self->blake2s_state != NULL) { - Hacl_Hash_Blake2s_free(self->blake2s_state); - self->blake2s_state = NULL; - } + BLAKE2_FREE(Blake2s, self->blake2s_state); break; default: Py_UNREACHABLE(); } +#undef BLAKE2_FREE return 0; } @@ -1041,7 +1034,7 @@ static PyType_Slot blake2b_type_slots[] = { {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, {Py_tp_new, py_blake2b_new}, - {0,0} + {0, 0} }; static PyType_Slot blake2s_type_slots[] = { @@ -1054,12 +1047,12 @@ static PyType_Slot blake2s_type_slots[] = { // only the constructor differs, so that it can receive a clinic-generated // default digest length suitable for blake2s {Py_tp_new, py_blake2s_new}, - {0,0} + {0, 0} }; static PyType_Spec blake2b_type_spec = { .name = "_blake2.blake2b", - .basicsize = sizeof(Blake2Object), + .basicsize = sizeof(Blake2Object), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2b_type_slots @@ -1067,7 +1060,7 @@ static PyType_Spec blake2b_type_spec = { static PyType_Spec blake2s_type_spec = { .name = "_blake2.blake2s", - .basicsize = sizeof(Blake2Object), + .basicsize = sizeof(Blake2Object), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2s_type_slots diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 6f4966825ec..75cf067c8aa 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,10 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_runtime.h" // _Py_SINGLETON() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "bottom($self, /)\n" @@ -14,19 +11,15 @@ PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "Push the panel to the bottom of the stack."); #define _CURSES_PANEL_PANEL_BOTTOM_METHODDEF \ - {"bottom", _PyCFunction_CAST(_curses_panel_panel_bottom), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_bottom__doc__}, + {"bottom", (PyCFunction)_curses_panel_panel_bottom, METH_NOARGS, _curses_panel_panel_bottom__doc__}, static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_bottom(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_bottom(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); - return NULL; - } - return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_hide__doc__, @@ -38,19 +31,15 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "This does not delete the object, it just makes the window on screen invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ - {"hide", _PyCFunction_CAST(_curses_panel_panel_hide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_hide__doc__}, + {"hide", (PyCFunction)_curses_panel_panel_hide, METH_NOARGS, _curses_panel_panel_hide__doc__}, static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_hide_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_hide(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_hide(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); - return NULL; - } - return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_show__doc__, @@ -60,19 +49,15 @@ PyDoc_STRVAR(_curses_panel_panel_show__doc__, "Display the panel (which might have been hidden)."); #define _CURSES_PANEL_PANEL_SHOW_METHODDEF \ - {"show", _PyCFunction_CAST(_curses_panel_panel_show), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_show__doc__}, + {"show", (PyCFunction)_curses_panel_panel_show, METH_NOARGS, _curses_panel_panel_show__doc__}, static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_show_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_show(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_show(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); - return NULL; - } - return _curses_panel_panel_show_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_show_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_top__doc__, @@ -82,19 +67,15 @@ PyDoc_STRVAR(_curses_panel_panel_top__doc__, "Push panel to the top of the stack."); #define _CURSES_PANEL_PANEL_TOP_METHODDEF \ - {"top", _PyCFunction_CAST(_curses_panel_panel_top), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_top__doc__}, + {"top", (PyCFunction)_curses_panel_panel_top, METH_NOARGS, _curses_panel_panel_top__doc__}, static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_top_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_top(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_top(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); - return NULL; - } - return _curses_panel_panel_top_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_top_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_above__doc__, @@ -158,36 +139,19 @@ PyDoc_STRVAR(_curses_panel_panel_move__doc__, "Move the panel to the screen coordinates (y, x)."); #define _CURSES_PANEL_PANEL_MOVE_METHODDEF \ - {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_move__doc__}, + {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_FASTCALL, _curses_panel_panel_move__doc__}, static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x); +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x); static PyObject * -_curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", "", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "move", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; int y; int x; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("move", nargs, 2, 2)) { goto exit; } y = PyLong_AsInt(args[0]); @@ -198,7 +162,7 @@ _curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *arg if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, cls, y, x); + return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, y, x); exit: return return_value; @@ -229,44 +193,24 @@ PyDoc_STRVAR(_curses_panel_panel_replace__doc__, "Change the window associated with the panel to the window win."); #define _CURSES_PANEL_PANEL_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(_curses_panel_panel_replace), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_replace__doc__}, + {"replace", (PyCFunction)_curses_panel_panel_replace, METH_O, _curses_panel_panel_replace__doc__}, static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win); static PyObject * -_curses_panel_panel_replace(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_replace(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "replace", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; PyCursesWindowObject *win; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyObject_TypeCheck(args[0], &PyCursesWindow_Type)) { - _PyArg_BadArgument("replace", "argument 1", (&PyCursesWindow_Type)->tp_name, args[0]); + if (!PyObject_TypeCheck(arg, &PyCursesWindow_Type)) { + _PyArg_BadArgument("replace", "argument", (&PyCursesWindow_Type)->tp_name, arg); goto exit; } - win = (PyCursesWindowObject *)args[0]; - return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, cls, win); + win = (PyCursesWindowObject *)arg; + return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, win); exit: return return_value; @@ -279,41 +223,19 @@ PyDoc_STRVAR(_curses_panel_panel_set_userptr__doc__, "Set the panel\'s user pointer to obj."); #define _CURSES_PANEL_PANEL_SET_USERPTR_METHODDEF \ - {"set_userptr", _PyCFunction_CAST(_curses_panel_panel_set_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_set_userptr__doc__}, + {"set_userptr", (PyCFunction)_curses_panel_panel_set_userptr, METH_O, _curses_panel_panel_set_userptr__doc__}, static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj); + PyObject *obj); static PyObject * -_curses_panel_panel_set_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_set_userptr(PyObject *self, PyObject *obj) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "set_userptr", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *obj; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - obj = args[0]; - return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, cls, obj); -exit: + return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, obj); + return return_value; } @@ -324,20 +246,15 @@ PyDoc_STRVAR(_curses_panel_panel_userptr__doc__, "Return the user pointer for the panel."); #define _CURSES_PANEL_PANEL_USERPTR_METHODDEF \ - {"userptr", _PyCFunction_CAST(_curses_panel_panel_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_userptr__doc__}, + {"userptr", (PyCFunction)_curses_panel_panel_userptr, METH_NOARGS, _curses_panel_panel_userptr__doc__}, static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls); +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_userptr(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); - return NULL; - } - return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_bottom_panel__doc__, @@ -424,4 +341,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=36853ecb4a979814 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db2fe491582784aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 81d10862726..b43155b6c24 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_heapq_heappush__doc__, @@ -32,7 +33,9 @@ _heapq_heappush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -61,7 +64,9 @@ _heapq_heappop(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -103,7 +108,9 @@ _heapq_heapreplace(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -140,7 +147,9 @@ _heapq_heappushpop(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -169,7 +178,9 @@ _heapq_heapify(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -203,7 +214,9 @@ _heapq_heappush_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -232,7 +245,9 @@ _heapq_heappop_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +281,9 @@ _heapq_heapreplace_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -295,7 +312,9 @@ _heapq_heapify_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,9 +351,11 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=f55d8595ce150c76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e83d50002c29a96d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index 2918a6bc7ab..c426cd6fe02 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -82,6 +82,39 @@ exit: return return_value; } +PyDoc_STRVAR(_lsprof_Profiler__pythrow_callback__doc__, +"_pythrow_callback($self, code, instruction_offset, exception, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF \ + {"_pythrow_callback", _PyCFunction_CAST(_lsprof_Profiler__pythrow_callback), METH_FASTCALL, _lsprof_Profiler__pythrow_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception); + +static PyObject * +_lsprof_Profiler__pythrow_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *exception; + + if (!_PyArg_CheckPositional("_pythrow_callback", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + exception = args[2]; + return_value = _lsprof_Profiler__pythrow_callback_impl((ProfilerObject *)self, code, instruction_offset, exception); + +exit: + return return_value; +} + PyDoc_STRVAR(_lsprof_Profiler__pyreturn_callback__doc__, "_pyreturn_callback($self, code, instruction_offset, retval, /)\n" "--\n" @@ -411,4 +444,4 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=fe231309776df7a7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9e46985561166c37 input=a9049054013a1b77]*/ diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index c7b49d4dee3..b404d5732ec 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -1715,11 +1715,11 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); #elif defined(_M_X64) - int info1[4] = { 0 }; + int info1[4] = {0}; __cpuidex(info1, 1, 0); eax1 = info1[0], ebx1 = info1[1], ecx1 = info1[2], edx1 = info1[3]; - int info7[4] = { 0 }; + int info7[4] = {0}; __cpuidex(info7, 7, 0); eax7 = info7[0], ebx7 = info7[1], ecx7 = info7[2], edx7 = info7[3]; #endif diff --git a/Modules/md5module.c b/Modules/md5module.c index 9b5ea2d6e02..08dbcd2cbce 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -120,7 +120,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) newobj->hash_state = Hacl_Hash_MD5_copy(self->hash_state); LEAVE_HASHLIB(self); if (newobj->hash_state == NULL) { - Py_DECREF(self); + Py_DECREF(newobj); return PyErr_NoMemory(); } return (PyObject *)newobj; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 92c9aa8b510..85c72779bac 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -716,12 +716,6 @@ select_error(void) # define SOCK_INPROGRESS_ERR EINPROGRESS #endif -#ifdef _MSC_VER -# define SUPPRESS_DEPRECATED_CALL __pragma(warning(suppress: 4996)) -#else -# define SUPPRESS_DEPRECATED_CALL -#endif - /* Convenience function to raise an error according to errno and return a NULL pointer from a function. */ @@ -3366,7 +3360,7 @@ sock_setsockopt(PyObject *self, PyObject *args) &level, &optname, &flag)) { #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { - int dummy; + DWORD dummy; res = WSAIoctl(get_sock_fd(s), SIO_TCP_SET_ACK_FREQUENCY, &flag, sizeof(flag), NULL, 0, &dummy, NULL, NULL); if (res >= 0) { @@ -6195,8 +6189,10 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyname(name); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS /* Some C libraries would require addr.__ss_family instead of @@ -6300,8 +6296,10 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyaddr(ap, al, af); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS ret = gethost_common(state, h, SAS2SA(&addr), sizeof(addr), af); @@ -6718,8 +6716,10 @@ _socket_inet_aton_impl(PyObject *module, const char *ip_addr) packed_addr = INADDR_BROADCAST; } else { - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS packed_addr = inet_addr(ip_addr); + _Py_COMP_DIAG_POP if (packed_addr == INADDR_NONE) { /* invalid address */ PyErr_SetString(PyExc_OSError, @@ -6762,8 +6762,10 @@ _socket_inet_ntoa_impl(PyObject *module, Py_buffer *packed_ip) memcpy(&packed_addr, packed_ip->buf, packed_ip->len); PyBuffer_Release(packed_ip); - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS return PyUnicode_FromString(inet_ntoa(packed_addr)); + _Py_COMP_DIAG_POP } #endif // HAVE_INET_NTOA diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index ce51e342241..cf35e705f35 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -100,8 +100,8 @@ <AdditionalDependencies>advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF %(AdditionalOptions)</AdditionalOptions> <AdditionalOptions Condition="$(MSVCHasBrokenARM64Clamping) == 'true' and $(Platform) == 'ARM64'">-d2:-pattern-opt-disable:-932189325 %(AdditionalOptions)</AdditionalOptions> - <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">/GENPROFILE %(AdditionalOptions)</AdditionalOptions> - <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">/USEPROFILE %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument' and $(PlatformToolset) != 'ClangCL'">/GENPROFILE %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate' and $(PlatformToolset) != 'ClangCL'">/USEPROFILE %(AdditionalOptions)</AdditionalOptions> </Link> <Lib> <LinkTimeCodeGeneration>false</LinkTimeCodeGeneration> diff --git a/Parser/parser.c b/Parser/parser.c index 82311b4f40e..ee0aeb4e187 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -1679,7 +1679,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); stmt_ty import_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_5_rule, p) + _PyPegen_lookahead(1, _tmp_5_rule, p) && (import_stmt_var = import_stmt_rule(p)) // import_stmt ) @@ -1917,7 +1917,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('def' | '@' | 'async') function_def")); stmt_ty function_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) + _PyPegen_lookahead(1, _tmp_6_rule, p) && (function_def_var = function_def_rule(p)) // function_def ) @@ -1959,7 +1959,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('class' | '@') class_def")); stmt_ty class_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) + _PyPegen_lookahead(1, _tmp_7_rule, p) && (class_def_var = class_def_rule(p)) // class_def ) @@ -1980,7 +1980,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('with' | 'async') with_stmt")); stmt_ty with_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) + _PyPegen_lookahead(1, _tmp_8_rule, p) && (with_stmt_var = with_stmt_rule(p)) // with_stmt ) @@ -2001,7 +2001,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('for' | 'async') for_stmt")); stmt_ty for_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) + _PyPegen_lookahead(1, _tmp_9_rule, p) && (for_stmt_var = for_stmt_rule(p)) // for_stmt ) @@ -3234,7 +3234,7 @@ del_stmt_rule(Parser *p) && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_16_rule, p) + _PyPegen_lookahead(1, _tmp_16_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -6877,7 +6877,7 @@ with_item_rule(Parser *p) && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); @@ -8466,7 +8466,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8700,7 +8700,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8738,7 +8738,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_43_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -9302,7 +9302,7 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_44_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); @@ -9417,7 +9417,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_44_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -15070,7 +15070,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_43_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -19099,7 +19099,7 @@ target_with_star_atom_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19143,7 +19143,7 @@ target_with_star_atom_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19490,7 +19490,7 @@ single_subscript_attribute_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19534,7 +19534,7 @@ single_subscript_attribute_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19644,7 +19644,7 @@ t_primary_raw(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME &t_lookahead")); @@ -19688,7 +19688,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' &t_lookahead")); @@ -19726,7 +19726,7 @@ t_primary_raw(Parser *p) && (b = genexp_rule(p)) // genexp && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary genexp &t_lookahead")); @@ -19770,7 +19770,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '(' arguments? ')' &t_lookahead")); @@ -19805,7 +19805,7 @@ t_primary_raw(Parser *p) if ( (a = atom_rule(p)) // atom && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "atom &t_lookahead")); @@ -19995,7 +19995,7 @@ del_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -20039,7 +20039,7 @@ del_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -20527,7 +20527,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_104_rule, p) + _PyPegen_lookahead(1, _tmp_104_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -20721,7 +20721,7 @@ invalid_arguments_rule(Parser *p) && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_110_rule, p) + _PyPegen_lookahead(1, _tmp_110_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20919,7 +20919,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_112_rule, p) + _PyPegen_lookahead(0, _tmp_112_rule, p) && (a = expression_rule(p)) // expression && @@ -21294,7 +21294,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_114_rule, p) + _PyPegen_lookahead(0, _tmp_114_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -21330,7 +21330,7 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_115_rule, p) + _PyPegen_lookahead(0, _tmp_115_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -21365,7 +21365,7 @@ invalid_expression_rule(Parser *p) && (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) expression_rule, p) + _PyPegen_lookahead_for_expr(0, expression_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction 'else' !expression")); @@ -21555,7 +21555,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_117_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -21581,7 +21581,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_118_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -21589,7 +21589,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_117_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -22499,7 +22499,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_125_rule, p) + _PyPegen_lookahead(1, _tmp_125_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -23448,7 +23448,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -23760,7 +23760,7 @@ invalid_dotted_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_138_rule, p) && (a = expression_rule(p)) // expression ) @@ -23811,7 +23811,7 @@ invalid_import_from_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_138_rule, p) && (a = expression_rule(p)) // expression ) @@ -24181,7 +24181,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_143_rule, p) + _PyPegen_lookahead(0, _tmp_143_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -25973,7 +25973,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_148_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -26083,7 +26083,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_148_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -26342,7 +26342,7 @@ invalid_fstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); @@ -26371,7 +26371,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26403,7 +26403,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26469,7 +26469,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_152_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -26594,7 +26594,7 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_152_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26620,7 +26620,7 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); @@ -26784,7 +26784,7 @@ invalid_tstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); @@ -26813,7 +26813,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26845,7 +26845,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26911,7 +26911,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_152_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -27036,7 +27036,7 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_152_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -27062,7 +27062,7 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); @@ -37501,7 +37501,7 @@ _tmp_168_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); expr_ty expression_without_invalid_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _PyPegen_string_token, p) + _PyPegen_lookahead(0, _PyPegen_string_token, p) && (expression_without_invalid_var = expression_without_invalid_rule(p)) // expression_without_invalid ) diff --git a/Parser/pegen.c b/Parser/pegen.c index 81aad470101..50641de27d3 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -379,44 +379,34 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) return 0; } -int -_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} - -int -_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} - -int -_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} - -// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED to disable sanitizer checks on -// undefined behavior (UBsan) in this function, rather than changing 'func' -// callback API. -int _Py_NO_SANITIZE_UNDEFINED -_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} +#define LOOKAHEAD1(NAME, RES_TYPE) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *), Parser *p) \ + { \ + int mark = p->mark; \ + void *res = func(p); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } + +LOOKAHEAD1(_PyPegen_lookahead, void *) +LOOKAHEAD1(_PyPegen_lookahead_for_expr, expr_ty) +LOOKAHEAD1(_PyPegen_lookahead_for_stmt, stmt_ty) +#undef LOOKAHEAD1 + +#define LOOKAHEAD2(NAME, RES_TYPE, T) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *, T), Parser *p, T arg) \ + { \ + int mark = p->mark; \ + void *res = func(p, arg); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } + +LOOKAHEAD2(_PyPegen_lookahead_with_int, Token *, int) +LOOKAHEAD2(_PyPegen_lookahead_with_string, expr_ty, const char *) +#undef LOOKAHEAD2 Token * _PyPegen_expect_token(Parser *p, int type) @@ -620,7 +610,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p) { Py_ssize_t size; PyBytes_AsStringAndSize(t->bytes, &the_token, &size); for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) { - if (strncmp(*keyword, the_token, (size_t)size) == 0) { + if (strlen(*keyword) == (size_t)size && + strncmp(*keyword, the_token, (size_t)size) == 0) { return _PyPegen_name_from_token(p, t); } } diff --git a/Parser/pegen.h b/Parser/pegen.h index 1862fd7407e..804f931871a 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -145,10 +145,11 @@ int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); int _PyPegen_is_memoized(Parser *p, int type, void *pres); -int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *); -int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); -int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*); int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); +int _PyPegen_lookahead_for_expr(int, expr_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_for_stmt(int, stmt_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); +int _PyPegen_lookahead_with_string(int, expr_ty (func)(Parser *, const char*), Parser *, const char*); Token *_PyPegen_expect_token(Parser *p, int type); void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c4b13da5db4..032e76f72af 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -985,12 +985,13 @@ dummy_func( STAT_INC(BINARY_OP, hit); } - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) { - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; INPUTS_DEAD(); frame->return_offset = INSTRUCTION_SIZE; + new_frame = PyStackRef_Wrap(pushed_frame); } macro(BINARY_OP_SUBSCR_GETITEM) = @@ -1296,20 +1297,21 @@ dummy_func( macro(SEND) = _SPECIALIZE_SEND + _SEND; - op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame: _PyInterpreterFrame *)) { + op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - gen_frame->previous = frame; + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); } macro(SEND_GEN) = @@ -2463,7 +2465,7 @@ dummy_func( _LOAD_ATTR_CLASS + _PUSH_NULL_CONDITIONAL; - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -2473,9 +2475,10 @@ dummy_func( DEOPT_IF(code->co_argcount != 1); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; DEAD(owner); + new_frame = PyStackRef_Wrap(pushed_frame); } macro(LOAD_ATTR_PROPERTY) = @@ -3344,7 +3347,7 @@ dummy_func( _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; - op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame: _PyInterpreterFrame*)) { + op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); #ifdef Py_GIL_DISABLED @@ -3356,14 +3359,15 @@ dummy_func( #endif DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; + pushed_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); } macro(FOR_ITER_GEN) = @@ -3715,7 +3719,7 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: @@ -3737,7 +3741,7 @@ dummy_func( if (temp == NULL) { ERROR_NO_POP(); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { @@ -3874,27 +3878,26 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } INPUTS_DEAD(); + new_frame = PyStackRef_Wrap(pushed_frame); } - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- )) { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. + op(_PUSH_FRAME, (new_frame -- )) { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); DEAD(new_frame); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4046,7 +4049,7 @@ dummy_func( PyStackRef_CLOSE(temp); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _PyInterpreterFrame *)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); @@ -4063,12 +4066,12 @@ dummy_func( _PyEval_FrameClearAndPop(tstate, shim); ERROR_NO_POP(); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); } macro(CALL_ALLOC_AND_ENTER_INIT) = @@ -4594,7 +4597,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: @@ -4621,7 +4624,7 @@ dummy_func( DEAD(callable); SYNC_SP(); ERROR_IF(temp == NULL); - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index ca4ca1cf123..12cd61db1b6 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -24,7 +24,7 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterError", + .tp_name = "concurrent.interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -37,7 +37,7 @@ PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterNotFoundError", + .tp_name = "concurrent.interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -51,7 +51,7 @@ PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFou static int _init_notshareableerror(exceptions_t *state) { - const char *name = "interpreters.NotShareableError"; + const char *name = "concurrent.interpreters.NotShareableError"; PyObject *base = PyExc_TypeError; PyObject *ns = NULL; PyObject *exctype = PyErr_NewException(name, base, ns); diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index cc5047d6bda..975c28eec10 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -71,7 +71,16 @@ EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { // ) function getPyEMCountArgsPtr() { - let isIOS = globalThis.navigator && /iPad|iPhone|iPod/.test(navigator.platform); + // Starting with iOS 18.3.1, WebKit on iOS has an issue with the garbage + // collector that breaks the call trampoline. See #130418 and + // https://bugs.webkit.org/show_bug.cgi?id=293113 for details. + let isIOS = globalThis.navigator && ( + /iPad|iPhone|iPod/.test(navigator.userAgent) || + // Starting with iPadOS 13, iPads might send a platform string that looks like a desktop Mac. + // To differentiate, we check if the platform is 'MacIntel' (common for Macs and newer iPads) + // AND if the device has multi-touch capabilities (navigator.maxTouchPoints > 1) + (navigator.platform === 'MacIntel' && typeof navigator.maxTouchPoints !== 'undefined' && navigator.maxTouchPoints > 1) + ) if (isIOS) { return 0; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d19605169d5..4f772f916d1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1551,15 +1551,16 @@ _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; getitem = stack_pointer[-1]; sub = stack_pointer[-2]; container = stack_pointer[-3]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; frame->return_offset = 6 ; - stack_pointer[-3].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-3] = new_frame; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1907,7 +1908,7 @@ case _SEND_GEN_FRAME: { _PyStackRef v; _PyStackRef receiver; - _PyInterpreterFrame *gen_frame; + _PyStackRef gen_frame; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; receiver = stack_pointer[-2]; @@ -1921,15 +1922,16 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2 + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; - stack_pointer[-1].bits = (uintptr_t)gen_frame; + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-1] = gen_frame; break; } @@ -3471,7 +3473,7 @@ case _LOAD_ATTR_PROPERTY_FRAME: { _PyStackRef owner; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *fget = (PyObject *)CURRENT_OPERAND0(); @@ -3496,9 +3498,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - stack_pointer[-1].bits = (uintptr_t)new_frame; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-1] = new_frame; break; } @@ -4467,7 +4470,7 @@ case _FOR_ITER_GEN_FRAME: { _PyStackRef iter; - _PyInterpreterFrame *gen_frame; + _PyStackRef gen_frame; oparg = CURRENT_OPARG(); iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); @@ -4487,14 +4490,15 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; + pushed_frame->previous = frame; frame->return_offset = (uint16_t)( 2 + oparg); - stack_pointer[0].bits = (uintptr_t)gen_frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[0] = gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -4775,7 +4779,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -4800,8 +4804,8 @@ if (temp == NULL) { JUMP_TO_ERROR(); } - new_frame = temp; - stack_pointer[0].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = new_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -5067,7 +5071,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5075,13 +5079,14 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -5091,7 +5096,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5099,13 +5104,14 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -5115,7 +5121,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5123,13 +5129,14 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -5139,7 +5146,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5147,13 +5154,14 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -5163,7 +5171,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5171,13 +5179,14 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -5187,34 +5196,35 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits; + _PyStackRef new_frame; + new_frame = stack_pointer[-1]; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -5429,7 +5439,7 @@ _PyStackRef *args; _PyStackRef self; _PyStackRef init; - _PyInterpreterFrame *init_frame; + _PyStackRef init_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; @@ -5453,10 +5463,10 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; - stack_pointer[0].bits = (uintptr_t)init_frame; + init_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = init_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -6309,7 +6319,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; @@ -6343,8 +6353,8 @@ if (temp == NULL) { JUMP_TO_ERROR(); } - new_frame = temp; - stack_pointer[0].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = new_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c8825df3ade..5ac519bb1b6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -604,7 +604,7 @@ _PyStackRef container; _PyStackRef getitem; _PyStackRef sub; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 5 cache entries */ // _CHECK_PEP_523 { @@ -650,19 +650,20 @@ // _BINARY_OP_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; frame->return_offset = 6 ; + new_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -1708,8 +1709,8 @@ _PyStackRef init; _PyStackRef self; _PyStackRef *args; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef init_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1792,17 +1793,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); } // _PUSH_FRAME { new_frame = init_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -1828,7 +1829,7 @@ _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1921,12 +1922,13 @@ args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -1940,11 +1942,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -1970,7 +1972,7 @@ _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -2056,7 +2058,7 @@ if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -2070,9 +2072,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -3040,7 +3042,7 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3127,7 +3129,7 @@ if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -3141,9 +3143,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -3304,7 +3306,7 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3364,7 +3366,7 @@ if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -3378,9 +3380,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4163,7 +4165,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4227,12 +4229,13 @@ args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -4246,11 +4249,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4275,7 +4278,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4334,7 +4337,7 @@ if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -4348,9 +4351,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -5785,8 +5788,8 @@ INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef gen_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -5818,21 +5821,22 @@ JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; + pushed_frame->previous = frame; frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -8650,7 +8654,7 @@ INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -8701,8 +8705,9 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -8716,11 +8721,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -10661,8 +10666,8 @@ static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef gen_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -10688,24 +10693,25 @@ JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2 + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b4220e2c627..babd3e46b8d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -373,7 +373,7 @@ dummy_func(void) { GETLOCAL(this_instr->operand0) = res; } - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { new_frame = NULL; ctx->done = true; } @@ -467,6 +467,15 @@ dummy_func(void) { res = sym_new_truthiness(ctx, value, false); } + op(_UNARY_INVERT, (value -- res)) { + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_not_null(ctx); + } + } + op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); @@ -688,7 +697,7 @@ dummy_func(void) { self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; new_frame = NULL; ctx->done = true; @@ -726,7 +735,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; PyCodeObject *co = NULL; @@ -747,10 +756,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); - + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); } } @@ -760,7 +768,7 @@ dummy_func(void) { self_or_null = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -769,10 +777,10 @@ dummy_func(void) { break; } - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { new_frame = NULL; ctx->done = true; } @@ -784,7 +792,7 @@ dummy_func(void) { self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { init_frame = NULL; ctx->done = true; } @@ -851,13 +859,13 @@ dummy_func(void) { } } - op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { + op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { gen_frame = NULL; /* We are about to hit the end of the trace */ ctx->done = true; } - op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame: _Py_UOpsAbstractFrame *)) { + op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { gen_frame = NULL; // We are about to hit the end of the trace: ctx->done = true; @@ -875,12 +883,12 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) { + op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -937,8 +945,11 @@ dummy_func(void) { } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -1234,6 +1245,20 @@ dummy_func(void) { sym_set_const(callable, list_append); } + op(_BINARY_SLICE, (container, start, stop -- res)) { + // Slicing a string/list/tuple always returns the same type. + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 960c6838004..adab110c5ce 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -285,8 +285,15 @@ } case _UNARY_INVERT: { + JitOptSymbol *value; JitOptSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } @@ -568,8 +575,19 @@ } case _BINARY_SLICE: { + JitOptSymbol *container; JitOptSymbol *res; - res = sym_new_not_null(ctx); + container = stack_pointer[-3]; + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -697,10 +715,10 @@ } case _BINARY_OP_SUBSCR_INIT_CALL: { - _Py_UOpsAbstractFrame *new_frame; + JitOptSymbol *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3] = (JitOptSymbol *)new_frame; + stack_pointer[-3] = new_frame; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -811,10 +829,10 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - _Py_UOpsAbstractFrame *gen_frame; + JitOptSymbol *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)gen_frame; + stack_pointer[-1] = gen_frame; break; } @@ -1305,12 +1323,12 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UOpsAbstractFrame *new_frame; + JitOptSymbol *new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; new_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)new_frame; + stack_pointer[-1] = new_frame; break; } @@ -1667,10 +1685,10 @@ } case _FOR_ITER_GEN_FRAME: { - _Py_UOpsAbstractFrame *gen_frame; + JitOptSymbol *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[0] = (JitOptSymbol *)gen_frame; + stack_pointer[0] = gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1839,7 +1857,7 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UOpsAbstractFrame *new_frame; + JitOptSymbol *new_frame; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1847,8 +1865,8 @@ ctx->done = true; break; } - new_frame = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1952,7 +1970,7 @@ case _INIT_CALL_PY_EXACT_ARGS: { JitOptSymbol **args; JitOptSymbol *self_or_null; - _Py_UOpsAbstractFrame *new_frame; + JitOptSymbol *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; @@ -1970,25 +1988,25 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + JitOptSymbol *new_frame; + new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); if (co == NULL) { ctx->done = true; @@ -2056,8 +2074,11 @@ JitOptSymbol *arg; JitOptSymbol *res; arg = stack_pointer[-1]; - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -2138,10 +2159,10 @@ } case _CREATE_INIT_FRAME: { - _Py_UOpsAbstractFrame *init_frame; + JitOptSymbol *init_frame; init_frame = NULL; ctx->done = true; - stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; + stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2305,10 +2326,10 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UOpsAbstractFrame *new_frame; + JitOptSymbol *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 6cbf1c8deaa..8f9b6cd4c49 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -13,6 +13,16 @@ If you need to add a new function ensure that is declared 'static'. extern "C" { #endif +#ifdef __clang__ + #define UNUSED __attribute__((unused)) +#elif defined(__GNUC__) + #define UNUSED __attribute__((unused)) +#elif defined(_MSC_VER) + #define UNUSED __pragma(warning(suppress: 4505)) +#else + #define UNUSED +#endif + #if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_MODULE define" #endif @@ -116,6 +126,8 @@ typedef struct { mach_port_t task; #elif defined(MS_WINDOWS) HANDLE hProcess; +#elif defined(__linux__) + int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; Py_ssize_t page_size; @@ -131,7 +143,7 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) } } -void +UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { for (int i = 0; i < MAX_PAGES; i++) { @@ -162,6 +174,8 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } +#elif defined(__linux__) + handle->memfd = -1; #endif handle->page_size = get_page_size(); for (int i = 0; i < MAX_PAGES; i++) { @@ -179,6 +193,11 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { CloseHandle(handle->hProcess); handle->hProcess = NULL; } +#elif defined(__linux__) + if (handle->memfd != -1) { + close(handle->memfd); + handle->memfd = -1; + } #endif handle->pid = 0; _Py_RemoteDebug_FreePageCache(handle); @@ -665,8 +684,6 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c } uintptr_t retval = 0; - int lines_processed = 0; - int matches_found = 0; while (fgets(line + linelen, linesz - linelen, maps_file) != NULL) { linelen = strlen(line); @@ -691,7 +708,6 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c line[linelen - 1] = '\0'; // and prepare to read the next line into the start of the buffer. linelen = 0; - lines_processed++; unsigned long start = 0; unsigned long path_pos = 0; @@ -712,7 +728,6 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c } if (strstr(filename, substr)) { - matches_found++; retval = search_elf_file_for_section(handle, secname, start, path); if (retval) { break; @@ -831,15 +846,10 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const MODULEENTRY32W moduleEntry; moduleEntry.dwSize = sizeof(moduleEntry); void* runtime_addr = NULL; - int modules_examined = 0; - int matches_found = 0; for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) { - modules_examined++; - // Look for either python executable or DLL if (wcsstr(moduleEntry.szModule, substr)) { - matches_found++; runtime_addr = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); if (runtime_addr != NULL) { break; @@ -907,6 +917,61 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) return address; } +#if defined(__linux__) && HAVE_PROCESS_VM_READV + +static int +open_proc_mem_fd(proc_handle_t *handle) +{ + char mem_file_path[64]; + sprintf(mem_file_path, "/proc/%d/mem", handle->pid); + + handle->memfd = open(mem_file_path, O_RDWR); + if (handle->memfd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "failed to open file %s: %s", mem_file_path, strerror(errno)); + return -1; + } + return 0; +} + +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +static int +read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t read_bytes = 0; + + do { + local[0].iov_base = (char*)dst + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + read_bytes = preadv(handle->memfd, local, 1, offset); + if (read_bytes < 0) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "preadv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += read_bytes; + } while ((size_t)read_bytes != local[0].iov_len); + return 0; +} + +#endif // __linux__ + // Platform-independent memory read function static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) @@ -928,6 +993,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -941,6 +1009,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { + if (errno == ENOSYS) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "process_vm_readv failed for PID %d at address 0x%lx " @@ -989,7 +1060,7 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #endif } -int +UNUSED static int _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, uintptr_t addr, size_t size, diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index dd55b7812d4..7aee87ef05a 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -24,6 +24,39 @@ read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* ds return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +#if defined(__linux__) && HAVE_PROCESS_VM_READV +static int +write_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (char*)src + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + written = pwritev(handle->memfd, local, 1, offset); + if (written < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +} +#endif // __linux__ + static int write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { @@ -39,6 +72,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return write_memory_fallback(handle, remote_address, len, src); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -52,6 +88,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); if (written < 0) { + if (errno == ENOSYS) { + return write_memory_fallback(handle, remote_address, len, src); + } PyErr_SetFromErrno(PyExc_OSError); return -1; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 1447f365336..fca9b29f9eb 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -135,15 +135,13 @@ class Flush: @dataclass class StackItem: name: str - type: str | None size: str peek: bool = False used: bool = False def __str__(self) -> str: size = f"[{self.size}]" if self.size else "" - type = "" if self.type is None else f"{self.type} " - return f"{type}{self.name}{size} {self.peek}" + return f"{self.name}{size} {self.peek}" def is_array(self) -> bool: return self.size != "" @@ -345,7 +343,7 @@ def override_error( def convert_stack_item( item: parser.StackEffect, replace_op_arg_1: str | None ) -> StackItem: - return StackItem(item.name, item.type, item.size) + return StackItem(item.name, item.size) def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None: "Unused items cannot be on the stack above used, non-peek items" @@ -683,6 +681,8 @@ NON_ESCAPING_FUNCTIONS = ( "PyStackRef_IsNullOrInt", "PyStackRef_IsError", "PyStackRef_IsValid", + "PyStackRef_Wrap", + "PyStackRef_Unwrap", ) @@ -811,7 +811,7 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: if len(stack_inputs) == 0: return False return all( - (s.name == other.name and s.type == other.type and s.size == other.size) + (s.name == other.name and s.size == other.size) for s, other in zip(stack_inputs, instr.outputs) ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 02f9a952754..47de205c0e9 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -56,9 +56,7 @@ def root_relative_path(filename: str) -> str: def type_and_null(var: StackItem) -> tuple[str, str]: - if var.type: - return var.type, "NULL" - elif var.is_array(): + if var.is_array(): return "_PyStackRef *", "NULL" else: return "_PyStackRef", "PyStackRef_NULL" diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index fda022a44e5..75805dbd7f3 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -73,8 +73,6 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): return "JitOptSymbol **" - if var.type: - return var.type return "JitOptSymbol *" @@ -230,7 +228,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") + stack = Stack() write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 9c9b0053a59..a6dac481875 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -247,12 +247,11 @@ class SimpleStmt(Stmt): @dataclass class StackEffect(Node): name: str = field(compare=False) # __eq__ only uses type, cond, size - type: str = "" # Optional `:type` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond def __repr__(self) -> str: - items = [self.name, self.type, self.size] + items = [self.name, self.size] while items and items[-1] == "": del items[-1] return f"StackEffect({', '.join(repr(item) for item in items)})" @@ -463,20 +462,13 @@ class Parser(PLexer): # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): - type_text = "" - if self.expect(lx.COLON): - type_text = self.require(lx.IDENTIFIER).text.strip() - if self.expect(lx.TIMES): - type_text += " *" size_text = "" if self.expect(lx.LBRACKET): - if type_text: - raise self.make_syntax_error("Unexpected [") if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) size_text = size.text.strip() - return StackEffect(tkn.text, type_text, size_text) + return StackEffect(tkn.text, size_text) return None @contextual diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index df168afa888..3a0e7e5d0d5 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -168,7 +168,7 @@ class Local: @staticmethod def register(name: str) -> "Local": - item = StackItem(name, None, "", False, True) + item = StackItem(name, "", False, True) return Local(item, None, True) def kill(self) -> None: @@ -216,13 +216,11 @@ def array_or_scalar(var: StackItem | Local) -> str: return "array" if var.is_array() else "scalar" class Stack: - def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> None: + def __init__(self) -> None: self.base_offset = PointerOffset.zero() self.physical_sp = PointerOffset.zero() self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] - self.extract_bits = extract_bits - self.cast_type = cast_type def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) @@ -268,10 +266,8 @@ class Stack: self.base_offset = self.logical_sp if var.name in UNUSED or not var.used: return Local.unused(var, self.base_offset) - cast = f"({var.type})" if (not indirect and var.type) else "" - bits = ".bits" if cast and self.extract_bits else "" c_offset = (self.base_offset - self.physical_sp).to_c() - assign = f"{var.name} = {cast}{indirect}stack_pointer[{c_offset}]{bits};\n" + assign = f"{var.name} = {indirect}stack_pointer[{c_offset}];\n" out.emit(assign) self._print(out) return Local.from_memory(var, self.base_offset) @@ -292,12 +288,8 @@ class Stack: out: CWriter, var: StackItem, stack_offset: PointerOffset, - cast_type: str, - extract_bits: bool, ) -> None: - cast = f"({cast_type})" if var.type else "" - bits = ".bits" if cast and extract_bits else "" - out.emit(f"stack_pointer[{stack_offset.to_c()}]{bits} = {cast}{var.name};\n") + out.emit(f"stack_pointer[{stack_offset.to_c()}] = {var.name};\n") def _save_physical_sp(self, out: CWriter) -> None: if self.physical_sp != self.logical_sp: @@ -320,7 +312,7 @@ class Stack: self._print(out) var.memory_offset = var_offset stack_offset = var_offset - self.physical_sp - Stack._do_emit(out, var.item, stack_offset, self.cast_type, self.extract_bits) + Stack._do_emit(out, var.item, stack_offset) self._print(out) var_offset = var_offset.push(var.item) @@ -350,7 +342,7 @@ class Stack: out.emit(self.as_comment() + "\n") def copy(self) -> "Stack": - other = Stack(self.extract_bits, self.cast_type) + other = Stack() other.base_offset = self.base_offset other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index b407a8a643b..02af1caff7d 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -140,6 +140,9 @@ if __name__ == '__main__': data = locale.locale_alias.copy() data.update(parse_glibc_supported(args.glibc_supported)) data.update(parse(args.locale_alias)) + # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist + # on all platforms. + data['c.utf8'] = 'C.UTF-8' while True: # Repeat optimization while the size is decreased. n = len(data) diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 09c5651f24a..04f66eec1a0 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -214,33 +214,47 @@ class CCallMakerVisitor(GrammarVisitor): call.assigned_variable_type = node.type return call + def assert_no_undefined_behavior( + self, call: FunctionCall, wrapper: str, expected_rtype: str | None, + ) -> None: + if call.return_type != expected_rtype: + raise RuntimeError( + f"{call.function} return type is incompatible with {wrapper}: " + f"expect: {expected_rtype}, actual: {call.return_type}" + ) + def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: call = self.generate_call(node.node) - if call.nodetype == NodeTypes.NAME_TOKEN: - return FunctionCall( - function=f"_PyPegen_lookahead_with_name", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + comment = None + if call.nodetype is NodeTypes.NAME_TOKEN: + function = "_PyPegen_lookahead_for_expr" + self.assert_no_undefined_behavior(call, function, "expr_ty") + elif call.nodetype is NodeTypes.STRING_TOKEN: + # _PyPegen_string_token() returns 'void *' instead of 'Token *'; + # in addition, the overall function call would return 'expr_ty'. + assert call.function == "_PyPegen_string_token" + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype == NodeTypes.SOFT_KEYWORD: - return FunctionCall( - function=f"_PyPegen_lookahead_with_string", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead_with_string" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: - return FunctionCall( - function=f"_PyPegen_lookahead_with_int", - arguments=[positive, call.function, *call.arguments], - return_type="int", - comment=f"token={node.node}", - ) + function = "_PyPegen_lookahead_with_int" + self.assert_no_undefined_behavior(call, function, "Token *") + comment = f"token={node.node}" + elif call.return_type == "expr_ty": + function = "_PyPegen_lookahead_for_expr" + elif call.return_type == "stmt_ty": + function = "_PyPegen_lookahead_for_stmt" else: - return FunctionCall( - function=f"_PyPegen_lookahead", - arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, None) + return FunctionCall( + function=function, + arguments=[positive, call.function, *call.arguments], + return_type="int", + comment=comment, + ) def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: return self.lookahead_call_helper(node, 1) |