diff options
47 files changed, 1438 insertions, 198 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6171571857..05f20e12f46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -260,7 +260,7 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 @@ -322,6 +322,81 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v4 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + build-wasi: name: 'WASI' needs: build-context @@ -628,7 +703,8 @@ jobs: - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl - build-wasi - test-hypothesis - build-asan @@ -643,7 +719,8 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, test-hypothesis, cifuzz, allowed-skips: >- @@ -661,7 +738,8 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, build-wasi, test-hypothesis, build-asan, diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 44e6a9ce2d0..0b64367e6c4 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -5,6 +5,7 @@ apt-get -yq install \ build-essential \ pkg-config \ ccache \ + cmake \ gdb \ lcov \ libb2-dev \ diff --git a/Doc/library/math.rst b/Doc/library/math.rst index bf7a00549fc..55f2de07553 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -42,6 +42,8 @@ noted otherwise, all return values are floats. :func:`fabs(x) <fabs>` Absolute value of *x* :func:`floor(x) <floor>` Floor of *x*, the largest integer less than or equal to *x* :func:`fma(x, y, z) <fma>` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmax(x, y) <fmax>` Maximum of two floating-point values +:func:`fmin(x, y) <fmin>` Minimum of two floating-point values :func:`fmod(x, y) <fmod>` Remainder of division ``x / y`` :func:`modf(x) <modf>` Fractional and integer parts of *x* :func:`remainder(x, y) <remainder>` Remainder of *x* with respect to *y* @@ -248,6 +250,30 @@ Floating point arithmetic .. versionadded:: 3.13 +.. function:: fmax(x, y) + + Get the larger of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmax` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.2). + + .. versionadded:: next + + +.. function:: fmin(x, y) + + Get the smaller of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmin` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.3). + + .. versionadded:: next + + .. function:: fmod(x, y) Return the floating-point remainder of ``x / y``, diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 06de152a742..37df13f8a1e 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -176,8 +176,8 @@ Cross platform :attr:`processor` is resolved late, on demand. Note: the first two attribute names differ from the names presented by - :func:`os.uname`, where they are named :attr:`sysname` and - :attr:`nodename`. + :func:`os.uname`, where they are named :attr:`!sysname` and + :attr:`!nodename`. Entries which cannot be determined are set to ``''``. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 23a2e0c3d0c..23808619524 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \ + serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -41,6 +42,21 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + for serializing and deserializing. This can be changed by supplying + *serializer* and *deserializer*, respectively. + + The *serializer* argument must be a callable which takes an object ``obj`` + and the *protocol* as inputs and returns the representation ``obj`` as a + :term:`bytes-like object`; the *protocol* value may be ignored by the + serializer. + + The *deserializer* argument must be callable which takes a serialized object + given as a :class:`bytes` object and returns the corresponding object. + + A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* + is not, or vice-versa. + .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. @@ -48,6 +64,10 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + .. versionchanged:: next + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -129,7 +149,8 @@ Restrictions explicitly. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: Shelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -147,6 +168,9 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. + The *serializer* and *deserializer* parameters have the same interpretation + as in :func:`~shelve.open`. + A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -161,8 +185,13 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') + +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, \ + serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -172,18 +201,27 @@ Restrictions modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* + parameters have the same interpretation as in :func:`~shelve.open`. + + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ + writeback=False, *, serializer=None, \ + deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *flag* parameter has the same interpretation as for the + :func:`.open` function. The optional *protocol*, *writeback*, *serializer* + and *deserializer* parameters have the same interpretation as in + :func:`~shelve.open`. + + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. .. _shelve-example: @@ -225,6 +263,20 @@ object):: d.close() # close it +Exceptions +---------- + +.. exception:: ShelveError + + Exception raised when one of the arguments *deserializer* and *serializer* + is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` + and :class:`DbfilenameShelf`. + + The *deserializer* and *serializer* arguments must be given together. + + .. versionadded:: next + + .. seealso:: Module :mod:`dbm` diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index ae2e324d0ab..a9930183f9a 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1078,8 +1078,9 @@ SSL Sockets (but passing a non-zero ``flags`` argument is not allowed) - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.sendfile` (it may be high-performant only when + the kernel TLS is enabled by setting :data:`~ssl.OP_ENABLE_KTLS` or when a + socket is plain-text, else :meth:`~socket.socket.send` will be used) - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop @@ -1113,6 +1114,11 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. + .. versionchanged:: next + Python now uses ``SSL_sendfile`` internally when possible. The + function sends a file more efficiently because it performs TLS encryption + in the kernel to avoid additional context switches. + SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.read(len=1024, buffer=None) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1fbb45ecd73..510225afab8 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -26,7 +26,6 @@ Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickletools.rst -Doc/library/platform.rst Doc/library/profile.rst Doc/library/pyexpat.rst Doc/library/resource.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e9b88458acd..010abb7d9b9 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -212,7 +212,7 @@ math * Add :func:`math.isnormal` and :func:`math.issubnormal` functions. (Contributed by Sergey B Kirpichev in :gh:`132908`.) -* Add :func:`math.signbit` function. +* Add :func:`math.fmax`, :func:`math.fmin` and :func:`math.signbit` functions. (Contributed by Bénédikt Tran in :gh:`135853`.) diff --git a/Include/Python.h b/Include/Python.h index 64be8014589..19417df698c 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -45,19 +45,19 @@ # endif #endif -// gh-111506: The free-threaded build is not compatible with the limited API -// or the stable ABI. -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -# error "The limited API is not currently supported in the free-threaded build" -#endif +#if defined(Py_GIL_DISABLED) +# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) +# error "Py_LIMITED_API is not currently supported in the free-threaded build" +# endif -#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) -# include <intrin.h> // __readgsqword() -#endif +# if defined(_MSC_VER) +# include <intrin.h> // __readgsqword() +# endif -#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) -# include <intrin.h> // __readgsqword() -#endif +# if defined(__MINGW32__) +# include <intrin.h> // __readgsqword() +# endif +#endif // Py_GIL_DISABLED // Include Python header files #include "pyport.h" diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 2a17c891dda..17634a93f8f 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*); PyAPI_DATA(PyTypeObject) PyModuleDef_Type; #endif +#ifndef _Py_OPAQUE_PYOBJECT typedef struct PyModuleDef_Base { PyObject_HEAD /* The function used to re-initialize the module. @@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base { 0, /* m_index */ \ _Py_NULL, /* m_copy */ \ } +#endif // _Py_OPAQUE_PYOBJECT #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 /* New in 3.5 */ @@ -104,6 +106,8 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif + +#ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; @@ -115,6 +119,7 @@ struct PyModuleDef { inquiry m_clear; freefunc m_free; }; +#endif // _Py_OPAQUE_PYOBJECT #ifdef __cplusplus } diff --git a/Include/object.h b/Include/object.h index c75e9db0cbd..b1bcc948187 100644 --- a/Include/object.h +++ b/Include/object.h @@ -56,6 +56,11 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif +#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API) +# error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API" +#endif + +#ifndef _Py_OPAQUE_PYOBJECT /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -99,6 +104,8 @@ whose size is determined when the object is allocated. * not necessarily a byte count. */ #define PyObject_VAR_HEAD PyVarObject ob_base; +#endif // !defined(_Py_OPAQUE_PYOBJECT) + #define Py_INVALID_SIZE (Py_ssize_t)-1 /* PyObjects are given a minimum alignment so that the least significant bits @@ -112,7 +119,9 @@ whose size is determined when the object is allocated. * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ -#ifndef Py_GIL_DISABLED +#ifdef _Py_OPAQUE_PYOBJECT + /* PyObject is opaque */ +#elif !defined(Py_GIL_DISABLED) struct _object { #if (defined(__GNUC__) || defined(__clang__)) \ && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) @@ -168,15 +177,18 @@ struct _object { Py_ssize_t ob_ref_shared; // shared (atomic) reference count PyTypeObject *ob_type; }; -#endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) -typedef struct { +#ifndef _Py_OPAQUE_PYOBJECT +struct PyVarObject { PyObject ob_base; Py_ssize_t ob_size; /* Number of items in variable part */ -} PyVarObject; +}; +#endif +typedef struct PyVarObject PyVarObject; /* Cast argument to PyVarObject* type. */ #define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op)) @@ -286,6 +298,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; +#ifndef _Py_OPAQUE_PYOBJECT // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { assert(Py_TYPE(ob) != &PyLong_Type); @@ -295,6 +308,7 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; @@ -304,6 +318,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -323,6 +338,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* diff --git a/Include/refcount.h b/Include/refcount.h index 457972b6dcf..ba34461fefc 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -117,6 +117,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #endif #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) @@ -140,6 +141,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) #endif } #define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) +#endif // !defined(_Py_OPAQUE_PYOBJECT) // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); diff --git a/Lib/email/message.py b/Lib/email/message.py index 41fcc2b9778..36e4b4a9f0b 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -313,6 +313,8 @@ class Message: # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 0e9bd98aa1f..6d4f84a5a6e 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -2,7 +2,7 @@ # Licensed to PSF under a Contributor Agreement. # -__doc__ = """hashlib module - A common interface to many hash functions. +__doc__ = r"""hashlib module - A common interface to many hash functions. new(name, data=b'', **kwargs) - returns a new hash object implementing the given hash function; initializing the hash @@ -12,7 +12,7 @@ Named constructor functions are also available, these are faster than using new(name): md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(), -sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256. +sha3_224(), sha3_256(), sha3_384(), sha3_512(), shake_128(), and shake_256(). More algorithms may be available on your platform but the above are guaranteed to exist. See the algorithms_guaranteed and algorithms_available attributes @@ -21,8 +21,8 @@ to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. -Choose your hash function wisely. Some have known collision weaknesses. -sha384 and sha512 will be slow on 32 bit platforms. +Choose your hash function wisely. Some have known collision weaknesses, +while others may be slower depending on the CPU architecture. Hash objects have these methods: - update(data): Update the hash object with the bytes in data. Repeated calls @@ -36,20 +36,20 @@ Hash objects have these methods: efficiently compute the digests of data that share a common initial substring. -For example, to obtain the digest of the byte string 'Nobody inspects the -spammish repetition': +Assuming that Python has been built with MD5 support, the following computes +the MD5 digest of the byte string b'Nobody inspects the spammish repetition': >>> import hashlib >>> m = hashlib.md5() >>> m.update(b"Nobody inspects") >>> m.update(b" the spammish repetition") >>> m.digest() - b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' + b'\xbbd\x9c\x83\xdd\x1e\xa5\xc9\xd9\xde\xc9\xa1\x8d\xf0\xff\xe9' More condensed: - >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() - 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' + >>> hashlib.md5(b"Nobody inspects the spammish repetition").hexdigest() + 'bb649c83dd1ea5c9d9dec9a18df0ffe9' """ @@ -203,7 +203,7 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): *digest* must either be a hash algorithm name as a *str*, a hash constructor, or a callable that returns a hash object. """ - # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy + # On Linux we could use AF_ALG sockets and sendfile() to achieve zero-copy # hashing with hardware acceleration. if isinstance(digest, str): digestobj = new(digest) diff --git a/Lib/shelve.py b/Lib/shelve.py index b53dc8b7a8e..1010be1e09d 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,12 +56,17 @@ entries in the cache, and empty the cache (d.sync() also synchronizes the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, dumps, loads from io import BytesIO import collections.abc -__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] +__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + + +class ShelveError(Exception): + pass + class _ClosedDict(collections.abc.MutableMapping): 'Marker for a closed dict. Access attempts raise a ValueError.' @@ -82,7 +87,7 @@ class Shelf(collections.abc.MutableMapping): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): self.dict = dict if protocol is None: protocol = DEFAULT_PROTOCOL @@ -91,6 +96,16 @@ class Shelf(collections.abc.MutableMapping): self.cache = {} self.keyencoding = keyencoding + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("serializer and deserializer must be " + "defined together") + else: + self.serializer = serializer + self.deserializer = deserializer + def __iter__(self): for k in self.dict.keys(): yield k.decode(self.keyencoding) @@ -110,8 +125,8 @@ class Shelf(collections.abc.MutableMapping): try: value = self.cache[key] except KeyError: - f = BytesIO(self.dict[key.encode(self.keyencoding)]) - value = Unpickler(f).load() + f = self.dict[key.encode(self.keyencoding)] + value = self.deserializer(f) if self.writeback: self.cache[key] = value return value @@ -119,10 +134,8 @@ class Shelf(collections.abc.MutableMapping): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - f = BytesIO() - p = Pickler(f, self._protocol) - p.dump(value) - self.dict[key.encode(self.keyencoding)] = f.getvalue() + serialized_value = self.serializer(value, self._protocol) + self.dict[key.encode(self.keyencoding)] = serialized_value def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -191,33 +204,29 @@ class BsdDbShelf(Shelf): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): - Shelf.__init__(self, dict, protocol, writeback, keyencoding) + keyencoding="utf-8", *, serializer=None, deserializer=None): + Shelf.__init__(self, dict, protocol, writeback, keyencoding, + serializer=serializer, deserializer=deserializer) def set_location(self, key): (key, value) = self.dict.set_location(key) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def next(self): (key, value) = next(self.dict) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def previous(self): (key, value) = self.dict.previous() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def first(self): (key, value) = self.dict.first() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def last(self): (key, value) = self.dict.last() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) class DbfilenameShelf(Shelf): @@ -227,9 +236,11 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False): + def __init__(self, filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, + serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -238,8 +249,8 @@ class DbfilenameShelf(Shelf): self.cache.clear() self.dict.clear() - -def open(filename, flag='c', protocol=None, writeback=False): +def open(filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -252,4 +263,5 @@ def open(filename, flag='c', protocol=None, writeback=False): See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback) + return DbfilenameShelf(filename, flag, protocol, writeback, + serializer=serializer, deserializer=deserializer) diff --git a/Lib/socket.py b/Lib/socket.py index 727b0e75f03..3073c012b19 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -56,6 +56,7 @@ import io import os import sys from enum import IntEnum, IntFlag +from functools import partial try: import errno @@ -348,75 +349,83 @@ class socket(_socket.socket): text.mode = mode return text - if hasattr(os, 'sendfile'): + def _sendfile_zerocopy(self, zerocopy_func, giveup_exc_type, file, + offset=0, count=None): + """ + Send a file using a zero-copy function. + """ + import selectors - def _sendfile_use_sendfile(self, file, offset=0, count=None): - # Lazy import to improve module import time - import selectors + self._check_sendfile_params(file, offset, count) + sockno = self.fileno() + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise giveup_exc_type(err) # not a regular file + try: + fsize = os.fstat(fileno).st_size + except OSError as err: + raise giveup_exc_type(err) # not a regular file + if not fsize: + return 0 # empty file + # Truncate to 1GiB to avoid OverflowError, see bpo-38319. + blocksize = min(count or fsize, 2 ** 30) + timeout = self.gettimeout() + if timeout == 0: + raise ValueError("non-blocking sockets are not supported") + # poll/select have the advantage of not requiring any + # extra file descriptor, contrarily to epoll/kqueue + # (also, they require a single syscall). + if hasattr(selectors, 'PollSelector'): + selector = selectors.PollSelector() + else: + selector = selectors.SelectSelector() + selector.register(sockno, selectors.EVENT_WRITE) - self._check_sendfile_params(file, offset, count) - sockno = self.fileno() - try: - fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: - raise _GiveupOnSendfile(err) # not a regular file - try: - fsize = os.fstat(fileno).st_size - except OSError as err: - raise _GiveupOnSendfile(err) # not a regular file - if not fsize: - return 0 # empty file - # Truncate to 1GiB to avoid OverflowError, see bpo-38319. - blocksize = min(count or fsize, 2 ** 30) - timeout = self.gettimeout() - if timeout == 0: - raise ValueError("non-blocking sockets are not supported") - # poll/select have the advantage of not requiring any - # extra file descriptor, contrarily to epoll/kqueue - # (also, they require a single syscall). - if hasattr(selectors, 'PollSelector'): - selector = selectors.PollSelector() - else: - selector = selectors.SelectSelector() - selector.register(sockno, selectors.EVENT_WRITE) - - total_sent = 0 - # localize variable access to minimize overhead - selector_select = selector.select - os_sendfile = os.sendfile - try: - while True: - if timeout and not selector_select(timeout): - raise TimeoutError('timed out') - if count: - blocksize = min(count - total_sent, blocksize) - if blocksize <= 0: - break - try: - sent = os_sendfile(sockno, fileno, offset, blocksize) - except BlockingIOError: - if not timeout: - # Block until the socket is ready to send some - # data; avoids hogging CPU resources. - selector_select() - continue - except OSError as err: - if total_sent == 0: - # We can get here for different reasons, the main - # one being 'file' is not a regular mmap(2)-like - # file, in which case we'll fall back on using - # plain send(). - raise _GiveupOnSendfile(err) - raise err from None - else: - if sent == 0: - break # EOF - offset += sent - total_sent += sent - return total_sent - finally: - if total_sent > 0 and hasattr(file, 'seek'): - file.seek(offset) + total_sent = 0 + # localize variable access to minimize overhead + selector_select = selector.select + try: + while True: + if timeout and not selector_select(timeout): + raise TimeoutError('timed out') + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + try: + sent = zerocopy_func(fileno, offset, blocksize) + except BlockingIOError: + if not timeout: + # Block until the socket is ready to send some + # data; avoids hogging CPU resources. + selector_select() + continue + except OSError as err: + if total_sent == 0: + # We can get here for different reasons, the main + # one being 'file' is not a regular mmap(2)-like + # file, in which case we'll fall back on using + # plain send(). + raise giveup_exc_type(err) + raise err from None + else: + if sent == 0: + break # EOF + offset += sent + total_sent += sent + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset) + + if hasattr(os, 'sendfile'): + def _sendfile_use_sendfile(self, file, offset=0, count=None): + return self._sendfile_zerocopy( + partial(os.sendfile, self.fileno()), + _GiveupOnSendfile, + file, offset, count, + ) else: def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile( diff --git a/Lib/ssl.py b/Lib/ssl.py index 7e3c4cbd6bb..86fb8990636 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -975,6 +975,10 @@ def _sslcopydoc(func): return func +class _GiveupOnSSLSendfile(Exception): + pass + + class SSLSocket(socket): """This class implements a subtype of socket.socket that wraps the underlying OS socket in an SSL context when necessary, and @@ -1266,15 +1270,26 @@ class SSLSocket(socket): return super().sendall(data, flags) def sendfile(self, file, offset=0, count=None): - """Send a file, possibly by using os.sendfile() if this is a - clear-text socket. Return the total number of bytes sent. + """Send a file, possibly by using an efficient sendfile() call if + the system supports it. Return the total number of bytes sent. """ - if self._sslobj is not None: - return self._sendfile_use_send(file, offset, count) - else: - # os.sendfile() works with plain sockets only + if self._sslobj is None: return super().sendfile(file, offset, count) + if not self._sslobj.uses_ktls_for_send(): + return self._sendfile_use_send(file, offset, count) + + sendfile = getattr(self._sslobj, "sendfile", None) + if sendfile is None: + return self._sendfile_use_send(file, offset, count) + + try: + return self._sendfile_zerocopy( + sendfile, _GiveupOnSSLSendfile, file, offset, count, + ) + except _GiveupOnSSLSendfile: + return self._sendfile_use_send(file, offset, count) + def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index bb0f8aa99da..2591e7ca6ab 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1899,6 +1899,8 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer # See gh-91153 class Boom: @@ -1916,6 +1918,39 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + def test_mutating_index_inbounds(self): + # gh-91153 continued + # Ensure buffer is not broken even if length is correct + + class MutatesOnIndex: + def __init__(self): + self.ba = bytearray(0x180) + + def __index__(self): + self.ba.clear() + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks + return 0 + + with self.subTest("skip_bounds_safety"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + class AssortedBytesTest(unittest.TestCase): # diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 93e7b2043d3..fb93c6ccbb6 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -12,7 +12,10 @@ import unittest from test import support -SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SOURCES = [ + os.path.join(os.path.dirname(__file__), 'extension.c'), + os.path.join(os.path.dirname(__file__), 'create_moduledef.c'), +] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -35,17 +38,22 @@ class BaseTests: def test_build(self): self.check_build('_test_cext') - def check_build(self, extension_name, std=None, limited=False): + def check_build(self, extension_name, std=None, limited=False, + opaque_pyobject=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, - std=std, limited=limited) + std=std, limited=limited, + opaque_pyobject=opaque_pyobject) - def _check_build(self, extension_name, python_exe, std, limited): + def _check_build(self, extension_name, python_exe, std, limited, + opaque_pyobject): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + for source in SOURCES: + dest = os.path.join(pkg_dir, os.path.basename(source)) + shutil.copy(source, dest) def run_cmd(operation, cmd): env = os.environ.copy() @@ -53,6 +61,8 @@ class BaseTests: env['CPYTHON_TEST_STD'] = std if limited: env['CPYTHON_TEST_LIMITED'] = '1' + if opaque_pyobject: + env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: @@ -107,6 +117,11 @@ class TestPublicCAPI(BaseTests, unittest.TestCase): def test_build_c11(self): self.check_build('_test_c11_cext', std='c11') + def test_build_opaque_pyobject(self): + # Test with _Py_OPAQUE_PYOBJECT + self.check_build('_test_limited_opaque_cext', limited=True, + opaque_pyobject=True) + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") def test_build_c99(self): # In public docs, we say C API is compatible with C11. However, diff --git a/Lib/test/test_cext/create_moduledef.c b/Lib/test/test_cext/create_moduledef.c new file mode 100644 index 00000000000..249c3163552 --- /dev/null +++ b/Lib/test/test_cext/create_moduledef.c @@ -0,0 +1,29 @@ +// Workaround for testing _Py_OPAQUE_PYOBJECT. +// See end of 'extension.c' + + +#undef _Py_OPAQUE_PYOBJECT +#undef Py_LIMITED_API +#include "Python.h" + + +// (repeated definition to avoid creating a header) +extern PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots); + +PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots) { + + static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, + }; + if (!_testcext_module.m_name) { + _testcext_module.m_name = name; + _testcext_module.m_doc = doc; + _testcext_module.m_methods = methods; + _testcext_module.m_slots = slots; + } + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 4be2f24c60d..73fc67ae59d 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -78,6 +78,9 @@ _testcext_exec( return 0; } +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + // Converting from function pointer to void* has undefined behavior, but // works on all known platforms, and CPython's module and type slots currently // need it. @@ -96,9 +99,10 @@ static PyModuleDef_Slot _testcext_slots[] = { _Py_COMP_DIAG_POP - PyDoc_STRVAR(_testcext_doc, "C test extension."); +#ifndef _Py_OPAQUE_PYOBJECT + static struct PyModuleDef _testcext_module = { PyModuleDef_HEAD_INIT, // m_base STR(MODULE_NAME), // m_name @@ -112,11 +116,30 @@ static struct PyModuleDef _testcext_module = { }; -#define _FUNC_NAME(NAME) PyInit_ ## NAME -#define FUNC_NAME(NAME) _FUNC_NAME(NAME) - PyMODINIT_FUNC FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcext_module); } + +#else // _Py_OPAQUE_PYOBJECT + +// Opaque PyObject means that PyModuleDef is also opaque and cannot be +// declared statically. See PEP 793. +// So, this part of module creation is split into a separate source file +// which uses non-limited API. + +// (repeated definition to avoid creating a header) +extern PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots); + + +PyMODINIT_FUNC +FUNC_NAME(MODULE_NAME)(void) +{ + return testcext_create_moduledef( + STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots); +} + +#endif // _Py_OPAQUE_PYOBJECT diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 587585e8086..4d71e4751f7 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -59,8 +59,11 @@ def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", "")) internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) + sources = [SOURCE] + if not internal: cflags = list(PUBLIC_CFLAGS) else: @@ -93,6 +96,11 @@ def main(): version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + # Define _Py_OPAQUE_PYOBJECT macro + if opaque_pyobject: + cflags.append(f'-D_Py_OPAQUE_PYOBJECT') + sources.append('create_moduledef.c') + if internal: cflags.append('-DTEST_INTERNAL_C_API=1') @@ -120,7 +128,7 @@ def main(): ext = Extension( module_name, - sources=[SOURCE], + sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, library_dirs=library_dirs) diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 23c39775a8b..b4128f70f18 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1055,6 +1055,15 @@ class TestEmailMessage(TestEmailMessageBase, TestEmailBase): # AttributeError: 'str' object has no attribute 'is_attachment' m.get_body() + def test_get_bytes_payload_with_quoted_printable_encoding(self): + # We use a memoryview to avoid directly changing the private payload + # and to prevent using the dedicated paths for string or bytes objects. + payload = memoryview(b'Some payload') + m = self._make_message() + m.add_header('Content-Transfer-Encoding', 'quoted-printable') + m.set_payload(payload) + self.assertEqual(m.get_payload(decode=True), payload) + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 0ea029b977b..4f3983d83c7 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5916,6 +5916,7 @@ class TestSignatureDefinitions(unittest.TestCase): def test_threading_module_has_signatures(self): import threading self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) def test_thread_module_has_signatures(self): import _thread diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 46cb54647b1..e3b0d4fa9ee 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -17,6 +17,7 @@ import sys eps = 1E-05 NAN = float('nan') +NNAN = float('-nan') INF = float('inf') NINF = float('-inf') FLOAT_MAX = sys.float_info.max @@ -636,6 +637,92 @@ class MathTests(unittest.TestCase): self.assertEqual(math.fmod(0.0, NINF), 0.0) self.assertRaises(ValueError, math.fmod, INF, INF) + def test_fmax(self): + self.assertRaises(TypeError, math.fmax) + self.assertRaises(TypeError, math.fmax, 'x', 'y') + + self.assertEqual(math.fmax(0., 0.), 0.) + self.assertEqual(math.fmax(1., 2.), 2.) + self.assertEqual(math.fmax(2., 1.), 2.) + + self.assertEqual(math.fmax(+1., +0.), 1.) + self.assertEqual(math.fmax(+0., +1.), 1.) + self.assertEqual(math.fmax(+1., -0.), 1.) + self.assertEqual(math.fmax(-0., +1.), 1.) + + self.assertEqual(math.fmax(-1., +0.), 0.) + self.assertEqual(math.fmax(+0., -1.), 0.) + self.assertEqual(math.fmax(-1., -0.), 0.) + self.assertEqual(math.fmax(-0., -1.), 0.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmax(INF, x), INF) + self.assertEqual(math.fmax(x, INF), INF) + self.assertEqual(math.fmax(NINF, x), x) + self.assertEqual(math.fmax(x, NINF), x) + + @requires_IEEE_754 + def test_fmax_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmax(NAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NAN))) + self.assertFalse(math.isnan(math.fmax(NNAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NNAN))) + # When both operands are NaNs, fmax() returns NaN (see C11, F.10.9.2) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmax(NAN, NAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NAN))) + + def test_fmin(self): + self.assertRaises(TypeError, math.fmin) + self.assertRaises(TypeError, math.fmin, 'x', 'y') + + self.assertEqual(math.fmin(0., 0.), 0.) + self.assertEqual(math.fmin(1., 2.), 1.) + self.assertEqual(math.fmin(2., 1.), 1.) + + self.assertEqual(math.fmin(+1., +0.), 0.) + self.assertEqual(math.fmin(+0., +1.), 0.) + self.assertEqual(math.fmin(+1., -0.), 0.) + self.assertEqual(math.fmin(-0., +1.), 0.) + + self.assertEqual(math.fmin(-1., +0.), -1.) + self.assertEqual(math.fmin(+0., -1.), -1.) + self.assertEqual(math.fmin(-1., -0.), -1.) + self.assertEqual(math.fmin(-0., -1.), -1.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmin(INF, x), x) + self.assertEqual(math.fmin(x, INF), x) + self.assertEqual(math.fmin(NINF, x), NINF) + self.assertEqual(math.fmin(x, NINF), NINF) + + @requires_IEEE_754 + def test_fmin_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmin(NAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NAN))) + self.assertFalse(math.isnan(math.fmin(NNAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NNAN))) + # When both operands are NaNs, fmin() returns NaN (see C11, F.10.9.3) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmin(NAN, NAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NAN))) + def testFrexp(self): self.assertRaises(TypeError, math.frexp) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 08c6562f2a2..64609ab9dd9 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,10 +1,11 @@ +import array import unittest import dbm import shelve import pickle import os -from test.support import os_helper +from test.support import import_helper, os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -165,6 +166,239 @@ class TestCase(unittest.TestCase): with shelve.Shelf({}) as s: self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) + def test_custom_serializer_and_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol): + if isinstance(obj, (bytes, bytearray, str)): + if protocol == 5: + return obj + return type(obj).__name__ + elif isinstance(obj, array.array): + return obj.tobytes() + raise TypeError(f"Unsupported type for serialization: {type(obj)}") + + def deserializer(data): + if isinstance(data, (bytes, bytearray, str)): + return data.decode("utf-8") + elif isinstance(data, array.array): + return array.array("b", data) + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual(s["foo"], str(bar)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + else: + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + + def test_custom_incomplete_serializer_and_deserializer(self): + dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with self.assertRaises(dbm_sqlite3.error): + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertIsNone(s["foo"]) + + def test_custom_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + data = obj.__class__.__name__ + if protocol == 5: + data = str(len(data)) + return data.encode("utf-8") + + def deserializer(data): + return data.decode("utf-8") + + def type_name_len(obj): + return f"{(len(type(obj).__name__))}" + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + bar = "bar" + bytes_obj = b"Hello, world!" + bytearray_obj = bytearray(b"\x00\x01\x02\x03\x04") + arr_obj = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_obj + s["bytearray_data"] = bytearray_obj + s["array_data"] = arr_obj + + if proto == 5: + self.assertEqual(s["foo"], type_name_len(bar)) + self.assertEqual(s["bytes_data"], type_name_len(bytes_obj)) + self.assertEqual(s["bytearray_data"], + type_name_len(bytearray_obj)) + self.assertEqual(s["array_data"], type_name_len(arr_obj)) + + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, type_name_len(bar)) + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + else: + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, "str") + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") + + def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertIsNone(s["foo"]) + self.assertNotEqual(s["foo"], "bar") + + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertEqual(s["foo"], "") + self.assertNotEqual(s["foo"], "bar") + + def test_missing_custom_deserializer(self): + def serializer(obj, protocol=None): + pass + + kwargs = dict(protocol=2, writeback=False, serializer=serializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + + def test_missing_custom_serializer(self): + def deserializer(data): + pass + + kwargs = dict(protocol=2, writeback=False, deserializer=deserializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + class TestShelveBase: type2test = shelve.Shelf diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f123f6ece40..9e519537ca5 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4316,19 +4316,30 @@ class ThreadedTests(unittest.TestCase): self.assertRaises(ValueError, s.write, b'hello') def test_sendfile(self): + """Try to send a file using kTLS if possible.""" TEST_DATA = b"x" * 512 with open(os_helper.TESTFN, 'wb') as f: f.write(TEST_DATA) self.addCleanup(os_helper.unlink, os_helper.TESTFN) client_context, server_context, hostname = testing_context() + client_context.options |= getattr(ssl, 'OP_ENABLE_KTLS', 0) server = ThreadedEchoServer(context=server_context, chatty=False) - with server: - with client_context.wrap_socket(socket.socket(), - server_hostname=hostname) as s: - s.connect((HOST, server.port)) + # kTLS seems to work only with a connection created before + # wrapping `sock` by the SSL context in contrast to calling + # `sock.connect()` after the wrapping. + with server, socket.create_connection((HOST, server.port)) as sock: + with client_context.wrap_socket( + sock, server_hostname=hostname + ) as ssock: + if support.verbose: + ktls_used = ssock._sslobj.uses_ktls_for_send() + print( + 'kTLS is', + 'available' if ktls_used else 'unavailable', + ) with open(os_helper.TESTFN, 'rb') as file: - s.sendfile(file) - self.assertEqual(s.recv(1024), TEST_DATA) + ssock.sendfile(file) + self.assertEqual(ssock.recv(1024), TEST_DATA) def test_session(self): client_context, server_context, hostname = testing_context() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 7ddacf07a2c..0e1a723ce3a 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1140,6 +1140,23 @@ class BaseTestUUID: weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -1214,13 +1231,25 @@ class BaseTestUUID: self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid def check_has_stable_libuuid_extractable_node(self): diff --git a/Misc/ACKS b/Misc/ACKS index d1490e1e46c..817b2125977 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1055,6 +1055,7 @@ Alexander Lakeev David Lam Thomas Lamb Valerie Lambert +Kliment Lamonov Peter Lamut Jean-Baptiste "Jiba" Lamy Ronan Lamy diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst new file mode 100644 index 00000000000..dc2f1e22ba5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -0,0 +1 @@ +Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst new file mode 100644 index 00000000000..c511c630214 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst @@ -0,0 +1,4 @@ +:mod:`ssl` now uses ``SSL_sendfile`` internally when it is possible (see +:data:`~ssl.OP_ENABLE_KTLS`). The function sends a file more efficiently +because it performs TLS encryption in the kernel to avoid additional context +switches. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst new file mode 100644 index 00000000000..735249b4dae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -0,0 +1,2 @@ +The :mod:`shelve` module now accepts custom serialization +and deserialization functions. diff --git a/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst b/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst new file mode 100644 index 00000000000..240ea72c69f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst @@ -0,0 +1,2 @@ +Add :func:`math.fmax` and :func:`math.fmin` to get the larger and smaller of +two floating-point values. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst new file mode 100644 index 00000000000..79b85320926 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst @@ -0,0 +1,2 @@ +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` when +the payload to decode is a :class:`bytes` object. Patch by Kliment Lamonov. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst new file mode 100644 index 00000000000..f3050ad5d5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst @@ -0,0 +1 @@ +Fix signature of :func:`threading.excepthook`. diff --git a/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst b/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst new file mode 100644 index 00000000000..ccd5bf11f04 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst @@ -0,0 +1,3 @@ +:mod:`!_hashlib`: avoid using deprecated functions +:manpage:`ERR_func_error_string` and :manpage:`EVP_MD_CTX_md` when using +OpenSSL 3.0 and later. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst b/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst new file mode 100644 index 00000000000..6885fba30db --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst @@ -0,0 +1 @@ +Add a new GitHub CI job to test the :mod:`ssl` module with `AWS-LC <https://github.com/aws/aws-lc>`_ as the backing cryptography and TLS library. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 90a7391ebb0..1a6c831e483 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -64,11 +64,15 @@ #define PY_EVP_MD_fetch(algorithm, properties) EVP_MD_fetch(NULL, algorithm, properties) #define PY_EVP_MD_up_ref(md) EVP_MD_up_ref(md) #define PY_EVP_MD_free(md) EVP_MD_free(md) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_get0_md(CTX) #else #define PY_EVP_MD const EVP_MD #define PY_EVP_MD_fetch(algorithm, properties) EVP_get_digestbyname(algorithm) #define PY_EVP_MD_up_ref(md) do {} while(0) #define PY_EVP_MD_free(md) do {} while(0) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_md(CTX) #endif /* hash alias map and fast lookup @@ -308,6 +312,14 @@ class _hashlib.HMAC "HMACobject *" "&PyType_Type" /* LCOV_EXCL_START */ +/* Thin wrapper around ERR_reason_error_string() returning non-NULL text. */ +static const char * +py_wrapper_ERR_reason_error_string(unsigned long errcode) +{ + const char *reason = ERR_reason_error_string(errcode); + return reason ? reason : "no reason"; +} + /* Set an exception of given type using the given OpenSSL error code. */ static void set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) @@ -317,8 +329,13 @@ set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ const char *lib = ERR_lib_error_string(errcode); +#ifdef Py_HAS_OPENSSL3_SUPPORT + // Since OpenSSL 3.0, ERR_func_error_string() always returns NULL. + const char *func = NULL; +#else const char *func = ERR_func_error_string(errcode); - const char *reason = ERR_reason_error_string(errcode); +#endif + const char *reason = py_wrapper_ERR_reason_error_string(errcode); if (lib && func) { PyErr_Format(exc_type, "[%s: %s] %s", lib, func, reason); @@ -838,7 +855,7 @@ static PyObject * _hashlib_HASH_get_name(PyObject *op, void *Py_UNUSED(closure)) { HASHobject *self = HASHobject_CAST(op); - const EVP_MD *md = EVP_MD_CTX_md(self->ctx); + const EVP_MD *md = PY_EVP_MD_CTX_md(self->ctx); if (md == NULL) { notify_ssl_error_occurred("missing EVP_MD for HASH context"); return NULL; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 014e624f6c2..24c243e330d 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -75,6 +75,33 @@ #endif +#ifdef BIO_get_ktls_send +# ifdef MS_WINDOWS +typedef long long Py_off_t; +# else +typedef off_t Py_off_t; +# endif + +static int +Py_off_t_converter(PyObject *arg, void *addr) +{ +#ifdef HAVE_LARGEFILE_SUPPORT + *((Py_off_t *)addr) = PyLong_AsLongLong(arg); +#else + *((Py_off_t *)addr) = PyLong_AsLong(arg); +#endif + return PyErr_Occurred() ? 0 : 1; +} + +/*[python input] + +class Py_off_t_converter(CConverter): + type = 'Py_off_t' + converter = 'Py_off_t_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=3fd9ca8ca6f0cbb8]*/ +#endif /* BIO_get_ktls_send */ struct py_ssl_error_code { const char *mnemonic; @@ -2444,6 +2471,184 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) /*[clinic input] @critical_section +_ssl._SSLSocket.uses_ktls_for_send + +Check if the Kernel TLS data-path is used for sending. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self) +/*[clinic end generated code: output=f9d95fbefceb5068 input=8d1ce4a131190e6b]*/ +{ +#ifdef BIO_get_ktls_send + int uses = BIO_get_ktls_send(SSL_get_wbio(self->ssl)); + // BIO_get_ktls_send() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_recv + +Check if the Kernel TLS data-path is used for receiving. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) +/*[clinic end generated code: output=ce38b00317a1f681 input=a13778a924fc7d44]*/ +{ +#ifdef BIO_get_ktls_recv + int uses = BIO_get_ktls_recv(SSL_get_rbio(self->ssl)); + // BIO_get_ktls_recv() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +#ifdef BIO_get_ktls_send +/*[clinic input] +@critical_section +_ssl._SSLSocket.sendfile + fd: int + offset: Py_off_t + size: size_t + flags: int = 0 + / + +Write size bytes from offset in the file descriptor fd to the SSL connection. + +This method uses the zero-copy technique and returns the number of bytes +written. It should be called only when Kernel TLS is used for sending data in +the connection. + +The meaning of flags is platform dependent. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags) +/*[clinic end generated code: output=0c6815b0719ca8d5 input=dfc1b162bb020de1]*/ +{ + Py_ssize_t retval; + int sockstate; + _PySSLError err; + PySocketSockObject *sock = GET_SOCKET(self); + PyTime_t timeout, deadline = 0; + int has_timeout; + + if (sock != NULL) { + if ((PyObject *)sock == Py_None) { + _setSSLError(get_state_sock(self), + "Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + /* just in case the blocking state of the socket has been changed */ + int nonblocking = (sock->sock_timeout >= 0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } + + timeout = GET_SOCKET_TIMEOUT(sock); + has_timeout = (timeout > 0); + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } + + sockstate = PySSL_select(sock, 1, timeout); + switch (sockstate) { + case SOCKET_HAS_TIMED_OUT: + PyErr_SetString(PyExc_TimeoutError, + "The write operation timed out"); + goto error; + case SOCKET_HAS_BEEN_CLOSED: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + case SOCKET_TOO_LARGE_FOR_SELECT: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket too large for select()."); + goto error; + } + + do { + PySSL_BEGIN_ALLOW_THREADS + retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); + err = _PySSL_errno(retval < 0, self->ssl, (int)retval); + PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) { + goto error; + } + + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } + + switch (err.ssl) { + case SSL_ERROR_WANT_READ: + sockstate = PySSL_select(sock, 0, timeout); + break; + case SSL_ERROR_WANT_WRITE: + sockstate = PySSL_select(sock, 1, timeout); + break; + default: + sockstate = SOCKET_OPERATION_OK; + break; + } + + if (sockstate == SOCKET_HAS_TIMED_OUT) { + PyErr_SetString(PyExc_TimeoutError, + "The sendfile operation timed out"); + goto error; + } + else if (sockstate == SOCKET_HAS_BEEN_CLOSED) { + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + } + else if (sockstate == SOCKET_IS_NONBLOCKING) { + break; + } + } while (err.ssl == SSL_ERROR_WANT_READ + || err.ssl == SSL_ERROR_WANT_WRITE); + + if (err.ssl == SSL_ERROR_SSL + && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED) + { + /* OpenSSL fails to return SSL_ERROR_SYSCALL if an error + * happens in sendfile(), and returns SSL_ERROR_SSL with + * SSL_R_UNINITIALIZED reason instead. */ + _setSSLError(get_state_sock(self), + "Some I/O error occurred in sendfile()", + PY_SSL_ERROR_SYSCALL, __FILE__, __LINE__); + goto error; + } + Py_XDECREF(sock); + if (retval < 0) { + return PySSL_SetError(self, __FILE__, __LINE__); + } + if (PySSL_ChainExceptions(self) < 0) { + return NULL; + } + return PyLong_FromSize_t(retval); +error: + Py_XDECREF(sock); + (void)PySSL_ChainExceptions(self); + return NULL; +} +#endif /* BIO_get_ktls_send */ + +/*[clinic input] +@critical_section _ssl._SSLSocket.write b: Py_buffer / @@ -3017,6 +3222,9 @@ static PyGetSetDef ssl_getsetlist[] = { static PyMethodDef PySSLMethods[] = { _SSL__SSLSOCKET_DO_HANDSHAKE_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF + _SSL__SSLSOCKET_SENDFILE_METHODDEF _SSL__SSLSOCKET_WRITE_METHODDEF _SSL__SSLSOCKET_READ_METHODDEF _SSL__SSLSOCKET_PENDING_METHODDEF diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 8886a9d6bd0..3540fead8e8 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2330,7 +2330,7 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +"_excepthook($module, args, /)\n\ --\n\ \n\ Handle uncaught Thread.run() exception."); diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index c6e2abd4d93..7027d873792 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -442,6 +443,115 @@ _ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(conte return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_send__doc__, +"uses_ktls_for_send($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for sending."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF \ + {"uses_ktls_for_send", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_send, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_send__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_send_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_recv__doc__, +"uses_ktls_for_recv($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for receiving."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF \ + {"uses_ktls_for_recv", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_recv, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_recv__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_recv_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(BIO_get_ktls_send) + +PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, +"sendfile($self, fd, offset, size, flags=0, /)\n" +"--\n" +"\n" +"Write size bytes from offset in the file descriptor fd to the SSL connection.\n" +"\n" +"This method uses the zero-copy technique and returns the number of bytes\n" +"written. It should be called only when Kernel TLS is used for sending data in\n" +"the connection.\n" +"\n" +"The meaning of flags is platform dependent."); + +#define _SSL__SSLSOCKET_SENDFILE_METHODDEF \ + {"sendfile", _PyCFunction_CAST(_ssl__SSLSocket_sendfile), METH_FASTCALL, _ssl__SSLSocket_sendfile__doc__}, + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags); + +static PyObject * +_ssl__SSLSocket_sendfile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + Py_off_t offset; + size_t size; + int flags = 0; + + if (!_PyArg_CheckPositional("sendfile", nargs, 3, 4)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + if (!Py_off_t_converter(args[1], &offset)) { + goto exit; + } + if (!_PyLong_Size_t_Converter(args[2], &size)) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + flags = PyLong_AsInt(args[3]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_sendfile_impl((PySSLSocket *)self, fd, offset, size, flags); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(BIO_get_ktls_send) */ + PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, "write($self, b, /)\n" "--\n" @@ -2893,6 +3003,10 @@ exit: #endif /* defined(_MSC_VER) */ +#ifndef _SSL__SSLSOCKET_SENDFILE_METHODDEF + #define _SSL__SSLSOCKET_SENDFILE_METHODDEF +#endif /* !defined(_SSL__SSLSOCKET_SENDFILE_METHODDEF) */ + #ifndef _SSL_ENUM_CERTIFICATES_METHODDEF #define _SSL_ENUM_CERTIFICATES_METHODDEF #endif /* !defined(_SSL_ENUM_CERTIFICATES_METHODDEF) */ @@ -2900,4 +3014,4 @@ exit: #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1adc3780d8ca682a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index a443c48faaa..246019f2206 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -84,6 +84,112 @@ PyDoc_STRVAR(math_floor__doc__, #define MATH_FLOOR_METHODDEF \ {"floor", (PyCFunction)math_floor, METH_O, math_floor__doc__}, +PyDoc_STRVAR(math_fmax__doc__, +"fmax($module, x, y, /)\n" +"--\n" +"\n" +"Return the larger of two floating-point arguments."); + +#define MATH_FMAX_METHODDEF \ + {"fmax", _PyCFunction_CAST(math_fmax), METH_FASTCALL, math_fmax__doc__}, + +static double +math_fmax_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmax(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmax", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmax_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_fmin__doc__, +"fmin($module, x, y, /)\n" +"--\n" +"\n" +"Return the smaller of two floating-point arguments."); + +#define MATH_FMIN_METHODDEF \ + {"fmin", _PyCFunction_CAST(math_fmin), METH_FASTCALL, math_fmin__doc__}, + +static double +math_fmin_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmin(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmin", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmin_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(math_signbit__doc__, "signbit($module, x, /)\n" "--\n" @@ -1212,4 +1318,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4e3fa94d026f027b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4fb180d4c25ff8fa input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 033de0b2907..7c2a421dd6a 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1214,6 +1214,40 @@ math_floor(PyObject *module, PyObject *number) return PyLong_FromDouble(floor(x)); } +/*[clinic input] +math.fmax -> double + + x: double + y: double + / + +Return the larger of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmax_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=00692358d312fee2 input=021596c027336ffe]*/ +{ + return fmax(x, y); +} + +/*[clinic input] +math.fmin -> double + + x: double + y: double + / + +Return the smaller of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmin_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=3d5b7826bd292dd9 input=d12e64ccc33f878a]*/ +{ + return fmin(x, y); +} + FUNC1AD(gamma, m_tgamma, "gamma($module, x, /)\n--\n\n" "Gamma function at x.", @@ -4192,7 +4226,9 @@ static PyMethodDef math_methods[] = { MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF MATH_FMA_METHODDEF + MATH_FMAX_METHODDEF MATH_FMOD_METHODDEF + MATH_FMIN_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b5d5ca9178e..bf30c06af5d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,7 +709,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +746,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +807,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +848,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index b1a5df91901..f4c8fde8346 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -1,12 +1,12 @@ #!./python -"""Run Python tests against multiple installations of OpenSSL and LibreSSL +"""Run Python tests against multiple installations of cryptography libraries The script - (1) downloads OpenSSL / LibreSSL tar bundle + (1) downloads the tar bundle (2) extracts it to ./src - (3) compiles OpenSSL / LibreSSL - (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ + (3) compiles the relevant library + (4) installs that library into ../multissl/$LIB/$VERSION/ (5) forces a recompilation of Python modules using the header and library files from ../multissl/$LIB/$VERSION/ (6) runs Python's test suite @@ -61,6 +61,10 @@ LIBRESSL_OLD_VERSIONS = [ LIBRESSL_RECENT_VERSIONS = [ ] +AWSLC_RECENT_VERSIONS = [ + "1.55.0", +] + # store files in ../multissl HERE = os.path.dirname(os.path.abspath(__file__)) PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) @@ -70,9 +74,9 @@ MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl')) parser = argparse.ArgumentParser( prog='multissl', description=( - "Run CPython tests with multiple OpenSSL and LibreSSL " + "Run CPython tests with multiple cryptography libraries" "versions." - ) + ), ) parser.add_argument( '--debug', @@ -103,6 +107,14 @@ parser.add_argument( ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) ) parser.add_argument( + '--awslc', + nargs='+', + default=(), + help=( + "AWS-LC versions, defaults to '{}' if no crypto library versions are given." + ).format(AWSLC_RECENT_VERSIONS) +) +parser.add_argument( '--tests', nargs='*', default=(), @@ -111,7 +123,7 @@ parser.add_argument( parser.add_argument( '--base-directory', default=MULTISSL_DIR, - help="Base directory for OpenSSL / LibreSSL sources and builds." + help="Base directory for crypto library sources and builds." ) parser.add_argument( '--no-network', @@ -124,8 +136,8 @@ parser.add_argument( choices=['library', 'modules', 'tests'], default='tests', help=( - "Which steps to perform. 'library' downloads and compiles OpenSSL " - "or LibreSSL. 'module' also compiles Python modules. 'tests' builds " + "Which steps to perform. 'library' downloads and compiles a crypto" + "library. 'module' also compiles Python modules. 'tests' builds " "all and runs the test suite." ) ) @@ -453,6 +465,34 @@ class BuildLibreSSL(AbstractBuilder): build_template = "libressl-{}" +class BuildAWSLC(AbstractBuilder): + library = "AWS-LC" + url_templates = ( + "https://github.com/aws/aws-lc/archive/refs/tags/v{v}.tar.gz", + ) + src_template = "aws-lc-{}.tar.gz" + build_template = "aws-lc-{}" + + def _build_src(self, config_args=()): + cwd = self.build_dir + log.info("Running build in {}".format(cwd)) + env = os.environ.copy() + env["LD_RUN_PATH"] = self.lib_dir # set rpath + if self.system: + env['SYSTEM'] = self.system + cmd = [ + "cmake", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_PREFIX_PATH={}".format(self.install_dir), + "-DCMAKE_INSTALL_PREFIX={}".format(self.install_dir), + "-DBUILD_SHARED_LIBS=ON", + "-DBUILD_TESTING=OFF", + "-DFIPS=OFF", + ] + self._subprocess_call(cmd, cwd=cwd, env=env) + self._subprocess_call(["make", "-j{}".format(self.jobs)], cwd=cwd, env=env) + + def configure_make(): if not os.path.isfile('Makefile'): log.info('Running ./configure') @@ -467,9 +507,10 @@ def configure_make(): def main(): args = parser.parse_args() - if not args.openssl and not args.libressl: + if not args.openssl and not args.libressl and not args.awslc: args.openssl = list(OPENSSL_RECENT_VERSIONS) args.libressl = list(LIBRESSL_RECENT_VERSIONS) + args.awslc = list(AWSLC_RECENT_VERSIONS) if not args.disable_ancient: args.openssl.extend(OPENSSL_OLD_VERSIONS) args.libressl.extend(LIBRESSL_OLD_VERSIONS) @@ -496,22 +537,15 @@ def main(): # download and register builder builds = [] - - for version in args.openssl: - build = BuildOpenSSL( - version, - args - ) - build.install() - builds.append(build) - - for version in args.libressl: - build = BuildLibreSSL( - version, - args - ) - build.install() - builds.append(build) + for build_class, versions in [ + (BuildOpenSSL, args.openssl), + (BuildLibreSSL, args.libressl), + (BuildAWSLC, args.awslc), + ]: + for version in versions: + build = build_class(version, args) + build.install() + builds.append(build) if args.steps in {'modules', 'tests'}: for build in builds: @@ -539,7 +573,7 @@ def main(): else: print('Executed all SSL tests.') - print('OpenSSL / LibreSSL versions:') + print('OpenSSL / LibreSSL / AWS-LC versions:') for build in builds: print(" * {0.library} {0.version}".format(build)) diff --git a/configure b/configure index 94a0b810333..4292f33ce21 100755 --- a/configure +++ b/configure @@ -30848,8 +30848,8 @@ main (void) OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ; diff --git a/configure.ac b/configure.ac index ade71bc011e..cc7a6e9397d 100644 --- a/configure.ac +++ b/configure.ac @@ -7529,8 +7529,8 @@ WITH_SAVE_ENV([ ], [ OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ])], [ac_cv_working_openssl_hashlib=yes], [ac_cv_working_openssl_hashlib=no]) ]) |