diff options
Diffstat (limited to 'Modules')
28 files changed, 933 insertions, 1067 deletions
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1bb65e0a649..a6606381e49 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -576,8 +576,16 @@ _ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) return PyLong_FromSsize_t(size); } +/*[clinic input] +@getter +@critical_section +_ctypes.CType_Type.__pointer_type__ + +[clinic start generated code]*/ + static PyObject * -ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) +_ctypes_CType_Type___pointer_type___get_impl(PyObject *self) +/*[clinic end generated code: output=718c9ff10b2b0012 input=ff7498aa6edf487c]*/ { ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *info; @@ -599,8 +607,16 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } +/*[clinic input] +@setter +@critical_section +_ctypes.CType_Type.__pointer_type__ + +[clinic start generated code]*/ + static int -ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored)) +_ctypes_CType_Type___pointer_type___set_impl(PyObject *self, PyObject *value) +/*[clinic end generated code: output=6259be8ea21693fa input=9b2dc2400c388982]*/ { ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *info; @@ -612,7 +628,7 @@ ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored)) return -1; } - Py_XSETREF(info->pointer_type, Py_XNewRef(tp)); + Py_XSETREF(info->pointer_type, Py_XNewRef(value)); return 0; } @@ -626,8 +642,7 @@ static PyMethodDef ctype_methods[] = { }; static PyGetSetDef ctype_getsets[] = { - { "__pointer_type__", ctype_get_pointer_type, ctype_set_pointer_type, - "pointer type", NULL }, + _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF { NULL, NULL } }; @@ -1254,9 +1269,11 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb return -1; } Py_XSETREF(stginfo->proto, Py_NewRef(proto)); + STGINFO_LOCK(info); if (info->pointer_type == NULL) { Py_XSETREF(info->pointer_type, Py_NewRef(self)); } + STGINFO_UNLOCK(); return 0; } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 856b0376e5e..404178ca623 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -92,7 +92,7 @@ module _ctypes #include <sanitizer/msan_interface.h> #endif -#if defined(_DEBUG) || defined(__MINGW32__) +#if defined(Py_DEBUG) || defined(__MINGW32__) /* Don't use structured exception handling on Windows if this is defined. MingW, AFAIK, doesn't support it. */ diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 92dfb8f83b7..d9a2ab19661 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -31,6 +31,56 @@ _ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const return _ctypes_CType_Type___sizeof___impl(self, cls); } +#if !defined(_ctypes_CType_Type___pointer_type___DOCSTR) +# define _ctypes_CType_Type___pointer_type___DOCSTR NULL +#endif +#if defined(_CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF) +# undef _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, (setter)_ctypes_CType_Type___pointer_type___set, _ctypes_CType_Type___pointer_type___DOCSTR}, +#else +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, NULL, _ctypes_CType_Type___pointer_type___DOCSTR}, +#endif + +static PyObject * +_ctypes_CType_Type___pointer_type___get_impl(PyObject *self); + +static PyObject * +_ctypes_CType_Type___pointer_type___get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CType_Type___pointer_type___get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CType_Type___pointer_type___DOCSTR) +# define _ctypes_CType_Type___pointer_type___DOCSTR NULL +#endif +#if defined(_CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF) +# undef _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, (setter)_ctypes_CType_Type___pointer_type___set, _ctypes_CType_Type___pointer_type___DOCSTR}, +#else +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", NULL, (setter)_ctypes_CType_Type___pointer_type___set, NULL}, +#endif + +static int +_ctypes_CType_Type___pointer_type___set_impl(PyObject *self, PyObject *value); + +static int +_ctypes_CType_Type___pointer_type___set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CType_Type___pointer_type___set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(CDataType_from_address__doc__, "from_address($self, value, /)\n" "--\n" @@ -1000,4 +1050,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=9fb75bf7e9a17df2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f4bc2a77ec073b8a input=a9049054013a1b77]*/ diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 8c3efa36353..fe24629f9f6 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -811,6 +811,8 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); + // The deepcopy() helper takes care of incrementing the refcount + // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); if (!tag) return NULL; @@ -845,11 +847,13 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) assert(!element->extra || !element->extra->length); if (self->extra) { - if (element_resize(element, self->extra->length) < 0) + Py_ssize_t expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + assert(!element->extra->length); goto error; + } - // TODO(picnixz): check for an evil child's __deepcopy__ on 'self' - for (i = 0; i < self->extra->length; i++) { + for (i = 0; self->extra && i < self->extra->length; i++) { PyObject* child = deepcopy(st, self->extra->children[i], memo); if (!child || !Element_Check(st, child)) { if (child) { @@ -859,11 +863,24 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) element->extra->length = i; goto error; } + if (self->extra && expected_count != self->extra->length) { + // 'self->extra' got mutated and 'element' may not have + // sufficient space to hold the next iteration's item. + expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + Py_DECREF(child); + element->extra->length = i; + goto error; + } + } element->extra->children[i] = child; } assert(!element->extra->length); - element->extra->length = self->extra->length; + // The original 'self->extra' may be gone at this point if deepcopy() + // has side-effects. However, 'i' is the number of copied items that + // we were able to successfully copy. + element->extra->length = i; } /* add object to memo dictionary (so deepcopy won't visit it again) */ @@ -906,13 +923,20 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) break; } } - if (simple) + if (simple) { return PyDict_Copy(object); + } /* Fall through to general case */ } else if (Element_CheckExact(st, object)) { - return _elementtree_Element___deepcopy___impl( + // The __deepcopy__() call may call arbitrary code even if the + // object to copy is a built-in XML element (one of its children + // any of its parents in its own __deepcopy__() implementation). + Py_INCREF(object); + PyObject *res = _elementtree_Element___deepcopy___impl( (ElementObject *)object, memo); + Py_DECREF(object); + return res; } } @@ -923,8 +947,11 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return NULL; } + Py_INCREF(object); PyObject *args[2] = {object, memo}; - return PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + PyObject *res = PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + Py_DECREF(object); + return res; } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e6c454faf4b..899eef50ecc 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -196,6 +196,19 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + /* keyword Placeholder prohibition */ + if (kw != NULL) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(kw, &pos, &key, &val)) { + if (val == phold) { + PyErr_SetString(PyExc_TypeError, + "Placeholder cannot be passed as a keyword argument"); + return NULL; + } + } + } + /* check wrapped function / object */ pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e45a2d1a16d..4cde5f87032 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,8 +1,10 @@ #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_sysmodule.h" // _PySys_GetSizeOf() -#include <stddef.h> // offsetof() +#include <stddef.h> // offsetof() #include "_iomodule.h" /*[clinic input] @@ -50,7 +52,7 @@ check_closed(bytesio *self) static int check_exports(bytesio *self) { - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return 1; @@ -68,15 +70,17 @@ check_exports(bytesio *self) return NULL; \ } -#define SHARED_BUF(self) (Py_REFCNT((self)->buf) > 1) +#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) /* Internal routine to get a line from the buffer of a BytesIO object. Returns the length between the current position to the next newline character. */ static Py_ssize_t -scan_eol(bytesio *self, Py_ssize_t len) +scan_eol_lock_held(bytesio *self, Py_ssize_t len) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *start, *n; Py_ssize_t maxlen; @@ -109,11 +113,13 @@ scan_eol(bytesio *self, Py_ssize_t len) The caller should ensure that the 'size' argument is non-negative and not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ static int -unshare_buffer(bytesio *self, size_t size) +unshare_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + PyObject *new_buf; assert(SHARED_BUF(self)); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); assert(size >= (size_t)self->string_size); new_buf = PyBytes_FromStringAndSize(NULL, size); if (new_buf == NULL) @@ -128,10 +134,12 @@ unshare_buffer(bytesio *self, size_t size) The caller should ensure that the 'size' argument is non-negative. Returns 0 on success, -1 otherwise. */ static int -resize_buffer(bytesio *self, size_t size) +resize_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(self->buf != NULL); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ @@ -160,7 +168,7 @@ resize_buffer(bytesio *self, size_t size) } if (SHARED_BUF(self)) { - if (unshare_buffer(self, alloc) < 0) + if (unshare_buffer_lock_held(self, alloc) < 0) return -1; } else { @@ -181,8 +189,10 @@ resize_buffer(bytesio *self, size_t size) Inlining is disabled because it's significantly decreases performance of writelines() in PGO build. */ Py_NO_INLINE static Py_ssize_t -write_bytes(bytesio *self, PyObject *b) +write_bytes_lock_held(bytesio *self, PyObject *b) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (check_closed(self)) { return -1; } @@ -202,13 +212,13 @@ write_bytes(bytesio *self, PyObject *b) assert(self->pos >= 0); size_t endpos = (size_t)self->pos + len; if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) { - if (resize_buffer(self, endpos) < 0) { + if (resize_buffer_lock_held(self, endpos) < 0) { len = -1; goto done; } } else if (SHARED_BUF(self)) { - if (unshare_buffer(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { + if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { len = -1; goto done; } @@ -245,13 +255,17 @@ write_bytes(bytesio *self, PyObject *b) static PyObject * bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) { + PyObject *ret; bytesio *self = bytesio_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); if (self->buf == NULL) { - Py_RETURN_TRUE; + ret = Py_True; } else { - Py_RETURN_FALSE; + ret = Py_False; } + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] @@ -311,6 +325,7 @@ _io_BytesIO_flush_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.getbuffer cls: defining_class @@ -321,7 +336,7 @@ Get a read-write view over the contents of the BytesIO object. static PyObject * _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) -/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ +/*[clinic end generated code: output=045091d7ce87fe4e input=8295764061be77fd]*/ { _PyIO_State *state = get_io_state_by_cls(cls); PyTypeObject *type = state->PyBytesIOBuffer_Type; @@ -340,6 +355,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _io.BytesIO.getvalue Retrieve the entire contents of the BytesIO object. @@ -347,16 +363,16 @@ Retrieve the entire contents of the BytesIO object. static PyObject * _io_BytesIO_getvalue_impl(bytesio *self) -/*[clinic end generated code: output=b3f6a3233c8fd628 input=4b403ac0af3973ed]*/ +/*[clinic end generated code: output=b3f6a3233c8fd628 input=c91bff398df0c352]*/ { CHECK_CLOSED(self); - if (self->string_size <= 1 || self->exports > 0) + if (self->string_size <= 1 || FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf), self->string_size); if (self->string_size != PyBytes_GET_SIZE(self->buf)) { if (SHARED_BUF(self)) { - if (unshare_buffer(self, self->string_size) < 0) + if (unshare_buffer_lock_held(self, self->string_size) < 0) return NULL; } else { @@ -384,6 +400,7 @@ _io_BytesIO_isatty_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.tell Current file position, an integer. @@ -391,22 +408,24 @@ Current file position, an integer. static PyObject * _io_BytesIO_tell_impl(bytesio *self) -/*[clinic end generated code: output=b54b0f93cd0e5e1d input=b106adf099cb3657]*/ +/*[clinic end generated code: output=b54b0f93cd0e5e1d input=2c7b0e8f82e05c4d]*/ { CHECK_CLOSED(self); return PyLong_FromSsize_t(self->pos); } static PyObject * -read_bytes(bytesio *self, Py_ssize_t size) +read_bytes_lock_held(bytesio *self, Py_ssize_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *output; assert(self->buf != NULL); assert(size <= self->string_size); if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && - self->exports == 0) { + FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { self->pos += size; return Py_NewRef(self->buf); } @@ -417,6 +436,7 @@ read_bytes(bytesio *self, Py_ssize_t size) } /*[clinic input] +@critical_section _io.BytesIO.read size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -429,7 +449,7 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9cc025f21c75bdd2 input=74344a39f431c3d7]*/ +/*[clinic end generated code: output=9cc025f21c75bdd2 input=9e2f7ff3075fdd39]*/ { Py_ssize_t n; @@ -443,11 +463,12 @@ _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) size = 0; } - return read_bytes(self, size); + return read_bytes_lock_held(self, size); } /*[clinic input] +@critical_section _io.BytesIO.read1 size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -460,12 +481,13 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=440a395bf9129ef5]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=a08fc9e507ab380c]*/ { return _io_BytesIO_read_impl(self, size); } /*[clinic input] +@critical_section _io.BytesIO.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -479,18 +501,19 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=4bff3c251df8ffcd input=e7c3fbd1744e2783]*/ +/*[clinic end generated code: output=4bff3c251df8ffcd input=db09d47e23cf2c9e]*/ { Py_ssize_t n; CHECK_CLOSED(self); - n = scan_eol(self, size); + n = scan_eol_lock_held(self, size); - return read_bytes(self, n); + return read_bytes_lock_held(self, n); } /*[clinic input] +@critical_section _io.BytesIO.readlines size as arg: object = None / @@ -504,7 +527,7 @@ total number of bytes in the lines returned. static PyObject * _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) -/*[clinic end generated code: output=09b8e34c880808ff input=691aa1314f2c2a87]*/ +/*[clinic end generated code: output=09b8e34c880808ff input=5c57d7d78e409985]*/ { Py_ssize_t maxsize, size, n; PyObject *result, *line; @@ -533,7 +556,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) return NULL; output = PyBytes_AS_STRING(self->buf) + self->pos; - while ((n = scan_eol(self, -1)) != 0) { + while ((n = scan_eol_lock_held(self, -1)) != 0) { self->pos += n; line = PyBytes_FromStringAndSize(output, n); if (!line) @@ -556,6 +579,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) } /*[clinic input] +@critical_section _io.BytesIO.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -568,7 +592,7 @@ is set not to block and has no data to read. static PyObject * _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) -/*[clinic end generated code: output=a5d407217dcf0639 input=1424d0fdce857919]*/ +/*[clinic end generated code: output=a5d407217dcf0639 input=093a8d330de3fcd1]*/ { Py_ssize_t len, n; @@ -592,8 +616,9 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) } /*[clinic input] +@critical_section _io.BytesIO.truncate - size: Py_ssize_t(accept={int, NoneType}, c_default="((bytesio *)self)->pos") = None + size: object = None / Truncate the file to at most size bytes. @@ -603,44 +628,68 @@ The current file position is unchanged. Returns the new size. [clinic start generated code]*/ static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9ad17650c15fa09b input=dae4295e11c1bbb4]*/ +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size) +/*[clinic end generated code: output=ab42491b4824f384 input=b4acb5f80481c053]*/ { CHECK_CLOSED(self); CHECK_EXPORTS(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative size value %zd", size); - return NULL; + Py_ssize_t new_size; + + if (size == Py_None) { + new_size = self->pos; + } + else { + new_size = PyLong_AsLong(size); + if (new_size == -1 && PyErr_Occurred()) { + return NULL; + } + if (new_size < 0) { + PyErr_Format(PyExc_ValueError, + "negative size value %zd", new_size); + return NULL; + } } - if (size < self->string_size) { - self->string_size = size; - if (resize_buffer(self, size) < 0) + if (new_size < self->string_size) { + self->string_size = new_size; + if (resize_buffer_lock_held(self, new_size) < 0) return NULL; } - return PyLong_FromSsize_t(size); + return PyLong_FromSsize_t(new_size); } static PyObject * -bytesio_iternext(PyObject *op) +bytesio_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + Py_ssize_t n; bytesio *self = bytesio_CAST(op); CHECK_CLOSED(self); - n = scan_eol(self, -1); + n = scan_eol_lock_held(self, -1); if (n == 0) return NULL; - return read_bytes(self, n); + return read_bytes_lock_held(self, n); +} + +static PyObject * +bytesio_iternext(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.seek pos: Py_ssize_t whence: int = 0 @@ -657,7 +706,7 @@ Returns the new absolute position. static PyObject * _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=c26204a68e9190e4 input=1e875e6ebc652948]*/ +/*[clinic end generated code: output=c26204a68e9190e4 input=20f05ddf659255df]*/ { CHECK_CLOSED(self); @@ -700,6 +749,7 @@ _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) } /*[clinic input] +@critical_section _io.BytesIO.write b: object / @@ -711,13 +761,14 @@ Return the number of bytes written. static PyObject * _io_BytesIO_write_impl(bytesio *self, PyObject *b) -/*[clinic end generated code: output=d3e46bcec8d9e21c input=f5ec7c8c64ed720a]*/ +/*[clinic end generated code: output=d3e46bcec8d9e21c input=46c0c17eac7474a4]*/ { - Py_ssize_t n = write_bytes(self, b); + Py_ssize_t n = write_bytes_lock_held(self, b); return n >= 0 ? PyLong_FromSsize_t(n) : NULL; } /*[clinic input] +@critical_section _io.BytesIO.writelines lines: object / @@ -731,7 +782,7 @@ each element. static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=e972539176fc8fc1]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=5d6a616ae39dc9ca]*/ { PyObject *it, *item; @@ -742,7 +793,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) return NULL; while ((item = PyIter_Next(it)) != NULL) { - Py_ssize_t ret = write_bytes(self, item); + Py_ssize_t ret = write_bytes_lock_held(self, item); Py_DECREF(item); if (ret < 0) { Py_DECREF(it); @@ -759,6 +810,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) } /*[clinic input] +@critical_section _io.BytesIO.close Disable all I/O operations. @@ -766,7 +818,7 @@ Disable all I/O operations. static PyObject * _io_BytesIO_close_impl(bytesio *self) -/*[clinic end generated code: output=1471bb9411af84a0 input=37e1f55556e61f60]*/ +/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/ { CHECK_EXPORTS(self); Py_CLEAR(self->buf); @@ -788,35 +840,49 @@ _io_BytesIO_close_impl(bytesio *self) function to use the efficient instance representation of PEP 307. */ + static PyObject * + bytesio_getstate_lock_held(PyObject *op) + { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + + bytesio *self = bytesio_CAST(op); + PyObject *initvalue = _io_BytesIO_getvalue_impl(self); + PyObject *dict; + PyObject *state; + + if (initvalue == NULL) + return NULL; + if (self->dict == NULL) { + dict = Py_NewRef(Py_None); + } + else { + dict = PyDict_Copy(self->dict); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + } + + state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); + Py_DECREF(initvalue); + return state; +} + static PyObject * bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) { - bytesio *self = bytesio_CAST(op); - PyObject *initvalue = _io_BytesIO_getvalue_impl(self); - PyObject *dict; - PyObject *state; - - if (initvalue == NULL) - return NULL; - if (self->dict == NULL) { - dict = Py_NewRef(Py_None); - } - else { - dict = PyDict_Copy(self->dict); - if (dict == NULL) { - Py_DECREF(initvalue); - return NULL; - } - } - - state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); - Py_DECREF(initvalue); - return state; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_getstate_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -bytesio_setstate(PyObject *op, PyObject *state) +bytesio_setstate_lock_held(PyObject *op, PyObject *state) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + PyObject *result; PyObject *position_obj; PyObject *dict; @@ -890,13 +956,23 @@ bytesio_setstate(PyObject *op, PyObject *state) Py_RETURN_NONE; } +static PyObject * +bytesio_setstate(PyObject *op, PyObject *state) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_setstate_lock_held(op, state); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesio_dealloc(PyObject *op) { bytesio *self = bytesio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_SystemError, "deallocated BytesIO object has exported buffers"); PyErr_Print(); @@ -932,6 +1008,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@critical_section _io.BytesIO.__init__ initial_bytes as initvalue: object(c_default="NULL") = b'' @@ -940,13 +1017,13 @@ Buffered I/O implementation using an in-memory bytes buffer. static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) -/*[clinic end generated code: output=65c0c51e24c5b621 input=aac7f31b67bf0fb6]*/ +/*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/ { /* In case, __init__ is called multiple times. */ self->string_size = 0; self->pos = 0; - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return -1; @@ -970,8 +1047,10 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) } static PyObject * -bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +bytesio_sizeof_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { @@ -984,6 +1063,16 @@ bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) return PyLong_FromSize_t(res); } +static PyObject * +bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_sizeof_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int bytesio_traverse(PyObject *op, visitproc visit, void *arg) { @@ -999,7 +1088,7 @@ bytesio_clear(PyObject *op) { bytesio *self = bytesio_CAST(op); Py_CLEAR(self->dict); - if (self->exports == 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { Py_CLEAR(self->buf); } return 0; @@ -1077,18 +1166,15 @@ PyType_Spec bytesio_spec = { */ static int -bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - if (view == NULL) { - PyErr_SetString(PyExc_BufferError, - "bytesiobuf_getbuffer: view==NULL argument is obsolete"); - return -1; - } - if (b->exports == 0 && SHARED_BUF(b)) { - if (unshare_buffer(b, b->string_size) < 0) + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b); + + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) { + if (unshare_buffer_lock_held(b, b->string_size) < 0) return -1; } @@ -1096,16 +1182,32 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) (void)PyBuffer_FillInfo(view, op, PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); - b->exports++; + FT_ATOMIC_ADD_SSIZE(b->exports, 1); return 0; } +static int +bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + if (view == NULL) { + PyErr_SetString(PyExc_BufferError, + "bytesiobuf_getbuffer: view==NULL argument is obsolete"); + return -1; + } + + int ret; + Py_BEGIN_CRITICAL_SECTION(bytesiobuf_CAST(op)->source); + ret = bytesiobuf_getbuffer_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - b->exports--; + FT_ATOMIC_ADD_SSIZE(b->exports, -1); } static int diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index aaf4884d173..8553ed05f70 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_BytesIO_readable__doc__, @@ -96,11 +97,18 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); - return NULL; + goto exit; } - return _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -118,7 +126,13 @@ _io_BytesIO_getvalue_impl(bytesio *self); static PyObject * _io_BytesIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_getvalue_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getvalue_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_isatty__doc__, @@ -156,7 +170,13 @@ _io_BytesIO_tell_impl(bytesio *self); static PyObject * _io_BytesIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_tell_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_tell_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_read__doc__, @@ -190,7 +210,9 @@ _io_BytesIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -227,7 +249,9 @@ _io_BytesIO_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read1_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +289,9 @@ _io_BytesIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readline_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -301,7 +327,9 @@ _io_BytesIO_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } arg = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readlines_impl((bytesio *)self, arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,7 +360,9 @@ _io_BytesIO_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readinto_impl((bytesio *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -356,13 +386,13 @@ PyDoc_STRVAR(_io_BytesIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_BytesIO_truncate), METH_FASTCALL, _io_BytesIO_truncate__doc__}, static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size); +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size); static PyObject * _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((bytesio *)self)->pos; + PyObject *size = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -370,11 +400,11 @@ _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + size = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_truncate_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -428,7 +458,9 @@ _io_BytesIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_seek_impl((bytesio *)self, pos, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -453,7 +485,9 @@ _io_BytesIO_write(PyObject *self, PyObject *b) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_write_impl((bytesio *)self, b); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -479,7 +513,9 @@ _io_BytesIO_writelines(PyObject *self, PyObject *lines) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_writelines_impl((bytesio *)self, lines); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -499,7 +535,13 @@ _io_BytesIO_close_impl(bytesio *self); static PyObject * _io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_close_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_close_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO___init____doc__, @@ -558,9 +600,11 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } initvalue = fastargs[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO___init___impl((bytesio *)self, initvalue); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=6dbfd82f4e9d4ef3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=580205daa01def2e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 8e8cd8df9ab..83165e5f7ad 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -149,13 +149,13 @@ PyDoc_STRVAR(_io_StringIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_StringIO_truncate), METH_FASTCALL, _io_StringIO_truncate__doc__}, static PyObject * -_io_StringIO_truncate_impl(stringio *self, Py_ssize_t size); +_io_StringIO_truncate_impl(stringio *self, PyObject *pos); static PyObject * _io_StringIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((stringio *)self)->pos; + PyObject *pos = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -163,12 +163,10 @@ _io_StringIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + pos = args[0]; skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_truncate_impl((stringio *)self, size); + return_value = _io_StringIO_truncate_impl((stringio *)self, pos); Py_END_CRITICAL_SECTION(); exit: @@ -552,4 +550,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=5bfaaab7f41ee6b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bccc25ef8e6ce9ef input=a9049054013a1b77]*/ diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 9d1bfa3ea05..56913fafefb 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -444,7 +444,7 @@ stringio_iternext(PyObject *op) /*[clinic input] @critical_section _io.StringIO.truncate - pos as size: Py_ssize_t(accept={int, NoneType}, c_default="((stringio *)self)->pos") = None + pos: object = None / Truncate size to pos. @@ -455,16 +455,26 @@ Returns the new absolute position. [clinic start generated code]*/ static PyObject * -_io_StringIO_truncate_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=eb3aef8e06701365 input=fa8a6c98bb2ba780]*/ +_io_StringIO_truncate_impl(stringio *self, PyObject *pos) +/*[clinic end generated code: output=c76c43b5ecfaf4e2 input=d59fd2ee49757ae6]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "Negative size value %zd", size); - return NULL; + Py_ssize_t size; + if (pos == Py_None) { + size = self->pos; + } + else { + size = PyLong_AsLong(pos); + if (size == -1 && PyErr_Occurred()) { + return NULL; + } + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "negative pos value %zd", size); + return NULL; + } } if (size < self->string_size) { diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f9b4c2a170e..462c2181fa6 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -17,6 +17,7 @@ #include <lzma.h> +#include "pycore_long.h" // _PyLong_UInt32_Converter() // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -223,8 +224,6 @@ FUNCNAME(PyObject *obj, void *ptr) \ return 1; \ } -INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter) -INT_TYPE_CONVERTER_FUNC(uint64_t, uint64_converter) INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter) INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter) INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) @@ -254,7 +253,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) return NULL; } if (preset_obj != NULL) { - int ok = uint32_converter(preset_obj, &preset); + int ok = _PyLong_UInt32_Converter(preset_obj, &preset); Py_DECREF(preset_obj); if (!ok) { return NULL; @@ -275,14 +274,14 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, - uint32_converter, &options->dict_size, - uint32_converter, &options->lc, - uint32_converter, &options->lp, - uint32_converter, &options->pb, + _PyLong_UInt32_Converter, &options->dict_size, + _PyLong_UInt32_Converter, &options->lc, + _PyLong_UInt32_Converter, &options->lp, + _PyLong_UInt32_Converter, &options->pb, lzma_mode_converter, &options->mode, - uint32_converter, &options->nice_len, + _PyLong_UInt32_Converter, &options->nice_len, lzma_mf_converter, &options->mf, - uint32_converter, &options->depth)) { + _PyLong_UInt32_Converter, &options->depth)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); @@ -301,7 +300,7 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) lzma_options_delta *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &dist)) { + &id, _PyLong_UInt32_Converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); return NULL; @@ -325,7 +324,7 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) lzma_options_bcj *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &start_offset)) { + &id, _PyLong_UInt32_Converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); return NULL; @@ -806,7 +805,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } - if (preset_obj != Py_None && !uint32_converter(preset_obj, &preset)) { + if (preset_obj != Py_None && !_PyLong_UInt32_Converter(preset_obj, &preset)) { return NULL; } @@ -1226,7 +1225,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, "Cannot specify memory limit with FORMAT_RAW"); return NULL; } - if (!uint64_converter(memlimit, &memlimit_)) { + if (!_PyLong_UInt64_Converter(memlimit, &memlimit_)) { return NULL; } } diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index 1bcda7702c2..e9d560666c1 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -9,23 +9,17 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pysqlite_connect__doc__, -"connect($module, /, database, timeout=5.0, detect_types=0,\n" +"connect($module, /, database, *, timeout=5.0, detect_types=0,\n" " isolation_level=\'\', check_same_thread=True,\n" -" factory=ConnectionType, cached_statements=128, uri=False, *,\n" +" factory=ConnectionType, cached_statements=128, uri=False,\n" " autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" "--\n" "\n" "Open a connection to the SQLite database file \'database\'.\n" "\n" "You can use \":memory:\" to open a database connection to a database that\n" -"resides in RAM instead of on disk.\n" -"\n" -"Note: Passing more than 1 positional argument to _sqlite3.connect() is\n" -"deprecated. Parameters \'timeout\', \'detect_types\', \'isolation_level\',\n" -"\'check_same_thread\', \'factory\', \'cached_statements\' and \'uri\' will\n" -"become keyword-only parameters in Python 3.15.\n" -""); +"resides in RAM instead of on disk."); #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=69b9b00da71c3c0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3d83139ba65e0bb5 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index c8e1d0b7a73..f0e9fdb8894 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -16,17 +16,6 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int cache_size, int uri, enum autocommit_mode autocommit); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.__init__'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.__init__'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.__init__'." -# endif -#endif - static int pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -72,25 +61,14 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; - if (nargs > 1 && nargs <= 8) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to _sqlite3.Connection()" - " is deprecated. Parameters 'timeout', 'detect_types', " - "'isolation_level', 'check_same_thread', 'factory', " - "'cached_statements' and 'uri' will become keyword-only " - "parameters in Python 3.15.", 1)) - { - goto exit; - } - } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } database = fastargs[0]; if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } if (fastargs[1]) { if (PyFloat_CheckExact(fastargs[1])) { @@ -104,7 +82,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[2]) { @@ -113,7 +91,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[3]) { @@ -121,7 +99,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[4]) { @@ -130,13 +108,13 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[5]) { factory = fastargs[5]; if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[6]) { @@ -145,7 +123,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[7]) { @@ -154,13 +132,9 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } if (!autocommit_converter(fastargs[8], &autocommit)) { goto exit; } @@ -424,15 +398,10 @@ pysqlite_connection_rollback(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(pysqlite_connection_create_function__doc__, -"create_function($self, /, name, narg, func, *, deterministic=False)\n" +"create_function($self, name, narg, func, /, *, deterministic=False)\n" "--\n" "\n" -"Creates a new function.\n" -"\n" -"Note: Passing keyword arguments \'name\', \'narg\' and \'func\' to\n" -"_sqlite3.Connection.create_function() is deprecated. Parameters\n" -"\'name\', \'narg\' and \'func\' will become positional-only in Python 3.15.\n" -""); +"Creates a new function."); #define PYSQLITE_CONNECTION_CREATE_FUNCTION_METHODDEF \ {"create_function", _PyCFunction_CAST(pysqlite_connection_create_function), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_function__doc__}, @@ -443,24 +412,13 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, int narg, PyObject *func, int deterministic); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.create_function'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.create_function'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.create_function'." -# endif -#endif - static PyObject * pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -469,7 +427,7 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(narg), &_Py_ID(func), &_Py_ID(deterministic), }, + .ob_item = { &_Py_ID(deterministic), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -478,7 +436,7 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "narg", "func", "deterministic", NULL}; + static const char * const _keywords[] = {"", "", "", "deterministic", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "create_function", @@ -497,18 +455,8 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject if (!args) { goto exit; } - if (nargs < 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword arguments 'name', 'narg' and 'func' to " - "_sqlite3.Connection.create_function() is deprecated. Parameters " - "'name', 'narg' and 'func' will become positional-only in Python " - "3.15.", 1)) - { - goto exit; - } - } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_function", "argument 'name'", "str", args[0]); + _PyArg_BadArgument("create_function", "argument 1", "str", args[0]); goto exit; } Py_ssize_t name_length; @@ -618,16 +566,10 @@ exit: #endif /* defined(HAVE_WINDOW_FUNCTIONS) */ PyDoc_STRVAR(pysqlite_connection_create_aggregate__doc__, -"create_aggregate($self, /, name, n_arg, aggregate_class)\n" +"create_aggregate($self, name, n_arg, aggregate_class, /)\n" "--\n" "\n" -"Creates a new aggregate.\n" -"\n" -"Note: Passing keyword arguments \'name\', \'n_arg\' and \'aggregate_class\'\n" -"to _sqlite3.Connection.create_aggregate() is deprecated. Parameters\n" -"\'name\', \'n_arg\' and \'aggregate_class\' will become positional-only in\n" -"Python 3.15.\n" -""); +"Creates a new aggregate."); #define PYSQLITE_CONNECTION_CREATE_AGGREGATE_METHODDEF \ {"create_aggregate", _PyCFunction_CAST(pysqlite_connection_create_aggregate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_aggregate__doc__}, @@ -638,42 +580,17 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, const char *name, int n_arg, PyObject *aggregate_class); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.create_aggregate'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.create_aggregate'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.create_aggregate'." -# endif -#endif - static PyObject * pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(n_arg), &_Py_ID(aggregate_class), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"name", "n_arg", "aggregate_class", NULL}; + static const char * const _keywords[] = {"", "", "", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "create_aggregate", @@ -690,18 +607,8 @@ pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject if (!args) { goto exit; } - if (nargs < 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' " - "to _sqlite3.Connection.create_aggregate() is deprecated. " - "Parameters 'name', 'n_arg' and 'aggregate_class' will become " - "positional-only in Python 3.15.", 1)) - { - goto exit; - } - } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_aggregate", "argument 'name'", "str", args[0]); + _PyArg_BadArgument("create_aggregate", "argument 1", "str", args[0]); goto exit; } Py_ssize_t name_length; @@ -725,15 +632,10 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_authorizer__doc__, -"set_authorizer($self, /, authorizer_callback)\n" +"set_authorizer($self, authorizer_callback, /)\n" "--\n" "\n" -"Set authorizer callback.\n" -"\n" -"Note: Passing keyword argument \'authorizer_callback\' to\n" -"_sqlite3.Connection.set_authorizer() is deprecated. Parameter\n" -"\'authorizer_callback\' will become positional-only in Python 3.15.\n" -""); +"Set authorizer callback."); #define PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF \ {"set_authorizer", _PyCFunction_CAST(pysqlite_connection_set_authorizer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_authorizer__doc__}, @@ -743,42 +645,17 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_authorizer'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_authorizer'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_authorizer'." -# endif -#endif - static PyObject * pysqlite_connection_set_authorizer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(authorizer_callback), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"authorizer_callback", NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_authorizer", @@ -793,16 +670,6 @@ pysqlite_connection_set_authorizer(PyObject *self, PyTypeObject *cls, PyObject * if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'authorizer_callback' to " - "_sqlite3.Connection.set_authorizer() is deprecated. Parameter " - "'authorizer_callback' will become positional-only in Python " - "3.15.", 1)) - { - goto exit; - } - } callable = args[0]; return_value = pysqlite_connection_set_authorizer_impl((pysqlite_Connection *)self, cls, callable); @@ -811,7 +678,7 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, -"set_progress_handler($self, /, progress_handler, n)\n" +"set_progress_handler($self, progress_handler, /, n)\n" "--\n" "\n" "Set progress handler callback.\n" @@ -824,12 +691,7 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled.\n" -"\n" -"Note: Passing keyword argument \'progress_handler\' to\n" -"_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n" -"\'progress_handler\' will become positional-only in Python 3.15.\n" -""); +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled."); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ {"set_progress_handler", _PyCFunction_CAST(pysqlite_connection_set_progress_handler), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, @@ -839,24 +701,13 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_progress_handler'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." -# endif -#endif - static PyObject * pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -865,7 +716,7 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), }, + .ob_item = { _Py_LATIN1_CHR('n'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -874,7 +725,7 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"progress_handler", "n", NULL}; + static const char * const _keywords[] = {"", "n", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_progress_handler", @@ -890,16 +741,6 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'progress_handler' to " - "_sqlite3.Connection.set_progress_handler() is deprecated. " - "Parameter 'progress_handler' will become positional-only in " - "Python 3.15.", 1)) - { - goto exit; - } - } callable = args[0]; n = PyLong_AsInt(args[1]); if (n == -1 && PyErr_Occurred()) { @@ -912,15 +753,10 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_trace_callback__doc__, -"set_trace_callback($self, /, trace_callback)\n" +"set_trace_callback($self, trace_callback, /)\n" "--\n" "\n" -"Set a trace callback called for each SQL statement (passed as unicode).\n" -"\n" -"Note: Passing keyword argument \'trace_callback\' to\n" -"_sqlite3.Connection.set_trace_callback() is deprecated. Parameter\n" -"\'trace_callback\' will become positional-only in Python 3.15.\n" -""); +"Set a trace callback called for each SQL statement (passed as unicode)."); #define PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF \ {"set_trace_callback", _PyCFunction_CAST(pysqlite_connection_set_trace_callback), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_trace_callback__doc__}, @@ -930,42 +766,17 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_trace_callback'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." -# endif -#endif - static PyObject * pysqlite_connection_set_trace_callback(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(trace_callback), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"trace_callback", NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_trace_callback", @@ -980,16 +791,6 @@ pysqlite_connection_set_trace_callback(PyObject *self, PyTypeObject *cls, PyObje if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'trace_callback' to " - "_sqlite3.Connection.set_trace_callback() is deprecated. " - "Parameter 'trace_callback' will become positional-only in Python" - " 3.15.", 1)) - { - goto exit; - } - } callable = args[0]; return_value = pysqlite_connection_set_trace_callback_impl((pysqlite_Connection *)self, cls, callable); @@ -1921,4 +1722,4 @@ exit: #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=2f325c2444b4bb47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6cb96e557133d553 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 2a184f78754..16ec6efc850 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -215,7 +215,7 @@ class sqlite3_int64_converter(CConverter): _sqlite3.Connection.__init__ as pysqlite_connection_init database: object - * [from 3.15] + * timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" @@ -223,7 +223,6 @@ _sqlite3.Connection.__init__ as pysqlite_connection_init factory: object(c_default='(PyObject*)clinic_state()->ConnectionType') = ConnectionType cached_statements as cache_size: int = 128 uri: bool = False - * autocommit: Autocommit(c_default='LEGACY_TRANSACTION_CONTROL') = sqlite3.LEGACY_TRANSACTION_CONTROL [clinic start generated code]*/ @@ -234,7 +233,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=219c3dbecbae7d99]*/ +/*[clinic end generated code: output=cba057313ea7712f input=5ca4883d8747a49b]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; @@ -1158,11 +1157,10 @@ check_num_params(pysqlite_Connection *self, const int n, const char *name) _sqlite3.Connection.create_function as pysqlite_connection_create_function cls: defining_class - / name: str narg: int func: object - / [from 3.15] + / * deterministic: bool = False @@ -1174,7 +1172,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int narg, PyObject *func, int deterministic) -/*[clinic end generated code: output=8a811529287ad240 input=c7c313b0ca8b519e]*/ +/*[clinic end generated code: output=8a811529287ad240 input=a896096ed5390ae1]*/ { int rc; int flags = SQLITE_UTF8; @@ -1366,11 +1364,10 @@ create_window_function_impl(pysqlite_Connection *self, PyTypeObject *cls, _sqlite3.Connection.create_aggregate as pysqlite_connection_create_aggregate cls: defining_class - / name: str n_arg: int aggregate_class: object - / [from 3.15] + / Creates a new aggregate. [clinic start generated code]*/ @@ -1380,7 +1377,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int n_arg, PyObject *aggregate_class) -/*[clinic end generated code: output=1b02d0f0aec7ff96 input=8087056db6eae1cf]*/ +/*[clinic end generated code: output=1b02d0f0aec7ff96 input=aa2773f6a42f7e17]*/ { int rc; @@ -1531,7 +1528,7 @@ _sqlite3.Connection.set_authorizer as pysqlite_connection_set_authorizer cls: defining_class authorizer_callback as callable: object - / [from 3.15] + / Set authorizer callback. [clinic start generated code]*/ @@ -1540,7 +1537,7 @@ static PyObject * pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=75fa60114fc971c3 input=a52bd4937c588752]*/ +/*[clinic end generated code: output=75fa60114fc971c3 input=e76469ab0bb1bbcd]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1576,7 +1573,7 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han A callable that takes no arguments. If the callable returns non-zero, the current query is terminated, and an exception is raised. - / [from 3.15] + / n: int The number of SQLite virtual machine instructions that are executed between invocations of 'progress_handler'. @@ -1590,7 +1587,7 @@ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=b4d6e2ef8b4d32f9]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=74c943f1ae7d8880]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1617,7 +1614,7 @@ _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback cls: defining_class trace_callback as callable: object - / [from 3.15] + / Set a trace callback called for each SQL statement (passed as unicode). [clinic start generated code]*/ @@ -1626,7 +1623,7 @@ static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=d91048c03bfcee05 input=d705d592ec03cf28]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=f4f59bf2f87f2026]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 27e8dab92e0..909ddd1f990 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -60,26 +60,16 @@ pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, pysqlite_state *state = pysqlite_get_state(module); PyObject *factory = (PyObject *)state->ConnectionType; - static const int FACTORY_POS = 5; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - if (nargs > 1 && nargs <= 8) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to sqlite3.connect()" - " is deprecated. Parameters 'timeout', 'detect_types', " - "'isolation_level', 'check_same_thread', 'factory', " - "'cached_statements' and 'uri' will become keyword-only " - "parameters in Python 3.15.", 1)) - { - return NULL; - } - } - if (nargs > FACTORY_POS) { - factory = args[FACTORY_POS]; + if (nargs > 1) { + PyErr_Format(PyExc_TypeError, + "connect() takes at most 1 positional arguments (%zd given)", nargs); + return NULL; } - else if (kwnames != NULL) { + if (kwnames != NULL) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwnames); i++) { PyObject *item = PyTuple_GET_ITEM(kwnames, i); // borrowed ref. - if (PyUnicode_CompareWithASCIIString(item, "factory") == 0) { + if (PyUnicode_EqualToUTF8(item, "factory")) { factory = args[nargs + i]; break; } diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 97a29f4d0e1..976da1340ec 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4427,7 +4427,7 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) FILE *f; DH *dh; -#if defined(MS_WINDOWS) && defined(_DEBUG) +#if defined(MS_WINDOWS) && defined(Py_DEBUG) PyErr_SetString(PyExc_NotImplementedError, "load_dh_params: unavailable on Windows debug build"); return NULL; @@ -6626,6 +6626,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_PSK", 1); #endif +#ifdef OPENSSL_NO_EXTERNAL_PSK_TLS13 + addbool(m, "HAS_PSK_TLS13", 0); +#else + addbool(m, "HAS_PSK_TLS13", 1); +#endif + #ifdef SSL_VERIFY_POST_HANDSHAKE addbool(m, "HAS_PHA", 1); #else diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 7c0b4876f43..f0a0a1674f3 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -175,7 +175,7 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, PySSLContext *self = PySSLContext_CAST(op); FILE *fp; -#if defined(MS_WINDOWS) && defined(_DEBUG) +#if defined(MS_WINDOWS) && defined(Py_DEBUG) PyErr_SetString(PyExc_NotImplementedError, "set_keylog_filename: unavailable on Windows debug build"); return -1; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ae060c95fd5..3030f45d72c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1165,6 +1165,47 @@ error: return NULL; } +static PyObject * +verify_stateless_code(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *codearg; + PyObject *globalnames = NULL; + PyObject *globalsns = NULL; + PyObject *builtinsns = NULL; + static char *kwlist[] = {"code", "globalnames", + "globalsns", "builtinsns", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O|O!O!O!:get_code_var_counts", kwlist, + &codearg, &PySet_Type, &globalnames, + &PyDict_Type, &globalsns, &PyDict_Type, &builtinsns)) + { + return NULL; + } + if (PyFunction_Check(codearg)) { + if (globalsns == NULL) { + globalsns = PyFunction_GET_GLOBALS(codearg); + } + if (builtinsns == NULL) { + builtinsns = PyFunction_GET_BUILTINS(codearg); + } + codearg = PyFunction_GET_CODE(codearg); + } + else if (!PyCode_Check(codearg)) { + PyErr_SetString(PyExc_TypeError, + "argument must be a code object or a function"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)codearg; + + if (_PyCode_VerifyStateless( + tstate, code, globalnames, globalsns, builtinsns) < 0) + { + return NULL; + } + Py_RETURN_NONE; +} + #ifdef _Py_TIER2 static PyObject * @@ -1948,6 +1989,16 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) goto error; } } + else if (strcmp(mode, "script") == 0) { + if (_PyCode_GetScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script-pure") == 0) { + if (_PyCode_GetPureScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } else { PyErr_Format(PyExc_ValueError, "unsupported mode %R", modeobj); goto error; @@ -2292,6 +2343,8 @@ static PyMethodDef module_functions[] = { {"get_co_localskinds", get_co_localskinds, METH_O, NULL}, {"get_code_var_counts", _PyCFunction_CAST(get_code_var_counts), METH_VARARGS | METH_KEYWORDS, NULL}, + {"verify_stateless_code", _PyCFunction_CAST(verify_stateless_code), + METH_VARARGS | METH_KEYWORDS, NULL}, #ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, diff --git a/Modules/_testlimitedcapi/import.c b/Modules/_testlimitedcapi/import.c index 3707dbedeea..f85daee57d7 100644 --- a/Modules/_testlimitedcapi/import.c +++ b/Modules/_testlimitedcapi/import.c @@ -108,20 +108,19 @@ pyimport_importmodule(PyObject *Py_UNUSED(module), PyObject *args) } -/* Test PyImport_ImportModuleNoBlock() */ +/* Test PyImport_ImportModuleNoBlock() (removed in 3.15) */ static PyObject * pyimport_importmodulenoblock(PyObject *Py_UNUSED(module), PyObject *args) { + // Get the function from the stable ABI. + PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock(const char *name); + const char *name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "z#", &name, &size)) { return NULL; } - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS return PyImport_ImportModuleNoBlock(name); - _Py_COMP_DIAG_POP } diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4d046859a15..c3852fe8973 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -7,7 +7,13 @@ Python module. # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" + +#include <zstd.h> // ZSTD_*() +#include <zdict.h> // ZDICT_*() /*[clinic input] module _zstd @@ -33,9 +39,6 @@ set_zstd_error(const _zstd_state* const state, case ERR_COMPRESS: msg = "Unable to compress zstd data: %s"; break; - case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "Unable to set pledged uncompressed content size: %s"; - break; case ERR_LOAD_D_DICT: msg = "Unable to load zstd dictionary or prefix for decompression: %s"; @@ -72,8 +75,7 @@ typedef struct { char parameter_name[32]; } ParameterInfo; -static const ParameterInfo cp_list[] = -{ +static const ParameterInfo cp_list[] = { {ZSTD_c_compressionLevel, "compression_level"}, {ZSTD_c_windowLog, "window_log"}, {ZSTD_c_hashLog, "hash_log"}, @@ -98,8 +100,7 @@ static const ParameterInfo cp_list[] = {ZSTD_c_overlapLog, "overlap_log"} }; -static const ParameterInfo dp_list[] = -{ +static const ParameterInfo dp_list[] = { {ZSTD_d_windowLogMax, "window_log_max"} }; @@ -151,8 +152,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } if (ZSTD_isError(bounds.error)) { PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid. (zstd v%s)", - type, name, ZSTD_versionString()); + "Zstd %s parameter \"%s\" is invalid.", + type, name); return; } @@ -160,10 +161,10 @@ set_parameter_error(const _zstd_state* const state, int is_compress, PyErr_Format(state->ZstdError, "Error when setting zstd %s parameter \"%s\", it " "should %d <= value <= %d, provided value is %d. " - "(zstd v%s, %d-bit build)", + "(%d-bit build)", type, name, bounds.lowerBound, bounds.upperBound, value_v, - ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t)); + 8*(int)sizeof(Py_ssize_t)); } static inline _zstd_state* @@ -176,7 +177,7 @@ get_zstd_state(PyObject *module) /*[clinic input] -_zstd._train_dict +_zstd.train_dict samples_bytes: PyBytesObject Concatenation of samples. @@ -190,9 +191,9 @@ Internal function, train a zstd dictionary on sample data. [clinic start generated code]*/ static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/ +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size) +/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/ { // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict // are pretty similar. We should see if we can refactor them to share that code. @@ -280,7 +281,7 @@ success: } /*[clinic input] -_zstd._finalize_dict +_zstd.finalize_dict custom_dict_bytes: PyBytesObject Custom dictionary content. @@ -298,11 +299,11 @@ Internal function, finalize a zstd dictionary. [clinic start generated code]*/ static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level) -/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/ +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level) +/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; @@ -399,7 +400,7 @@ success: /*[clinic input] -_zstd._get_param_bounds +_zstd.get_param_bounds parameter: int The parameter to get bounds. @@ -410,9 +411,8 @@ Internal function, get CompressionParameter/DecompressionParameter bounds. [clinic start generated code]*/ static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress) -/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/ +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) +/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/ { ZSTD_bounds bound; if (is_compress) { @@ -469,7 +469,7 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._get_frame_info +_zstd.get_frame_info frame_buffer: Py_buffer A bytes-like object, containing the header of a zstd frame. @@ -478,8 +478,8 @@ Internal function, get zstd frame infomation from a frame header. [clinic start generated code]*/ static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/ +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) +/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/ { uint64_t decompressed_size; uint32_t dict_id; @@ -511,7 +511,7 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._set_parameter_types +_zstd.set_parameter_types c_parameter_type: object(subclass_of='&PyType_Type') CompressionParameter IntEnum type object @@ -522,9 +522,9 @@ Internal function, set CompressionParameter/DecompressionParameter types for val [clinic start generated code]*/ static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type) -/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/ +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type) +/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/ { _zstd_state* const mod_state = get_zstd_state(module); @@ -547,289 +547,144 @@ _zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, } static PyMethodDef _zstd_methods[] = { - _ZSTD__TRAIN_DICT_METHODDEF - _ZSTD__FINALIZE_DICT_METHODDEF - _ZSTD__GET_PARAM_BOUNDS_METHODDEF + _ZSTD_TRAIN_DICT_METHODDEF + _ZSTD_FINALIZE_DICT_METHODDEF + _ZSTD_GET_PARAM_BOUNDS_METHODDEF _ZSTD_GET_FRAME_SIZE_METHODDEF - _ZSTD__GET_FRAME_INFO_METHODDEF - _ZSTD__SET_PARAMETER_TYPES_METHODDEF - - {0} + _ZSTD_GET_FRAME_INFO_METHODDEF + _ZSTD_SET_PARAMETER_TYPES_METHODDEF + {NULL, NULL} }; - -#define ADD_INT_PREFIX_MACRO(module, macro) \ - do { \ - if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \ - return -1; \ - } \ - } while(0) - static int -add_parameters(PyObject *module) -{ - /* If add new parameters, please also add to cp_list/dp_list above. */ - - /* Compression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog); - - /* Decompression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax); - - /* ZSTD_strategy enum */ - ADD_INT_PREFIX_MACRO(module, ZSTD_fast); - ADD_INT_PREFIX_MACRO(module, ZSTD_dfast); - ADD_INT_PREFIX_MACRO(module, ZSTD_greedy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btopt); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2); - - return 0; -} - -static inline PyObject * -get_zstd_version_info(void) +_zstd_exec(PyObject *m) { - uint32_t ver = ZSTD_versionNumber(); - uint32_t major, minor, release; - - major = ver / 10000; - minor = (ver / 100) % 100; - release = ver % 100; - - return Py_BuildValue("III", major, minor, release); -} - -static inline int -add_vars_to_module(PyObject *module) -{ - PyObject *obj; - - /* zstd_version, a str. */ - if (PyModule_AddStringConstant(module, "zstd_version", - ZSTD_versionString()) < 0) { - return -1; - } - - /* zstd_version_info, a tuple. */ - obj = get_zstd_version_info(); - if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* Add zstd parameters */ - if (add_parameters(module) < 0) { - return -1; - } - - /* _compressionLevel_values: (default, min, max) - ZSTD_defaultCLevel() was added in zstd v1.5.0 */ - obj = Py_BuildValue("iii", -#if ZSTD_VERSION_NUMBER < 10500 - ZSTD_CLEVEL_DEFAULT, -#else - ZSTD_defaultCLevel(), -#endif - ZSTD_minCLevel(), - ZSTD_maxCLevel()); - if (PyModule_AddObjectRef(module, - "_compressionLevel_values", - obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_CStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_CStreamInSize(), - (uint32_t)ZSTD_CStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_DStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_DStreamInSize(), - (uint32_t)ZSTD_DStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_CONFIG */ - obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", - Py_False, - Py_True, -/* User mremap output buffer */ -#if defined(HAVE_MREMAP) - Py_True -#else - Py_False -#endif - ); - if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - return 0; -} - -#define ADD_STR_TO_STATE_MACRO(STR) \ - do { \ - mod_state->str_##STR = PyUnicode_FromString(#STR); \ - if (mod_state->str_##STR == NULL) { \ - return -1; \ - } \ - } while(0) - -static inline int -add_type_to_module(PyObject *module, const char *name, - PyType_Spec *type_spec, PyTypeObject **dest) -{ - PyObject *temp = PyType_FromModuleAndSpec(module, type_spec, NULL); - - if (PyModule_AddObjectRef(module, name, temp) < 0) { - Py_XDECREF(temp); - return -1; - } - - *dest = (PyTypeObject*) temp; - - return 0; -} - -static inline int -add_constant_to_type(PyTypeObject *type, const char *name, long value) -{ - PyObject *temp; - - temp = PyLong_FromLong(value); - if (temp == NULL) { - return -1; - } - - int rc = PyObject_SetAttrString((PyObject*) type, name, temp); - Py_DECREF(temp); - return rc; -} - -static int _zstd_exec(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); +#define ADD_TYPE(TYPE, SPEC) \ +do { \ + TYPE = (PyTypeObject *)PyType_FromModuleAndSpec(m, &(SPEC), NULL); \ + if (TYPE == NULL) { \ + return -1; \ + } \ + if (PyModule_AddType(m, TYPE) < 0) { \ + return -1; \ + } \ +} while (0) + +#define ADD_INT_MACRO(MACRO) \ + if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \ + return -1; \ + } + +#define ADD_INT_CONST_TO_TYPE(TYPE, NAME, VALUE) \ +do { \ + PyObject *v = PyLong_FromLong((VALUE)); \ + if (v == NULL || PyObject_SetAttrString((PyObject *)(TYPE), \ + (NAME), v) < 0) { \ + Py_XDECREF(v); \ + return -1; \ + } \ + Py_DECREF(v); \ +} while (0) + + _zstd_state* const mod_state = get_zstd_state(m); /* Reusable objects & variables */ - mod_state->empty_bytes = PyBytes_FromStringAndSize(NULL, 0); - if (mod_state->empty_bytes == NULL) { - return -1; - } - - mod_state->empty_readonly_memoryview = - PyMemoryView_FromMemory((char*)mod_state, 0, PyBUF_READ); - if (mod_state->empty_readonly_memoryview == NULL) { - return -1; - } - - /* Add str to module state */ - ADD_STR_TO_STATE_MACRO(read); - ADD_STR_TO_STATE_MACRO(readinto); - ADD_STR_TO_STATE_MACRO(write); - ADD_STR_TO_STATE_MACRO(flush); - mod_state->CParameter_type = NULL; mod_state->DParameter_type = NULL; - /* Add variables to module */ - if (add_vars_to_module(module) < 0) { - return -1; - } - - /* ZstdError */ + /* Create and add heap types */ + ADD_TYPE(mod_state->ZstdDict_type, zstd_dict_type_spec); + ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec); + ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec); mod_state->ZstdError = PyErr_NewExceptionWithDoc( - "_zstd.ZstdError", - "Call to the underlying zstd library failed.", - NULL, NULL); + "compression.zstd.ZstdError", + "An error occurred in the zstd library.", + NULL, NULL); if (mod_state->ZstdError == NULL) { return -1; } - - if (PyModule_AddObjectRef(module, "ZstdError", mod_state->ZstdError) < 0) { + if (PyModule_AddType(m, (PyTypeObject *)mod_state->ZstdError) < 0) { Py_DECREF(mod_state->ZstdError); return -1; } - /* ZstdDict */ - if (add_type_to_module(module, - "ZstdDict", - &zstddict_type_spec, - &mod_state->ZstdDict_type) < 0) { + /* Add constants */ + if (PyModule_AddIntConstant(m, "zstd_version_number", + ZSTD_versionNumber()) < 0) { return -1; } - // ZstdCompressor - if (add_type_to_module(module, - "ZstdCompressor", - &zstdcompressor_type_spec, - &mod_state->ZstdCompressor_type) < 0) { - return -1; - } - - // Add EndDirective enum to ZstdCompressor - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "CONTINUE", - ZSTD_e_continue) < 0) { + if (PyModule_AddStringConstant(m, "zstd_version", + ZSTD_versionString()) < 0) { return -1; } - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_BLOCK", - ZSTD_e_flush) < 0) { +#if ZSTD_VERSION_NUMBER >= 10500 + if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT", + ZSTD_defaultCLevel()) < 0) { return -1; } +#else + ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT); +#endif - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_FRAME", - ZSTD_e_end) < 0) { + if (PyModule_Add(m, "ZSTD_DStreamOutSize", + PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) { return -1; } - // ZstdDecompressor - if (add_type_to_module(module, - "ZstdDecompressor", - &ZstdDecompressor_type_spec, - &mod_state->ZstdDecompressor_type) < 0) { - return -1; - } + /* Add zstd compression parameters. All should also be in cp_list. */ + ADD_INT_MACRO(ZSTD_c_compressionLevel); + ADD_INT_MACRO(ZSTD_c_windowLog); + ADD_INT_MACRO(ZSTD_c_hashLog); + ADD_INT_MACRO(ZSTD_c_chainLog); + ADD_INT_MACRO(ZSTD_c_searchLog); + ADD_INT_MACRO(ZSTD_c_minMatch); + ADD_INT_MACRO(ZSTD_c_targetLength); + ADD_INT_MACRO(ZSTD_c_strategy); + + ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching); + ADD_INT_MACRO(ZSTD_c_ldmHashLog); + ADD_INT_MACRO(ZSTD_c_ldmMinMatch); + ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog); + ADD_INT_MACRO(ZSTD_c_ldmHashRateLog); + + ADD_INT_MACRO(ZSTD_c_contentSizeFlag); + ADD_INT_MACRO(ZSTD_c_checksumFlag); + ADD_INT_MACRO(ZSTD_c_dictIDFlag); + + ADD_INT_MACRO(ZSTD_c_nbWorkers); + ADD_INT_MACRO(ZSTD_c_jobSize); + ADD_INT_MACRO(ZSTD_c_overlapLog); + + /* Add zstd decompression parameters. All should also be in dp_list. */ + ADD_INT_MACRO(ZSTD_d_windowLogMax); + + /* Add ZSTD_strategy enum members */ + ADD_INT_MACRO(ZSTD_fast); + ADD_INT_MACRO(ZSTD_dfast); + ADD_INT_MACRO(ZSTD_greedy); + ADD_INT_MACRO(ZSTD_lazy); + ADD_INT_MACRO(ZSTD_lazy2); + ADD_INT_MACRO(ZSTD_btlazy2); + ADD_INT_MACRO(ZSTD_btopt); + ADD_INT_MACRO(ZSTD_btultra); + ADD_INT_MACRO(ZSTD_btultra2); + + /* Add ZSTD_EndDirective enum members to ZstdCompressor */ + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "CONTINUE", ZSTD_e_continue); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_BLOCK", ZSTD_e_flush); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_FRAME", ZSTD_e_end); + + /* Make ZstdCompressor immutable (set Py_TPFLAGS_IMMUTABLETYPE) */ + PyType_Freeze(mod_state->ZstdCompressor_type); + +#undef ADD_TYPE +#undef ADD_INT_MACRO +#undef ADD_ZSTD_COMPRESSOR_INT_CONST return 0; } @@ -839,13 +694,6 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg) { _zstd_state* const mod_state = get_zstd_state(module); - Py_VISIT(mod_state->empty_bytes); - Py_VISIT(mod_state->empty_readonly_memoryview); - Py_VISIT(mod_state->str_read); - Py_VISIT(mod_state->str_readinto); - Py_VISIT(mod_state->str_write); - Py_VISIT(mod_state->str_flush); - Py_VISIT(mod_state->ZstdDict_type); Py_VISIT(mod_state->ZstdCompressor_type); @@ -863,13 +711,6 @@ _zstd_clear(PyObject *module) { _zstd_state* const mod_state = get_zstd_state(module); - Py_CLEAR(mod_state->empty_bytes); - Py_CLEAR(mod_state->empty_readonly_memoryview); - Py_CLEAR(mod_state->str_read); - Py_CLEAR(mod_state->str_readinto); - Py_CLEAR(mod_state->str_write); - Py_CLEAR(mod_state->str_flush); - Py_CLEAR(mod_state->ZstdDict_type); Py_CLEAR(mod_state->ZstdCompressor_type); @@ -890,20 +731,21 @@ _zstd_free(void *module) static struct PyModuleDef_Slot _zstd_slots[] = { {Py_mod_exec, _zstd_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0} + {0, NULL}, }; -struct PyModuleDef _zstdmodule = { - PyModuleDef_HEAD_INIT, +static struct PyModuleDef _zstdmodule = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zstd", + .m_doc = "Implementation module for Zstandard compression.", .m_size = sizeof(_zstd_state), .m_slots = _zstd_slots, .m_methods = _zstd_methods, .m_traverse = _zstd_traverse, .m_clear = _zstd_clear, - .m_free = _zstd_free + .m_free = _zstd_free, }; PyMODINIT_FUNC diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 9322ee259c5..00e0d2177f3 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,4 +1,3 @@ -#pragma once /* Low level interface to Meta's zstd library for use in the compression.zstd Python module. @@ -6,138 +5,29 @@ Python module. /* Declarations shared between different parts of the _zstd module*/ -#include "Python.h" +#ifndef ZSTD_MODULE_H +#define ZSTD_MODULE_H -#include "zstd.h" -#include "zdict.h" - - -/* Forward declaration of module state */ -typedef struct _zstd_state _zstd_state; - -/* Forward reference of module def */ -extern PyModuleDef _zstdmodule; - -/* For clinic type calculations */ -static inline _zstd_state * -get_zstd_state_from_type(PyTypeObject *type) { - PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule); - if (module == NULL) { - return NULL; - } - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_zstd_state *)state; -} - -extern PyType_Spec zstddict_type_spec; -extern PyType_Spec zstdcompressor_type_spec; -extern PyType_Spec ZstdDecompressor_type_spec; - -struct _zstd_state { - PyObject *empty_bytes; - PyObject *empty_readonly_memoryview; - PyObject *str_read; - PyObject *str_readinto; - PyObject *str_write; - PyObject *str_flush; +/* Type specs */ +extern PyType_Spec zstd_dict_type_spec; +extern PyType_Spec zstd_compressor_type_spec; +extern PyType_Spec zstd_decompressor_type_spec; +typedef struct { + /* Module heap types. */ PyTypeObject *ZstdDict_type; PyTypeObject *ZstdCompressor_type; PyTypeObject *ZstdDecompressor_type; PyObject *ZstdError; + /* enum types set by set_parameter_types. */ PyTypeObject *CParameter_type; PyTypeObject *DParameter_type; -}; - -typedef struct { - PyObject_HEAD - - /* Reusable compress/decompress dictionary, they are created once and - can be shared by multiple threads concurrently, since its usage is - read-only. - c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ - ZSTD_DDict *d_dict; - PyObject *c_dicts; - - /* Content of the dictionary, bytes object. */ - PyObject *dict_content; - /* Dictionary id */ - uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDict; - -typedef struct { - PyObject_HEAD - - /* Compression context */ - ZSTD_CCtx *cctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Last mode, initialized to ZSTD_e_end */ - int last_mode; - - /* (nbWorker >= 1) ? 1 : 0 */ - int use_multithread; - - /* Compression level */ - int compression_level; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdCompressor; - -typedef struct { - PyObject_HEAD - - /* Decompression context */ - ZSTD_DCtx *dctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Unconsumed input data */ - char *input_buffer; - size_t input_buffer_size; - size_t in_begin, in_end; - - /* Unused data */ - PyObject *unused_data; - - /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ - char needs_input; - - /* For decompress(), 0 or 1. - 1 when both input and output streams are at a frame edge, means a - frame is completely decoded and fully flushed, or the decompressor - just be initialized. */ - char at_frame_edge; - - /* For ZstdDecompressor, 0 or 1. - 1 means the end of the first frame has been reached. */ - char eof; - - /* Used for fast reset above three variables */ - char _unused_char_for_align; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDecompressor; - -typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function -} decompress_type; +} _zstd_state; typedef enum { ERR_DECOMPRESS, ERR_COMPRESS, - ERR_SET_PLEDGED_INPUT_SIZE, ERR_LOAD_D_DICT, ERR_LOAD_C_DICT, @@ -147,7 +37,7 @@ typedef enum { ERR_SET_C_LEVEL, ERR_TRAIN_DICT, - ERR_FINALIZE_DICT + ERR_FINALIZE_DICT, } error_type; typedef enum { @@ -156,11 +46,6 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; -static inline int -mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) { - return in->size == in->pos && out->size != out->pos; -} - /* Format error message and set ZstdError. */ extern void set_zstd_error(const _zstd_state* const state, @@ -170,27 +55,4 @@ extern void set_parameter_error(const _zstd_state* const state, int is_compress, int key_v, int value_v); -static const char init_twice_msg[] = "__init__ method is called twice."; - -extern int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict); - -extern int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict); - -extern int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char *arg_type); - -extern int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options); - -extern PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length, - Py_ssize_t initial_size, - decompress_type type); - -extern PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive); +#endif // !ZSTD_MODULE_H diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 319b1214833..c902eef4f8e 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -3,9 +3,13 @@ Low level interface to Meta's zstd library for use in the compression.zstd Python module. */ -#include "_zstdmodule.h" +#ifndef ZSTD_BUFFER_H +#define ZSTD_BUFFER_H + #include "pycore_blocks_output_buffer.h" +#include <zstd.h> // ZSTD_outBuffer + /* Blocks output buffer wrapper code */ /* Initialize the buffer, and grow the buffer. @@ -102,3 +106,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } + +#endif // !ZSTD_BUFFER_H diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index 2f8225389b7..fc9f49813df 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -9,8 +9,8 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() -PyDoc_STRVAR(_zstd__train_dict__doc__, -"_train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" +PyDoc_STRVAR(_zstd_train_dict__doc__, +"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" "Internal function, train a zstd dictionary on sample data.\n" @@ -22,31 +22,31 @@ PyDoc_STRVAR(_zstd__train_dict__doc__, " dict_size\n" " The size of the dictionary."); -#define _ZSTD__TRAIN_DICT_METHODDEF \ - {"_train_dict", _PyCFunction_CAST(_zstd__train_dict), METH_FASTCALL, _zstd__train_dict__doc__}, +#define _ZSTD_TRAIN_DICT_METHODDEF \ + {"train_dict", _PyCFunction_CAST(_zstd_train_dict), METH_FASTCALL, _zstd_train_dict__doc__}, static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size); +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size); static PyObject * -_zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *samples_bytes; PyObject *samples_sizes; Py_ssize_t dict_size; - if (!_PyArg_CheckPositional("_train_dict", nargs, 3, 3)) { + if (!_PyArg_CheckPositional("train_dict", nargs, 3, 3)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_train_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("train_dict", "argument 1", "bytes", args[0]); goto exit; } samples_bytes = (PyBytesObject *)args[0]; if (!PyTuple_Check(args[1])) { - _PyArg_BadArgument("_train_dict", "argument 2", "tuple", args[1]); + _PyArg_BadArgument("train_dict", "argument 2", "tuple", args[1]); goto exit; } samples_sizes = args[1]; @@ -62,15 +62,15 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } dict_size = ival; } - return_value = _zstd__train_dict_impl(module, samples_bytes, samples_sizes, dict_size); + return_value = _zstd_train_dict_impl(module, samples_bytes, samples_sizes, dict_size); exit: return return_value; } -PyDoc_STRVAR(_zstd__finalize_dict__doc__, -"_finalize_dict($module, custom_dict_bytes, samples_bytes,\n" -" samples_sizes, dict_size, compression_level, /)\n" +PyDoc_STRVAR(_zstd_finalize_dict__doc__, +"finalize_dict($module, custom_dict_bytes, samples_bytes, samples_sizes,\n" +" dict_size, compression_level, /)\n" "--\n" "\n" "Internal function, finalize a zstd dictionary.\n" @@ -86,17 +86,17 @@ PyDoc_STRVAR(_zstd__finalize_dict__doc__, " compression_level\n" " Optimize for a specific zstd compression level, 0 means default."); -#define _ZSTD__FINALIZE_DICT_METHODDEF \ - {"_finalize_dict", _PyCFunction_CAST(_zstd__finalize_dict), METH_FASTCALL, _zstd__finalize_dict__doc__}, +#define _ZSTD_FINALIZE_DICT_METHODDEF \ + {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level); +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level); static PyObject * -_zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *custom_dict_bytes; @@ -105,21 +105,21 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t dict_size; int compression_level; - if (!_PyArg_CheckPositional("_finalize_dict", nargs, 5, 5)) { + if (!_PyArg_CheckPositional("finalize_dict", nargs, 5, 5)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_finalize_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("finalize_dict", "argument 1", "bytes", args[0]); goto exit; } custom_dict_bytes = (PyBytesObject *)args[0]; if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("_finalize_dict", "argument 2", "bytes", args[1]); + _PyArg_BadArgument("finalize_dict", "argument 2", "bytes", args[1]); goto exit; } samples_bytes = (PyBytesObject *)args[1]; if (!PyTuple_Check(args[2])) { - _PyArg_BadArgument("_finalize_dict", "argument 3", "tuple", args[2]); + _PyArg_BadArgument("finalize_dict", "argument 3", "tuple", args[2]); goto exit; } samples_sizes = args[2]; @@ -139,14 +139,14 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (compression_level == -1 && PyErr_Occurred()) { goto exit; } - return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); + return_value = _zstd_finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); exit: return return_value; } -PyDoc_STRVAR(_zstd__get_param_bounds__doc__, -"_get_param_bounds($module, /, parameter, is_compress)\n" +PyDoc_STRVAR(_zstd_get_param_bounds__doc__, +"get_param_bounds($module, /, parameter, is_compress)\n" "--\n" "\n" "Internal function, get CompressionParameter/DecompressionParameter bounds.\n" @@ -156,15 +156,14 @@ PyDoc_STRVAR(_zstd__get_param_bounds__doc__, " is_compress\n" " True for CompressionParameter, False for DecompressionParameter."); -#define _ZSTD__GET_PARAM_BOUNDS_METHODDEF \ - {"_get_param_bounds", _PyCFunction_CAST(_zstd__get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd__get_param_bounds__doc__}, +#define _ZSTD_GET_PARAM_BOUNDS_METHODDEF \ + {"get_param_bounds", _PyCFunction_CAST(_zstd_get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd_get_param_bounds__doc__}, static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress); +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress); static PyObject * -_zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -190,7 +189,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg static const char * const _keywords[] = {"parameter", "is_compress", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_param_bounds", + .fname = "get_param_bounds", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -211,7 +210,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg if (is_compress < 0) { goto exit; } - return_value = _zstd__get_param_bounds_impl(module, parameter, is_compress); + return_value = _zstd_get_param_bounds_impl(module, parameter, is_compress); exit: return return_value; @@ -288,8 +287,8 @@ exit: return return_value; } -PyDoc_STRVAR(_zstd__get_frame_info__doc__, -"_get_frame_info($module, /, frame_buffer)\n" +PyDoc_STRVAR(_zstd_get_frame_info__doc__, +"get_frame_info($module, /, frame_buffer)\n" "--\n" "\n" "Internal function, get zstd frame infomation from a frame header.\n" @@ -297,14 +296,14 @@ PyDoc_STRVAR(_zstd__get_frame_info__doc__, " frame_buffer\n" " A bytes-like object, containing the header of a zstd frame."); -#define _ZSTD__GET_FRAME_INFO_METHODDEF \ - {"_get_frame_info", _PyCFunction_CAST(_zstd__get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd__get_frame_info__doc__}, +#define _ZSTD_GET_FRAME_INFO_METHODDEF \ + {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); static PyObject * -_zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -330,7 +329,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"frame_buffer", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_frame_info", + .fname = "get_frame_info", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -345,7 +344,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (PyObject_GetBuffer(args[0], &frame_buffer, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _zstd__get_frame_info_impl(module, &frame_buffer); + return_value = _zstd_get_frame_info_impl(module, &frame_buffer); exit: /* Cleanup for frame_buffer */ @@ -356,8 +355,8 @@ exit: return return_value; } -PyDoc_STRVAR(_zstd__set_parameter_types__doc__, -"_set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" +PyDoc_STRVAR(_zstd_set_parameter_types__doc__, +"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "--\n" "\n" "Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" @@ -367,15 +366,15 @@ PyDoc_STRVAR(_zstd__set_parameter_types__doc__, " d_parameter_type\n" " DecompressionParameter IntEnum type object"); -#define _ZSTD__SET_PARAMETER_TYPES_METHODDEF \ - {"_set_parameter_types", _PyCFunction_CAST(_zstd__set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd__set_parameter_types__doc__}, +#define _ZSTD_SET_PARAMETER_TYPES_METHODDEF \ + {"set_parameter_types", _PyCFunction_CAST(_zstd_set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd_set_parameter_types__doc__}, static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type); +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type); static PyObject * -_zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -401,7 +400,7 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n static const char * const _keywords[] = {"c_parameter_type", "d_parameter_type", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_set_parameter_types", + .fname = "set_parameter_types", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -415,18 +414,18 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (!PyObject_TypeCheck(args[0], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); + _PyArg_BadArgument("set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); goto exit; } c_parameter_type = args[0]; if (!PyObject_TypeCheck(args[1], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); + _PyArg_BadArgument("set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); goto exit; } d_parameter_type = args[1]; - return_value = _zstd__set_parameter_types_impl(module, c_parameter_type, d_parameter_type); + return_value = _zstd_set_parameter_types_impl(module, c_parameter_type, d_parameter_type); exit: return return_value; } -/*[clinic end generated code: output=189c462236a7096c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8445b658dcdcbb9c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b735981e747..355a27d2734 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -7,27 +7,53 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type" +class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ - +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Compression context */ + ZSTD_CCtx *cctx; + /* ZstdDict object in use */ + PyObject *dict; + + /* Last mode, initialized to ZSTD_e_end */ + int last_mode; + + /* (nbWorker >= 1) ? 1 : 0 */ + int use_multithread; + + /* Compression level */ + int compression_level; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) -int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char* arg_type) +#include "clinic/compressor.c.h" + +static int +_zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, + const char *arg_name, const char* arg_type) { size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -197,8 +223,9 @@ success: return cdict; } -int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { +static int +_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) +{ size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -291,10 +318,6 @@ load: return 0; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/compressor.c.h" -#undef clinic_state - static PyObject * _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) { @@ -304,7 +327,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->use_multithread = 0; @@ -371,12 +394,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, PyObject *options, PyObject *zstd_dict) /*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ { - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; if (level != Py_None && options != Py_None) { PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); @@ -385,20 +407,20 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, /* Set compressLevel/options to compression context */ if (level != Py_None) { - if (_PyZstd_set_c_parameters(self, level, "level", "int") < 0) { + if (_zstd_set_c_parameters(self, level, "level", "int") < 0) { return -1; } } if (options != Py_None) { - if (_PyZstd_set_c_parameters(self, options, "options", "dict") < 0) { + if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) { return -1; } } /* Load dictionary to compression context */ if (zstd_dict != Py_None) { - if (_PyZstd_load_c_dict(self, zstd_dict) < 0) { + if (_zstd_load_c_dict(self, zstd_dict) < 0) { return -1; } @@ -412,7 +434,7 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, return 0; } -PyObject * +static PyObject * compress_impl(ZstdCompressor *self, Py_buffer *data, ZSTD_EndDirective end_directive) { @@ -487,6 +509,12 @@ error: return NULL; } +static inline int +mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) +{ + return in->size == in->pos && out->size != out->pos; +} + static PyObject * compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) { @@ -655,8 +683,7 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) static PyMethodDef ZstdCompressor_methods[] = { _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF - - {0} + {NULL, NULL} }; PyDoc_STRVAR(ZstdCompressor_last_mode_doc, @@ -668,7 +695,7 @@ PyDoc_STRVAR(ZstdCompressor_last_mode_doc, static PyMemberDef ZstdCompressor_members[] = { {"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode), Py_READONLY, ZstdCompressor_last_mode_doc}, - {0} + {NULL} }; static int @@ -696,12 +723,15 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_clear, ZstdCompressor_clear}, - {0} + {0, 0} }; -PyType_Spec zstdcompressor_type_spec = { - .name = "_zstd.ZstdCompressor", +PyType_Spec zstd_compressor_type_spec = { + .name = "compression.zstd.ZstdCompressor", .basicsize = sizeof(ZstdCompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + // Py_TPFLAGS_IMMUTABLETYPE is not used here as several + // associated constants need to be added to the type. + // PyType_Freeze is called later to set the flag. + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = zstdcompressor_slots, }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index a4be180c008..a654a9540e1 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -7,22 +7,65 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type" +class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Decompression context */ + ZSTD_DCtx *dctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Unconsumed input data */ + char *input_buffer; + size_t input_buffer_size; + size_t in_begin, in_end; + + /* Unused data */ + PyObject *unused_data; + + /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ + char needs_input; + + /* For decompress(), 0 or 1. + 1 when both input and output streams are at a frame edge, means a + frame is completely decoded and fully flushed, or the decompressor + just be initialized. */ + char at_frame_edge; + + /* For ZstdDecompressor, 0 or 1. + 1 means the end of the first frame has been reached. */ + char eof; + + /* Used for fast reset above three variables */ + char _unused_char_for_align; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) +#include "clinic/decompressor.c.h" + static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { @@ -61,8 +104,8 @@ _get_DDict(ZstdDict *self) } /* Set decompression parameters to decompression context */ -int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) +static int +_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) { size_t zstd_ret; PyObject *key, *value; @@ -120,8 +163,8 @@ _PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) } /* Load dictionary or prefix to decompression context */ -int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +static int +_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -215,17 +258,14 @@ load: return 0; } - +typedef enum { + TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class + TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function +} decompress_type; /* - Given the two types of decompressors (defined in _zstdmodule.h): - - typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function - } decompress_type; - - Decompress implementation for <D>, <E>, pseudo code: + Given the two types of decompressors (defined above), + decompress implementation for <D>, <E>, pseudo code: initialize_output_buffer while True: @@ -276,7 +316,7 @@ load: output stream: ====================| ^ */ -PyObject * +static PyObject * decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, Py_ssize_t max_length, Py_ssize_t initial_size, @@ -290,13 +330,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, /* The first AFE check for setting .at_frame_edge flag */ if (type == TYPE_ENDLESS_DECOMPRESSOR) { if (self->at_frame_edge && in->pos == in->size) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); - return ret; + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } } @@ -380,7 +414,7 @@ error: return NULL; } -void +static void decompressor_reset_session(ZstdDecompressor *self, decompress_type type) { @@ -405,7 +439,7 @@ decompressor_reset_session(ZstdDecompressor *self, ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only); } -PyObject * +static PyObject * stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length, decompress_type type) { @@ -622,7 +656,7 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; @@ -701,15 +735,15 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Load dictionary to decompression context */ if (zstd_dict != Py_None) { - if (_PyZstd_load_d_dict(self, zstd_dict) < 0) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { return -1; } @@ -720,7 +754,7 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /* Set option to decompression context */ if (options != Py_None) { - if (_PyZstd_set_d_parameters(self, options) < 0) { + if (_zstd_set_d_parameters(self, options) < 0) { return -1; } } @@ -747,16 +781,8 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) { PyObject *ret; - /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - if (!self->eof) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else { if (self->unused_data == NULL) { @@ -772,8 +798,6 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) } } - Py_END_CRITICAL_SECTION(); - return ret; } @@ -818,14 +842,9 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, return ret; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/decompressor.c.h" -#undef clinic_state - static PyMethodDef ZstdDecompressor_methods[] = { _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF - - {0} + {NULL, NULL} }; PyDoc_STRVAR(ZstdDecompressor_eof_doc, @@ -840,17 +859,14 @@ PyDoc_STRVAR(ZstdDecompressor_needs_input_doc, static PyMemberDef ZstdDecompressor_members[] = { {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof), Py_READONLY, ZstdDecompressor_eof_doc}, - {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input), Py_READONLY, ZstdDecompressor_needs_input_doc}, - - {0} + {NULL} }; static PyGetSetDef ZstdDecompressor_getset[] = { _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF - - {0} + {NULL} }; static int @@ -880,12 +896,13 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_clear, ZstdDecompressor_clear}, - {0} + {0, 0} }; -PyType_Spec ZstdDecompressor_type_spec = { - .name = "_zstd.ZstdDecompressor", +PyType_Spec zstd_decompressor_type_spec = { + .name = "compression.zstd.ZstdDecompressor", .basicsize = sizeof(ZstdDecompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = ZstdDecompressor_slots, }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index a19224c4a64..47bc8a84ca2 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -7,17 +7,21 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" +class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" +#include "clinic/zstddict.c.h" -#include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() #define ZstdDict_CAST(op) ((ZstdDict *)op) @@ -31,7 +35,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U } self->dict_content = NULL; - self->inited = 0; + self->initialized = 0; self->d_dict = NULL; /* ZSTD_CDict dict */ @@ -92,11 +96,11 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, /*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Check dict_content's type */ self->dict_content = PyBytes_FromObject(dict_content); @@ -135,10 +139,6 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, return 0; } -#define clinic_state() (get_zstd_state(type)) -#include "clinic/zstddict.c.h" -#undef clinic_state - PyDoc_STRVAR(ZstdDict_dictid_doc, "ID of zstd dictionary, a 32-bit unsigned int value.\n\n" "Non-zero means ordinary dictionary, was created by zstd functions, follow\n" @@ -161,7 +161,7 @@ ZstdDict_str(PyObject *ob) static PyMemberDef ZstdDict_members[] = { {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc}, {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc}, - {0} + {NULL} }; /*[clinic input] @@ -231,12 +231,9 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) static PyGetSetDef ZstdDict_getset[] = { _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF - - {0} + {NULL} }; static Py_ssize_t @@ -275,12 +272,13 @@ static PyType_Slot zstddict_slots[] = { {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, {Py_tp_clear, ZstdDict_clear}, - {0} + {0, 0} }; -PyType_Spec zstddict_type_spec = { - .name = "_zstd.ZstdDict", +PyType_Spec zstd_dict_type_spec = { + .name = "compression.zstd.ZstdDict", .basicsize = sizeof(ZstdDict), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = zstddict_slots, }; diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h new file mode 100644 index 00000000000..e82498f5dd1 --- /dev/null +++ b/Modules/_zstd/zstddict.h @@ -0,0 +1,31 @@ +/* +Low level interface to Meta's zstd library for use in the compression.zstd +Python module. +*/ + +#ifndef ZSTD_DICT_H +#define ZSTD_DICT_H + +#include <stdbool.h> // bool +#include <zstd.h> // ZSTD_DDict + +typedef struct { + PyObject_HEAD + + /* Reusable compress/decompress dictionary, they are created once and + can be shared by multiple threads concurrently, since its usage is + read-only. + c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ + ZSTD_DDict *d_dict; + PyObject *c_dicts; + + /* Content of the dictionary, bytes object. */ + PyObject *dict_content; + /* Dictionary id */ + uint32_t dict_id; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDict; + +#endif // !ZSTD_DICT_H diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 70ebbaa876b..573903be87e 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UInt16_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_socket_socket_close__doc__, @@ -369,4 +370,4 @@ exit: #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=c971b79d2193b426 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07776dd21d1e3b56 input=a9049054013a1b77]*/ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 47958379263..92c9aa8b510 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -111,7 +111,6 @@ Local naming conventions: #include "pycore_moduleobject.h" // _PyModule_GetState #include "pycore_time.h" // _PyTime_AsMilliseconds() #include "pycore_pystate.h" // _Py_AssertHoldsTstate() -#include "pycore_pyatomic_ft_wrappers.h" #ifdef _Py_MEMORY_SANITIZER # include <sanitizer/msan_interface.h> @@ -565,7 +564,6 @@ static int sock_cloexec_works = -1; static inline void set_sock_fd(PySocketSockObject *s, SOCKET_T fd) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT _Py_atomic_store_int_relaxed((int *)&s->sock_fd, (int)fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -575,15 +573,11 @@ set_sock_fd(PySocketSockObject *s, SOCKET_T fd) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - s->sock_fd = fd; -#endif } static inline SOCKET_T get_sock_fd(PySocketSockObject *s) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT return (SOCKET_T)_Py_atomic_load_int_relaxed((int *)&s->sock_fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -593,9 +587,6 @@ get_sock_fd(PySocketSockObject *s) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - return s->sock_fd; -#endif } #define _PySocketSockObject_CAST(op) ((PySocketSockObject *)(op)) @@ -638,33 +629,22 @@ _PyLong_##NAME##_Converter(PyObject *obj, void *ptr) \ return 1; \ } -UNSIGNED_INT_CONVERTER(UInt16, uint16_t) -UNSIGNED_INT_CONVERTER(UInt32, uint32_t) - #if defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) # ifdef MS_WINDOWS UNSIGNED_INT_CONVERTER(NetIfindex, NET_IFINDEX) # else - UNSIGNED_INT_CONVERTER(NetIfindex, unsigned int) +# define _PyLong_NetIfindex_Converter _PyLong_UnsignedInt_Converter # define NET_IFINDEX unsigned int # endif #endif // defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) /*[python input] -class uint16_converter(CConverter): - type = "uint16_t" - converter = '_PyLong_UInt16_Converter' - -class uint32_converter(CConverter): - type = "uint32_t" - converter = '_PyLong_UInt32_Converter' - class NET_IFINDEX_converter(CConverter): type = "NET_IFINDEX" converter = '_PyLong_NetIfindex_Converter' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3de2e4a03fbf83b8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=1cf809c40a407c34]*/ /*[clinic input] module _socket |