aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Modules
diff options
context:
space:
mode:
Diffstat (limited to 'Modules')
-rw-r--r--Modules/_ctypes/callbacks.c9
-rw-r--r--Modules/_ctypes/callproc.c8
-rw-r--r--Modules/_ctypes/ctypes.h14
-rw-r--r--Modules/_interpchannelsmodule.c176
-rw-r--r--Modules/_interpqueuesmodule.c175
-rw-r--r--Modules/_interpreters_common.h54
-rw-r--r--Modules/_interpretersmodule.c180
-rw-r--r--Modules/_localemodule.c12
-rw-r--r--Modules/_pickle.c4
-rw-r--r--Modules/_threadmodule.c24
-rw-r--r--Modules/_zstd/_zstdmodule.c118
-rw-r--r--Modules/_zstd/buffer.h9
-rw-r--r--Modules/_zstd/clinic/decompressor.c.h11
-rw-r--r--Modules/_zstd/clinic/zstddict.c.h39
-rw-r--r--Modules/_zstd/compressor.c205
-rw-r--r--Modules/_zstd/decompressor.c187
-rw-r--r--Modules/_zstd/zstddict.c39
-rw-r--r--Modules/_zstd/zstddict.h3
-rw-r--r--Modules/main.c16
-rw-r--r--Modules/pwdmodule.c25
20 files changed, 759 insertions, 549 deletions
diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c
index ec113e41d16..fd508ae61f2 100644
--- a/Modules/_ctypes/callbacks.c
+++ b/Modules/_ctypes/callbacks.c
@@ -11,18 +11,9 @@
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_runtime.h" // _Py_ID()
-#ifdef MS_WIN32
-# include <malloc.h>
-#endif
-
#include <ffi.h>
#include "ctypes.h"
-#ifdef HAVE_ALLOCA_H
-/* AIX needs alloca.h for alloca() */
-#include <alloca.h>
-#endif
-
/**************************************************************/
static int
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 9f5ad9f57b7..65a0f14e2b0 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -77,16 +77,8 @@ module _ctypes
#include <mach-o/dyld.h>
#endif
-#ifdef MS_WIN32
-#include <malloc.h>
-#endif
-
#include <ffi.h>
#include "ctypes.h"
-#ifdef HAVE_ALLOCA_H
-/* AIX needs alloca.h for alloca() */
-#include <alloca.h>
-#endif
#ifdef _Py_MEMORY_SANITIZER
#include <sanitizer/msan_interface.h>
diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
index 2d859ed63e1..6a45c11e61a 100644
--- a/Modules/_ctypes/ctypes.h
+++ b/Modules/_ctypes/ctypes.h
@@ -1,5 +1,17 @@
-#if defined (__SVR4) && defined (__sun)
+/* Get a definition of alloca(). */
+#if (defined (__SVR4) && defined (__sun)) || defined(HAVE_ALLOCA_H)
# include <alloca.h>
+#elif defined(MS_WIN32)
+# include <malloc.h>
+#endif
+
+/* If the system does not define alloca(), we have to hope for a compiler builtin. */
+#ifndef alloca
+# if defined __GNUC__ || (__clang_major__ >= 4)
+# define alloca __builtin_alloca
+# else
+# error "Could not define alloca() on your platform."
+# endif
#endif
#include <stdbool.h>
diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c
index f9fa1dab291..bfd805bf5e4 100644
--- a/Modules/_interpchannelsmodule.c
+++ b/Modules/_interpchannelsmodule.c
@@ -20,9 +20,11 @@
#endif
#define REGISTERS_HEAP_TYPES
+#define HAS_FALLBACK
#define HAS_UNBOUND_ITEMS
#include "_interpreters_common.h"
#undef HAS_UNBOUND_ITEMS
+#undef HAS_FALLBACK
#undef REGISTERS_HEAP_TYPES
@@ -523,7 +525,7 @@ typedef struct _channelitem {
int64_t interpid;
_PyXIData_t *data;
_waiting_t *waiting;
- int unboundop;
+ unboundop_t unboundop;
struct _channelitem *next;
} _channelitem;
@@ -536,7 +538,7 @@ _channelitem_ID(_channelitem *item)
static void
_channelitem_init(_channelitem *item,
int64_t interpid, _PyXIData_t *data,
- _waiting_t *waiting, int unboundop)
+ _waiting_t *waiting, unboundop_t unboundop)
{
if (interpid < 0) {
interpid = _get_interpid(data);
@@ -583,7 +585,7 @@ _channelitem_clear(_channelitem *item)
static _channelitem *
_channelitem_new(int64_t interpid, _PyXIData_t *data,
- _waiting_t *waiting, int unboundop)
+ _waiting_t *waiting, unboundop_t unboundop)
{
_channelitem *item = GLOBAL_MALLOC(_channelitem);
if (item == NULL) {
@@ -694,7 +696,7 @@ _channelqueue_free(_channelqueue *queue)
static int
_channelqueue_put(_channelqueue *queue,
int64_t interpid, _PyXIData_t *data,
- _waiting_t *waiting, int unboundop)
+ _waiting_t *waiting, unboundop_t unboundop)
{
_channelitem *item = _channelitem_new(interpid, data, waiting, unboundop);
if (item == NULL) {
@@ -798,7 +800,7 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid,
}
queue->count -= 1;
- int unboundop;
+ unboundop_t unboundop;
_channelitem_popped(item, p_data, p_waiting, &unboundop);
}
@@ -1083,16 +1085,18 @@ typedef struct _channel {
PyThread_type_lock mutex;
_channelqueue *queue;
_channelends *ends;
- struct {
- int unboundop;
+ struct _channeldefaults {
+ unboundop_t unboundop;
+ xidata_fallback_t fallback;
} defaults;
int open;
struct _channel_closing *closing;
} _channel_state;
static _channel_state *
-_channel_new(PyThread_type_lock mutex, int unboundop)
+_channel_new(PyThread_type_lock mutex, struct _channeldefaults defaults)
{
+ assert(check_unbound(defaults.unboundop));
_channel_state *chan = GLOBAL_MALLOC(_channel_state);
if (chan == NULL) {
return NULL;
@@ -1109,7 +1113,7 @@ _channel_new(PyThread_type_lock mutex, int unboundop)
GLOBAL_FREE(chan);
return NULL;
}
- chan->defaults.unboundop = unboundop;
+ chan->defaults = defaults;
chan->open = 1;
chan->closing = NULL;
return chan;
@@ -1130,7 +1134,7 @@ _channel_free(_channel_state *chan)
static int
_channel_add(_channel_state *chan, int64_t interpid,
- _PyXIData_t *data, _waiting_t *waiting, int unboundop)
+ _PyXIData_t *data, _waiting_t *waiting, unboundop_t unboundop)
{
int res = -1;
PyThread_acquire_lock(chan->mutex, WAIT_LOCK);
@@ -1611,7 +1615,7 @@ done:
struct channel_id_and_info {
int64_t id;
- int unboundop;
+ struct _channeldefaults defaults;
};
static struct channel_id_and_info *
@@ -1628,7 +1632,7 @@ _channels_list_all(_channels *channels, int64_t *count)
for (int64_t i=0; ref != NULL; ref = ref->next, i++) {
ids[i] = (struct channel_id_and_info){
.id = ref->cid,
- .unboundop = ref->chan->defaults.unboundop,
+ .defaults = ref->chan->defaults,
};
}
*count = channels->numopen;
@@ -1714,13 +1718,13 @@ _channel_finish_closing(_channel_state *chan) {
// Create a new channel.
static int64_t
-channel_create(_channels *channels, int unboundop)
+channel_create(_channels *channels, struct _channeldefaults defaults)
{
PyThread_type_lock mutex = PyThread_allocate_lock();
if (mutex == NULL) {
return ERR_CHANNEL_MUTEX_INIT;
}
- _channel_state *chan = _channel_new(mutex, unboundop);
+ _channel_state *chan = _channel_new(mutex, defaults);
if (chan == NULL) {
PyThread_free_lock(mutex);
return -1;
@@ -1752,7 +1756,7 @@ channel_destroy(_channels *channels, int64_t cid)
// Optionally request to be notified when it is received.
static int
channel_send(_channels *channels, int64_t cid, PyObject *obj,
- _waiting_t *waiting, int unboundop)
+ _waiting_t *waiting, unboundop_t unboundop, xidata_fallback_t fallback)
{
PyThreadState *tstate = _PyThreadState_GET();
PyInterpreterState *interp = tstate->interp;
@@ -1779,7 +1783,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj,
PyThread_release_lock(mutex);
return -1;
}
- if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) {
+ if (_PyObject_GetXIData(tstate, obj, fallback, data) != 0) {
PyThread_release_lock(mutex);
GLOBAL_FREE(data);
return -1;
@@ -1823,7 +1827,8 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting)
// Like channel_send(), but strictly wait for the object to be received.
static int
channel_send_wait(_channels *channels, int64_t cid, PyObject *obj,
- int unboundop, PY_TIMEOUT_T timeout)
+ unboundop_t unboundop, PY_TIMEOUT_T timeout,
+ xidata_fallback_t fallback)
{
// We use a stack variable here, so we must ensure that &waiting
// is not held by any channel item at the point this function exits.
@@ -1834,7 +1839,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj,
}
/* Queue up the object. */
- int res = channel_send(channels, cid, obj, &waiting, unboundop);
+ int res = channel_send(channels, cid, obj, &waiting, unboundop, fallback);
if (res < 0) {
assert(waiting.status == WAITING_NO_STATUS);
goto finally;
@@ -2006,6 +2011,20 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid,
}
static int
+channel_get_defaults(_channels *channels, int64_t cid, struct _channeldefaults *defaults)
+{
+ PyThread_type_lock mutex = NULL;
+ _channel_state *channel = NULL;
+ int err = _channels_lookup(channels, cid, &mutex, &channel);
+ if (err != 0) {
+ return err;
+ }
+ *defaults = channel->defaults;
+ PyThread_release_lock(mutex);
+ return 0;
+}
+
+static int
_channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count)
{
PyThread_type_lock mutex = NULL;
@@ -2881,20 +2900,27 @@ clear_interpreter(void *data)
static PyObject *
channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"unboundop", NULL};
- int unboundop;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist,
- &unboundop))
+ static char *kwlist[] = {"unboundop", "fallback", NULL};
+ int unboundarg = -1;
+ int fallbackarg = -1;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:create", kwlist,
+ &unboundarg, &fallbackarg))
{
return NULL;
}
- if (!check_unbound(unboundop)) {
- PyErr_Format(PyExc_ValueError,
- "unsupported unboundop %d", unboundop);
+ struct _channeldefaults defaults = {0};
+ if (resolve_unboundop(unboundarg, UNBOUND_REPLACE,
+ &defaults.unboundop) < 0)
+ {
+ return NULL;
+ }
+ if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK,
+ &defaults.fallback) < 0)
+ {
return NULL;
}
- int64_t cid = channel_create(&_globals.channels, unboundop);
+ int64_t cid = channel_create(&_globals.channels, defaults);
if (cid < 0) {
(void)handle_channel_error(-1, self, cid);
return NULL;
@@ -2987,7 +3013,9 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
}
assert(cidobj != NULL);
- PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop);
+ PyObject *item = Py_BuildValue("Oii", cidobj,
+ cur->defaults.unboundop,
+ cur->defaults.fallback);
Py_DECREF(cidobj);
if (item == NULL) {
Py_SETREF(ids, NULL);
@@ -3075,40 +3103,54 @@ receive end.");
static PyObject *
channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout",
- NULL};
+ static char *kwlist[] = {"cid", "obj", "unboundop", "fallback",
+ "blocking", "timeout", NULL};
struct channel_id_converter_data cid_data = {
.module = self,
};
PyObject *obj;
- int unboundop = UNBOUND_REPLACE;
+ int unboundarg = -1;
+ int fallbackarg = -1;
int blocking = 1;
PyObject *timeout_obj = NULL;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist,
+ if (!PyArg_ParseTupleAndKeywords(args, kwds,
+ "O&O|ii$pO:channel_send", kwlist,
channel_id_converter, &cid_data, &obj,
- &unboundop, &blocking, &timeout_obj))
+ &unboundarg, &fallbackarg,
+ &blocking, &timeout_obj))
{
return NULL;
}
- if (!check_unbound(unboundop)) {
- PyErr_Format(PyExc_ValueError,
- "unsupported unboundop %d", unboundop);
- return NULL;
- }
-
int64_t cid = cid_data.cid;
PY_TIMEOUT_T timeout;
if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) {
return NULL;
}
+ struct _channeldefaults defaults = {-1, -1};
+ if (unboundarg < 0 || fallbackarg < 0) {
+ int err = channel_get_defaults(&_globals.channels, cid, &defaults);
+ if (handle_channel_error(err, self, cid)) {
+ return NULL;
+ }
+ }
+ unboundop_t unboundop;
+ if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) {
+ return NULL;
+ }
+ xidata_fallback_t fallback;
+ if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) {
+ return NULL;
+ }
/* Queue up the object. */
int err = 0;
if (blocking) {
- err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout);
+ err = channel_send_wait(
+ &_globals.channels, cid, obj, unboundop, timeout, fallback);
}
else {
- err = channel_send(&_globals.channels, cid, obj, NULL, unboundop);
+ err = channel_send(
+ &_globals.channels, cid, obj, NULL, unboundop, fallback);
}
if (handle_channel_error(err, self, cid)) {
return NULL;
@@ -3126,32 +3168,44 @@ By default this waits for the object to be received.");
static PyObject *
channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout",
- NULL};
+ static char *kwlist[] = {"cid", "obj", "unboundop", "fallback",
+ "blocking", "timeout", NULL};
struct channel_id_converter_data cid_data = {
.module = self,
};
PyObject *obj;
- int unboundop = UNBOUND_REPLACE;
- int blocking = 1;
+ int unboundarg = -1;
+ int fallbackarg = -1;
+ int blocking = -1;
PyObject *timeout_obj = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
- "O&O|i$pO:channel_send_buffer", kwlist,
+ "O&O|ii$pO:channel_send_buffer", kwlist,
channel_id_converter, &cid_data, &obj,
- &unboundop, &blocking, &timeout_obj)) {
- return NULL;
- }
- if (!check_unbound(unboundop)) {
- PyErr_Format(PyExc_ValueError,
- "unsupported unboundop %d", unboundop);
+ &unboundarg, &fallbackarg,
+ &blocking, &timeout_obj))
+ {
return NULL;
}
-
int64_t cid = cid_data.cid;
PY_TIMEOUT_T timeout;
if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) {
return NULL;
}
+ struct _channeldefaults defaults = {-1, -1};
+ if (unboundarg < 0 || fallbackarg < 0) {
+ int err = channel_get_defaults(&_globals.channels, cid, &defaults);
+ if (handle_channel_error(err, self, cid)) {
+ return NULL;
+ }
+ }
+ unboundop_t unboundop;
+ if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) {
+ return NULL;
+ }
+ xidata_fallback_t fallback;
+ if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) {
+ return NULL;
+ }
PyObject *tempobj = PyMemoryView_FromObject(obj);
if (tempobj == NULL) {
@@ -3162,10 +3216,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds)
int err = 0;
if (blocking) {
err = channel_send_wait(
- &_globals.channels, cid, tempobj, unboundop, timeout);
+ &_globals.channels, cid, tempobj, unboundop, timeout, fallback);
}
else {
- err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop);
+ err = channel_send(
+ &_globals.channels, cid, tempobj, NULL, unboundop, fallback);
}
Py_DECREF(tempobj);
if (handle_channel_error(err, self, cid)) {
@@ -3197,7 +3252,7 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds)
cid = cid_data.cid;
PyObject *obj = NULL;
- int unboundop = 0;
+ unboundop_t unboundop = 0;
int err = channel_recv(&_globals.channels, cid, &obj, &unboundop);
if (err == ERR_CHANNEL_EMPTY && dflt != NULL) {
// Use the default.
@@ -3388,17 +3443,14 @@ channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds)
}
int64_t cid = cid_data.cid;
- PyThread_type_lock mutex = NULL;
- _channel_state *channel = NULL;
- int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel);
+ struct _channeldefaults defaults = {0};
+ int err = channel_get_defaults(&_globals.channels, cid, &defaults);
if (handle_channel_error(err, self, cid)) {
return NULL;
}
- int unboundop = channel->defaults.unboundop;
- PyThread_release_lock(mutex);
- PyObject *defaults = Py_BuildValue("i", unboundop);
- return defaults;
+ PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback);
+ return res;
}
PyDoc_STRVAR(channelsmod_get_channel_defaults_doc,
diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c
index 209fcdfd0cd..ffc52c8ee74 100644
--- a/Modules/_interpqueuesmodule.c
+++ b/Modules/_interpqueuesmodule.c
@@ -9,9 +9,11 @@
#include "pycore_crossinterp.h" // _PyXIData_t
#define REGISTERS_HEAP_TYPES
+#define HAS_FALLBACK
#define HAS_UNBOUND_ITEMS
#include "_interpreters_common.h"
#undef HAS_UNBOUND_ITEMS
+#undef HAS_FALLBACK
#undef REGISTERS_HEAP_TYPES
@@ -401,14 +403,13 @@ typedef struct _queueitem {
meaning the interpreter has been destroyed. */
int64_t interpid;
_PyXIData_t *data;
- int fmt;
- int unboundop;
+ unboundop_t unboundop;
struct _queueitem *next;
} _queueitem;
static void
_queueitem_init(_queueitem *item,
- int64_t interpid, _PyXIData_t *data, int fmt, int unboundop)
+ int64_t interpid, _PyXIData_t *data, unboundop_t unboundop)
{
if (interpid < 0) {
interpid = _get_interpid(data);
@@ -422,7 +423,6 @@ _queueitem_init(_queueitem *item,
*item = (_queueitem){
.interpid = interpid,
.data = data,
- .fmt = fmt,
.unboundop = unboundop,
};
}
@@ -446,14 +446,14 @@ _queueitem_clear(_queueitem *item)
}
static _queueitem *
-_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop)
+_queueitem_new(int64_t interpid, _PyXIData_t *data, int unboundop)
{
_queueitem *item = GLOBAL_MALLOC(_queueitem);
if (item == NULL) {
PyErr_NoMemory();
return NULL;
}
- _queueitem_init(item, interpid, data, fmt, unboundop);
+ _queueitem_init(item, interpid, data, unboundop);
return item;
}
@@ -476,10 +476,9 @@ _queueitem_free_all(_queueitem *item)
static void
_queueitem_popped(_queueitem *item,
- _PyXIData_t **p_data, int *p_fmt, int *p_unboundop)
+ _PyXIData_t **p_data, unboundop_t *p_unboundop)
{
*p_data = item->data;
- *p_fmt = item->fmt;
*p_unboundop = item->unboundop;
// We clear them here, so they won't be released in _queueitem_clear().
item->data = NULL;
@@ -527,16 +526,16 @@ typedef struct _queue {
_queueitem *first;
_queueitem *last;
} items;
- struct {
- int fmt;
+ struct _queuedefaults {
+ xidata_fallback_t fallback;
int unboundop;
} defaults;
} _queue;
static int
-_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop)
+_queue_init(_queue *queue, Py_ssize_t maxsize, struct _queuedefaults defaults)
{
- assert(check_unbound(unboundop));
+ assert(check_unbound(defaults.unboundop));
PyThread_type_lock mutex = PyThread_allocate_lock();
if (mutex == NULL) {
return ERR_QUEUE_ALLOC;
@@ -547,10 +546,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop)
.items = {
.maxsize = maxsize,
},
- .defaults = {
- .fmt = fmt,
- .unboundop = unboundop,
- },
+ .defaults = defaults,
};
return 0;
}
@@ -631,8 +627,7 @@ _queue_unlock(_queue *queue)
}
static int
-_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data,
- int fmt, int unboundop)
+_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, int unboundop)
{
int err = _queue_lock(queue);
if (err < 0) {
@@ -648,7 +643,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data,
return ERR_QUEUE_FULL;
}
- _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop);
+ _queueitem *item = _queueitem_new(interpid, data, unboundop);
if (item == NULL) {
_queue_unlock(queue);
return -1;
@@ -668,8 +663,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data,
}
static int
-_queue_next(_queue *queue,
- _PyXIData_t **p_data, int *p_fmt, int *p_unboundop)
+_queue_next(_queue *queue, _PyXIData_t **p_data, int *p_unboundop)
{
int err = _queue_lock(queue);
if (err < 0) {
@@ -688,7 +682,7 @@ _queue_next(_queue *queue,
}
queue->items.count -= 1;
- _queueitem_popped(item, p_data, p_fmt, p_unboundop);
+ _queueitem_popped(item, p_data, p_unboundop);
_queue_unlock(queue);
return 0;
@@ -1035,8 +1029,7 @@ finally:
struct queue_id_and_info {
int64_t id;
- int fmt;
- int unboundop;
+ struct _queuedefaults defaults;
};
static struct queue_id_and_info *
@@ -1053,8 +1046,7 @@ _queues_list_all(_queues *queues, int64_t *p_count)
for (int64_t i=0; ref != NULL; ref = ref->next, i++) {
ids[i].id = ref->qid;
assert(ref->queue != NULL);
- ids[i].fmt = ref->queue->defaults.fmt;
- ids[i].unboundop = ref->queue->defaults.unboundop;
+ ids[i].defaults = ref->queue->defaults;
}
*p_count = queues->count;
@@ -1090,13 +1082,14 @@ _queue_free(_queue *queue)
// Create a new queue.
static int64_t
-queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop)
+queue_create(_queues *queues, Py_ssize_t maxsize,
+ struct _queuedefaults defaults)
{
_queue *queue = GLOBAL_MALLOC(_queue);
if (queue == NULL) {
return ERR_QUEUE_ALLOC;
}
- int err = _queue_init(queue, maxsize, fmt, unboundop);
+ int err = _queue_init(queue, maxsize, defaults);
if (err < 0) {
GLOBAL_FREE(queue);
return (int64_t)err;
@@ -1125,7 +1118,8 @@ queue_destroy(_queues *queues, int64_t qid)
// Push an object onto the queue.
static int
-queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop)
+queue_put(_queues *queues, int64_t qid, PyObject *obj, unboundop_t unboundop,
+ xidata_fallback_t fallback)
{
PyThreadState *tstate = PyThreadState_Get();
@@ -1138,27 +1132,27 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop)
assert(queue != NULL);
// Convert the object to cross-interpreter data.
- _PyXIData_t *data = _PyXIData_New();
- if (data == NULL) {
+ _PyXIData_t *xidata = _PyXIData_New();
+ if (xidata == NULL) {
_queue_unmark_waiter(queue, queues->mutex);
return -1;
}
- if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) {
+ if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) {
_queue_unmark_waiter(queue, queues->mutex);
- GLOBAL_FREE(data);
+ GLOBAL_FREE(xidata);
return -1;
}
- assert(_PyXIData_INTERPID(data) ==
+ assert(_PyXIData_INTERPID(xidata) ==
PyInterpreterState_GetID(tstate->interp));
// Add the data to the queue.
int64_t interpid = -1; // _queueitem_init() will set it.
- int res = _queue_add(queue, interpid, data, fmt, unboundop);
+ int res = _queue_add(queue, interpid, xidata, unboundop);
_queue_unmark_waiter(queue, queues->mutex);
if (res != 0) {
// We may chain an exception here:
- (void)_release_xid_data(data, 0);
- GLOBAL_FREE(data);
+ (void)_release_xid_data(xidata, 0);
+ GLOBAL_FREE(xidata);
return res;
}
@@ -1169,7 +1163,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop)
// XXX Support a "wait" mutex?
static int
queue_get(_queues *queues, int64_t qid,
- PyObject **res, int *p_fmt, int *p_unboundop)
+ PyObject **res, int *p_unboundop)
{
int err;
*res = NULL;
@@ -1185,7 +1179,7 @@ queue_get(_queues *queues, int64_t qid,
// Pop off the next item from the queue.
_PyXIData_t *data = NULL;
- err = _queue_next(queue, &data, p_fmt, p_unboundop);
+ err = _queue_next(queue, &data, p_unboundop);
_queue_unmark_waiter(queue, queues->mutex);
if (err != 0) {
return err;
@@ -1217,6 +1211,20 @@ queue_get(_queues *queues, int64_t qid,
}
static int
+queue_get_defaults(_queues *queues, int64_t qid,
+ struct _queuedefaults *p_defaults)
+{
+ _queue *queue = NULL;
+ int err = _queues_lookup(queues, qid, &queue);
+ if (err != 0) {
+ return err;
+ }
+ *p_defaults = queue->defaults;
+ _queue_unmark_waiter(queue, queues->mutex);
+ return 0;
+}
+
+static int
queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize)
{
_queue *queue = NULL;
@@ -1474,22 +1482,28 @@ qidarg_converter(PyObject *arg, void *ptr)
static PyObject *
queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL};
+ static char *kwlist[] = {"maxsize", "unboundop", "fallback", NULL};
Py_ssize_t maxsize;
- int fmt;
- int unboundop;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist,
- &maxsize, &fmt, &unboundop))
+ int unboundarg = -1;
+ int fallbackarg = -1;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|ii:create", kwlist,
+ &maxsize, &unboundarg, &fallbackarg))
{
return NULL;
}
- if (!check_unbound(unboundop)) {
- PyErr_Format(PyExc_ValueError,
- "unsupported unboundop %d", unboundop);
+ struct _queuedefaults defaults = {0};
+ if (resolve_unboundop(unboundarg, UNBOUND_REPLACE,
+ &defaults.unboundop) < 0)
+ {
+ return NULL;
+ }
+ if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK,
+ &defaults.fallback) < 0)
+ {
return NULL;
}
- int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop);
+ int64_t qid = queue_create(&_globals.queues, maxsize, defaults);
if (qid < 0) {
(void)handle_queue_error((int)qid, self, qid);
return NULL;
@@ -1511,7 +1525,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds)
}
PyDoc_STRVAR(queuesmod_create_doc,
-"create(maxsize, fmt, unboundop) -> qid\n\
+"create(maxsize, unboundop, fallback) -> qid\n\
\n\
Create a new cross-interpreter queue and return its unique generated ID.\n\
It is a new reference as though bind() had been called on the queue.\n\
@@ -1560,8 +1574,9 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
}
struct queue_id_and_info *cur = qids;
for (int64_t i=0; i < count; cur++, i++) {
- PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt,
- cur->unboundop);
+ PyObject *item = Py_BuildValue("Lii", cur->id,
+ cur->defaults.unboundop,
+ cur->defaults.fallback);
if (item == NULL) {
Py_SETREF(ids, NULL);
break;
@@ -1575,34 +1590,44 @@ finally:
}
PyDoc_STRVAR(queuesmod_list_all_doc,
-"list_all() -> [(qid, fmt)]\n\
+"list_all() -> [(qid, unboundop, fallback)]\n\
\n\
Return the list of IDs for all queues.\n\
-Each corresponding default format is also included.");
+Each corresponding default unbound op and fallback is also included.");
static PyObject *
queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL};
+ static char *kwlist[] = {"qid", "obj", "unboundop", "fallback", NULL};
qidarg_converter_data qidarg = {0};
PyObject *obj;
- int fmt;
- int unboundop;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist,
- qidarg_converter, &qidarg, &obj, &fmt,
- &unboundop))
+ int unboundarg = -1;
+ int fallbackarg = -1;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|ii$p:put", kwlist,
+ qidarg_converter, &qidarg, &obj,
+ &unboundarg, &fallbackarg))
{
return NULL;
}
int64_t qid = qidarg.id;
- if (!check_unbound(unboundop)) {
- PyErr_Format(PyExc_ValueError,
- "unsupported unboundop %d", unboundop);
+ struct _queuedefaults defaults = {-1, -1};
+ if (unboundarg < 0 || fallbackarg < 0) {
+ int err = queue_get_defaults(&_globals.queues, qid, &defaults);
+ if (handle_queue_error(err, self, qid)) {
+ return NULL;
+ }
+ }
+ unboundop_t unboundop;
+ if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) {
+ return NULL;
+ }
+ xidata_fallback_t fallback;
+ if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) {
return NULL;
}
/* Queue up the object. */
- int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop);
+ int err = queue_put(&_globals.queues, qid, obj, unboundop, fallback);
// This is the only place that raises QueueFull.
if (handle_queue_error(err, self, qid)) {
return NULL;
@@ -1612,7 +1637,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds)
}
PyDoc_STRVAR(queuesmod_put_doc,
-"put(qid, obj, fmt)\n\
+"put(qid, obj)\n\
\n\
Add the object's data to the queue.");
@@ -1628,27 +1653,26 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds)
int64_t qid = qidarg.id;
PyObject *obj = NULL;
- int fmt = 0;
int unboundop = 0;
- int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop);
+ int err = queue_get(&_globals.queues, qid, &obj, &unboundop);
// This is the only place that raises QueueEmpty.
if (handle_queue_error(err, self, qid)) {
return NULL;
}
if (obj == NULL) {
- return Py_BuildValue("Oii", Py_None, fmt, unboundop);
+ return Py_BuildValue("Oi", Py_None, unboundop);
}
- PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None);
+ PyObject *res = Py_BuildValue("OO", obj, Py_None);
Py_DECREF(obj);
return res;
}
PyDoc_STRVAR(queuesmod_get_doc,
-"get(qid) -> (obj, fmt)\n\
+"get(qid) -> (obj, unboundop)\n\
\n\
Return a new object from the data at the front of the queue.\n\
-The object's format is also returned.\n\
+The unbound op is also returned.\n\
\n\
If there is nothing to receive then raise QueueEmpty.");
@@ -1748,17 +1772,14 @@ queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds)
}
int64_t qid = qidarg.id;
- _queue *queue = NULL;
- int err = _queues_lookup(&_globals.queues, qid, &queue);
+ struct _queuedefaults defaults = {0};
+ int err = queue_get_defaults(&_globals.queues, qid, &defaults);
if (handle_queue_error(err, self, qid)) {
return NULL;
}
- int fmt = queue->defaults.fmt;
- int unboundop = queue->defaults.unboundop;
- _queue_unmark_waiter(queue, _globals.queues.mutex);
- PyObject *defaults = Py_BuildValue("ii", fmt, unboundop);
- return defaults;
+ PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback);
+ return res;
}
PyDoc_STRVAR(queuesmod_get_queue_defaults_doc,
diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h
index d73cbca3635..40fd51d752e 100644
--- a/Modules/_interpreters_common.h
+++ b/Modules/_interpreters_common.h
@@ -39,10 +39,37 @@ _get_interpid(_PyXIData_t *data)
}
+#ifdef HAS_FALLBACK
+static int
+resolve_fallback(int arg, xidata_fallback_t dflt,
+ xidata_fallback_t *p_fallback)
+{
+ if (arg < 0) {
+ *p_fallback = dflt;
+ return 0;
+ }
+ xidata_fallback_t fallback;
+ if (arg == _PyXIDATA_XIDATA_ONLY) {
+ fallback =_PyXIDATA_XIDATA_ONLY;
+ }
+ else if (arg == _PyXIDATA_FULL_FALLBACK) {
+ fallback = _PyXIDATA_FULL_FALLBACK;
+ }
+ else {
+ PyErr_Format(PyExc_ValueError, "unsupported fallback %d", arg);
+ return -1;
+ }
+ *p_fallback = fallback;
+ return 0;
+}
+#endif
+
+
/* unbound items ************************************************************/
#ifdef HAS_UNBOUND_ITEMS
+typedef int unboundop_t;
#define UNBOUND_REMOVE 1
#define UNBOUND_ERROR 2
#define UNBOUND_REPLACE 3
@@ -53,6 +80,7 @@ _get_interpid(_PyXIData_t *data)
// object is released but the underlying data is copied (with the "raw"
// allocator) and used when the item is popped off the queue.
+#ifndef NDEBUG
static int
check_unbound(int unboundop)
{
@@ -65,5 +93,31 @@ check_unbound(int unboundop)
return 0;
}
}
+#endif
+
+static int
+resolve_unboundop(int arg, unboundop_t dflt, unboundop_t *p_unboundop)
+{
+ if (arg < 0) {
+ *p_unboundop = dflt;
+ return 0;
+ }
+ unboundop_t op;
+ if (arg == UNBOUND_REMOVE) {
+ op = UNBOUND_REMOVE;
+ }
+ else if (arg == UNBOUND_ERROR) {
+ op = UNBOUND_ERROR;
+ }
+ else if (arg == UNBOUND_REPLACE) {
+ op = UNBOUND_REPLACE;
+ }
+ else {
+ PyErr_Format(PyExc_ValueError, "unsupported unboundop %d", arg);
+ return -1;
+ }
+ *p_unboundop = op;
+ return 0;
+}
#endif
diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c
index f3c571e717f..f4807fd214b 100644
--- a/Modules/_interpretersmodule.c
+++ b/Modules/_interpretersmodule.c
@@ -8,6 +8,8 @@
#include "Python.h"
#include "pycore_code.h" // _PyCode_HAS_EXECUTORS()
#include "pycore_crossinterp.h" // _PyXIData_t
+#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
+#include "pycore_function.h" // _PyFunction_VerifyStateless()
#include "pycore_interp.h" // _PyInterpreterState_IDIncref()
#include "pycore_modsupport.h" // _PyArg_BadArgument()
#include "pycore_namespace.h" // _PyNamespace_New()
@@ -374,34 +376,17 @@ check_code_str(PyUnicodeObject *text)
return NULL;
}
-static const char *
-check_code_object(PyCodeObject *code)
+#ifndef NDEBUG
+static int
+code_has_args(PyCodeObject *code)
{
assert(code != NULL);
- if (code->co_argcount > 0
+ return (code->co_argcount > 0
|| code->co_posonlyargcount > 0
|| code->co_kwonlyargcount > 0
- || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS))
- {
- return "arguments not supported";
- }
- if (code->co_ncellvars > 0) {
- return "closures not supported";
- }
- // We trust that no code objects under co_consts have unbound cell vars.
-
- if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) {
- return "only basic functions are supported";
- }
- if (code->_co_monitoring != NULL) {
- return "only basic functions are supported";
- }
- if (code->co_extra != NULL) {
- return "only basic functions are supported";
- }
-
- return NULL;
+ || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS));
}
+#endif
#define RUN_TEXT 1
#define RUN_CODE 2
@@ -429,8 +414,10 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p)
flags = RUN_TEXT;
}
else {
- assert(PyCode_Check(arg)
- && (check_code_object((PyCodeObject *)arg) == NULL));
+ assert(PyCode_Check(arg));
+ assert(_PyCode_VerifyStateless(
+ PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0);
+ assert(!code_has_args((PyCodeObject *)arg));
flags = RUN_CODE;
// Serialize the code object.
@@ -542,42 +529,54 @@ _run_in_interpreter(PyInterpreterState *interp,
PyObject **p_excinfo)
{
assert(!PyErr_Occurred());
- _PyXI_session session = {0};
+ _PyXI_session *session = _PyXI_NewSession();
+ if (session == NULL) {
+ return -1;
+ }
// Prep and switch interpreters.
- if (_PyXI_Enter(&session, interp, shareables) < 0) {
+ if (_PyXI_Enter(session, interp, shareables) < 0) {
if (PyErr_Occurred()) {
// If an error occured at this step, it means that interp
// was not prepared and switched.
+ _PyXI_FreeSession(session);
return -1;
}
// Now, apply the error from another interpreter:
- PyObject *excinfo = _PyXI_ApplyError(session.error);
+ PyObject *excinfo = _PyXI_ApplyCapturedException(session);
if (excinfo != NULL) {
*p_excinfo = excinfo;
}
assert(PyErr_Occurred());
+ _PyXI_FreeSession(session);
return -1;
}
// Run the script.
- int res = _run_script(session.main_ns, codestr, codestrlen, flags);
+ int res = -1;
+ PyObject *mainns = _PyXI_GetMainNamespace(session);
+ if (mainns == NULL) {
+ goto finally;
+ }
+ res = _run_script(mainns, codestr, codestrlen, flags);
+finally:
// Clean up and switch back.
- _PyXI_Exit(&session);
+ _PyXI_Exit(session);
// Propagate any exception out to the caller.
assert(!PyErr_Occurred());
if (res < 0) {
- PyObject *excinfo = _PyXI_ApplyCapturedException(&session);
+ PyObject *excinfo = _PyXI_ApplyCapturedException(session);
if (excinfo != NULL) {
*p_excinfo = excinfo;
}
}
else {
- assert(!_PyXI_HasCapturedException(&session));
+ assert(!_PyXI_HasCapturedException(session));
}
+ _PyXI_FreeSession(session);
return res;
}
@@ -922,22 +921,27 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs)
}
}
- _PyXI_session session = {0};
+ _PyXI_session *session = _PyXI_NewSession();
+ if (session == NULL) {
+ return NULL;
+ }
// Prep and switch interpreters, including apply the updates.
- if (_PyXI_Enter(&session, interp, updates) < 0) {
+ if (_PyXI_Enter(session, interp, updates) < 0) {
if (!PyErr_Occurred()) {
- _PyXI_ApplyCapturedException(&session);
+ _PyXI_ApplyCapturedException(session);
assert(PyErr_Occurred());
}
else {
- assert(!_PyXI_HasCapturedException(&session));
+ assert(!_PyXI_HasCapturedException(session));
}
+ _PyXI_FreeSession(session);
return NULL;
}
// Clean up and switch back.
- _PyXI_Exit(&session);
+ _PyXI_Exit(session);
+ _PyXI_FreeSession(session);
Py_RETURN_NONE;
}
@@ -949,7 +953,8 @@ Bind the given attributes in the interpreter's __main__ module.");
static PyUnicodeObject *
-convert_script_arg(PyObject *arg, const char *fname, const char *displayname,
+convert_script_arg(PyThreadState *tstate,
+ PyObject *arg, const char *fname, const char *displayname,
const char *expected)
{
PyUnicodeObject *str = NULL;
@@ -968,8 +973,8 @@ convert_script_arg(PyObject *arg, const char *fname, const char *displayname,
const char *err = check_code_str(str);
if (err != NULL) {
Py_DECREF(str);
- PyErr_Format(PyExc_ValueError,
- "%.200s(): bad script text (%s)", fname, err);
+ _PyErr_Format(tstate, PyExc_ValueError,
+ "%.200s(): bad script text (%s)", fname, err);
return NULL;
}
@@ -977,51 +982,44 @@ convert_script_arg(PyObject *arg, const char *fname, const char *displayname,
}
static PyCodeObject *
-convert_code_arg(PyObject *arg, const char *fname, const char *displayname,
+convert_code_arg(PyThreadState *tstate,
+ PyObject *arg, const char *fname, const char *displayname,
const char *expected)
{
- const char *kind = NULL;
+ PyObject *cause;
PyCodeObject *code = NULL;
if (PyFunction_Check(arg)) {
- if (PyFunction_GetClosure(arg) != NULL) {
- PyErr_Format(PyExc_ValueError,
- "%.200s(): closures not supported", fname);
- return NULL;
+ // For now we allow globals, so we can't use
+ // _PyFunction_VerifyStateless().
+ PyObject *codeobj = PyFunction_GetCode(arg);
+ if (_PyCode_VerifyStateless(
+ tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) {
+ goto chained;
}
- code = (PyCodeObject *)PyFunction_GetCode(arg);
- if (code == NULL) {
- if (PyErr_Occurred()) {
- // This chains.
- PyErr_Format(PyExc_ValueError,
- "%.200s(): bad func", fname);
- }
- else {
- PyErr_Format(PyExc_ValueError,
- "%.200s(): func.__code__ missing", fname);
- }
- return NULL;
- }
- Py_INCREF(code);
- kind = "func";
+ code = (PyCodeObject *)Py_NewRef(codeobj);
}
else if (PyCode_Check(arg)) {
+ if (_PyCode_VerifyStateless(
+ tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) {
+ goto chained;
+ }
code = (PyCodeObject *)Py_NewRef(arg);
- kind = "code object";
}
else {
_PyArg_BadArgument(fname, displayname, expected, arg);
return NULL;
}
- const char *err = check_code_object(code);
- if (err != NULL) {
- Py_DECREF(code);
- PyErr_Format(PyExc_ValueError,
- "%.200s(): bad %s (%s)", fname, kind, err);
- return NULL;
- }
-
return code;
+
+chained:
+ cause = _PyErr_GetRaisedException(tstate);
+ assert(cause != NULL);
+ _PyArg_BadArgument(fname, displayname, expected, arg);
+ PyObject *exc = _PyErr_GetRaisedException(tstate);
+ PyException_SetCause(exc, cause);
+ _PyErr_SetRaisedException(tstate, exc);
+ return NULL;
}
static int
@@ -1057,12 +1055,14 @@ _interp_exec(PyObject *self, PyInterpreterState *interp,
static PyObject *
interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
{
+#define FUNCNAME MODULE_NAME_STR ".exec"
+ PyThreadState *tstate = _PyThreadState_GET();
static char *kwlist[] = {"id", "code", "shared", "restrict", NULL};
PyObject *id, *code;
PyObject *shared = NULL;
int restricted = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
- "OO|O$p:" MODULE_NAME_STR ".exec", kwlist,
+ "OO|O$p:" FUNCNAME, kwlist,
&id, &code, &shared, &restricted))
{
return NULL;
@@ -1077,12 +1077,12 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
const char *expected = "a string, a function, or a code object";
if (PyUnicode_Check(code)) {
- code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec",
- "argument 2", expected);
+ code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME,
+ "argument 2", expected);
}
else {
- code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec",
- "argument 2", expected);
+ code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME,
+ "argument 2", expected);
}
if (code == NULL) {
return NULL;
@@ -1096,6 +1096,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
return excinfo;
}
Py_RETURN_NONE;
+#undef FUNCNAME
}
PyDoc_STRVAR(exec_doc,
@@ -1118,13 +1119,15 @@ is ignored, including its __globals__ dict.");
static PyObject *
interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
{
+#define FUNCNAME MODULE_NAME_STR ".run_string"
+ PyThreadState *tstate = _PyThreadState_GET();
static char *kwlist[] = {"id", "script", "shared", "restrict", NULL};
PyObject *id, *script;
PyObject *shared = NULL;
int restricted = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
- "OU|O$p:" MODULE_NAME_STR ".run_string",
- kwlist, &id, &script, &shared, &restricted))
+ "OU|O$p:" FUNCNAME, kwlist,
+ &id, &script, &shared, &restricted))
{
return NULL;
}
@@ -1136,7 +1139,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
- script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".run_string",
+ script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME,
"argument 2", "a string");
if (script == NULL) {
return NULL;
@@ -1150,6 +1153,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
return excinfo;
}
Py_RETURN_NONE;
+#undef FUNCNAME
}
PyDoc_STRVAR(run_string_doc,
@@ -1162,13 +1166,15 @@ Execute the provided string in the identified interpreter.\n\
static PyObject *
interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
{
+#define FUNCNAME MODULE_NAME_STR ".run_func"
+ PyThreadState *tstate = _PyThreadState_GET();
static char *kwlist[] = {"id", "func", "shared", "restrict", NULL};
PyObject *id, *func;
PyObject *shared = NULL;
int restricted = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
- "OO|O$p:" MODULE_NAME_STR ".run_func",
- kwlist, &id, &func, &shared, &restricted))
+ "OO|O$p:" FUNCNAME, kwlist,
+ &id, &func, &shared, &restricted))
{
return NULL;
}
@@ -1180,7 +1186,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
- PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec",
+ PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME,
"argument 2",
"a function or a code object");
if (code == NULL) {
@@ -1195,6 +1201,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
return excinfo;
}
Py_RETURN_NONE;
+#undef FUNCNAME
}
PyDoc_STRVAR(run_func_doc,
@@ -1209,6 +1216,8 @@ are not supported. Methods and other callables are not supported either.\n\
static PyObject *
interp_call(PyObject *self, PyObject *args, PyObject *kwds)
{
+#define FUNCNAME MODULE_NAME_STR ".call"
+ PyThreadState *tstate = _PyThreadState_GET();
static char *kwlist[] = {"id", "callable", "args", "kwargs",
"restrict", NULL};
PyObject *id, *callable;
@@ -1216,7 +1225,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *kwargs_obj = NULL;
int restricted = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
- "OO|OO$p:" MODULE_NAME_STR ".call", kwlist,
+ "OO|OO$p:" FUNCNAME, kwlist,
&id, &callable, &args_obj, &kwargs_obj,
&restricted))
{
@@ -1231,15 +1240,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
}
if (args_obj != NULL) {
- PyErr_SetString(PyExc_ValueError, "got unexpected args");
+ _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected args");
return NULL;
}
if (kwargs_obj != NULL) {
- PyErr_SetString(PyExc_ValueError, "got unexpected kwargs");
+ _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected kwargs");
return NULL;
}
- PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call",
+ PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME,
"argument 2", "a function");
if (code == NULL) {
return NULL;
@@ -1253,6 +1262,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
return excinfo;
}
Py_RETURN_NONE;
+#undef FUNCNAME
}
PyDoc_STRVAR(call_doc,
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index ad618398d5b..c1f56008b7c 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -692,7 +692,17 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
result = result != NULL ? result : "";
char *oldloc = NULL;
if (langinfo_constants[i].category != LC_CTYPE
- && !is_all_ascii(result)
+ && (
+#ifdef __GLIBC__
+ // gh-133740: Always change the locale for ALT_DIGITS and ERA
+# ifdef ALT_DIGITS
+ item == ALT_DIGITS ||
+# endif
+# ifdef ERA
+ item == ERA ||
+# endif
+#endif
+ !is_all_ascii(result))
&& change_locale(langinfo_constants[i].category, &oldloc) < 0)
{
return NULL;
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index d260f1a68f8..29ef0cb0c2e 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1879,6 +1879,10 @@ _checkmodule(PyObject *module_name, PyObject *module,
_PyUnicode_EqualToASCIIString(module_name, "__main__")) {
return -1;
}
+ if (PyUnicode_Check(module_name) &&
+ _PyUnicode_EqualToASCIIString(module_name, "__mp_main__")) {
+ return -1;
+ }
PyObject *candidate = getattribute(module, dotted_path, 0);
if (candidate == NULL) {
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index 77286ed2a74..cc83be4b5ff 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -296,6 +296,12 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state)
continue;
}
+ // Keep handles for threads that have not been started yet. They are
+ // safe to start in the child process.
+ if (handle->state == THREAD_HANDLE_NOT_STARTED) {
+ continue;
+ }
+
// Mark all threads as done. Any attempts to join or detach the
// underlying OS thread (if any) could crash. We are the only thread;
// it's safe to set this non-atomically.
@@ -1022,6 +1028,11 @@ rlock_traverse(PyObject *self, visitproc visit, void *arg)
return 0;
}
+static int
+rlock_locked_impl(rlockobject *self)
+{
+ return PyMutex_IsLocked(&self->lock.mutex);
+}
static void
rlock_dealloc(PyObject *self)
@@ -1111,7 +1122,7 @@ static PyObject *
rlock_locked(PyObject *op, PyObject *Py_UNUSED(ignored))
{
rlockobject *self = rlockobject_CAST(op);
- int is_locked = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock);
+ int is_locked = rlock_locked_impl(self);
return PyBool_FromLong(is_locked);
}
@@ -1219,10 +1230,17 @@ rlock_repr(PyObject *op)
{
rlockobject *self = rlockobject_CAST(op);
PyThread_ident_t owner = self->lock.thread;
- size_t count = self->lock.level + 1;
+ int locked = rlock_locked_impl(self);
+ size_t count;
+ if (locked) {
+ count = self->lock.level + 1;
+ }
+ else {
+ count = 0;
+ }
return PyUnicode_FromFormat(
"<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>",
- owner ? "locked" : "unlocked",
+ locked ? "locked" : "unlocked",
Py_TYPE(self)->tp_name, owner,
count, self);
}
diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c
index 17d3bff1e98..56ad999e5cd 100644
--- a/Modules/_zstd/_zstdmodule.c
+++ b/Modules/_zstd/_zstdmodule.c
@@ -28,41 +28,42 @@ set_zstd_error(const _zstd_state* const state,
char *msg;
assert(ZSTD_isError(zstd_ret));
- switch (type)
- {
- case ERR_DECOMPRESS:
- msg = "Unable to decompress Zstandard data: %s";
- break;
- case ERR_COMPRESS:
- msg = "Unable to compress Zstandard data: %s";
- break;
-
- case ERR_LOAD_D_DICT:
- msg = "Unable to load Zstandard dictionary or prefix for decompression: %s";
- break;
- case ERR_LOAD_C_DICT:
- msg = "Unable to load Zstandard dictionary or prefix for compression: %s";
- break;
-
- case ERR_GET_C_BOUNDS:
- msg = "Unable to get zstd compression parameter bounds: %s";
- break;
- case ERR_GET_D_BOUNDS:
- msg = "Unable to get zstd decompression parameter bounds: %s";
- break;
- case ERR_SET_C_LEVEL:
- msg = "Unable to set zstd compression level: %s";
- break;
-
- case ERR_TRAIN_DICT:
- msg = "Unable to train the Zstandard dictionary: %s";
- break;
- case ERR_FINALIZE_DICT:
- msg = "Unable to finalize the Zstandard dictionary: %s";
- break;
-
- default:
- Py_UNREACHABLE();
+ switch (type) {
+ case ERR_DECOMPRESS:
+ msg = "Unable to decompress Zstandard data: %s";
+ break;
+ case ERR_COMPRESS:
+ msg = "Unable to compress Zstandard data: %s";
+ break;
+
+ case ERR_LOAD_D_DICT:
+ msg = "Unable to load Zstandard dictionary or prefix for "
+ "decompression: %s";
+ break;
+ case ERR_LOAD_C_DICT:
+ msg = "Unable to load Zstandard dictionary or prefix for "
+ "compression: %s";
+ break;
+
+ case ERR_GET_C_BOUNDS:
+ msg = "Unable to get zstd compression parameter bounds: %s";
+ break;
+ case ERR_GET_D_BOUNDS:
+ msg = "Unable to get zstd decompression parameter bounds: %s";
+ break;
+ case ERR_SET_C_LEVEL:
+ msg = "Unable to set zstd compression level: %s";
+ break;
+
+ case ERR_TRAIN_DICT:
+ msg = "Unable to train the Zstandard dictionary: %s";
+ break;
+ case ERR_FINALIZE_DICT:
+ msg = "Unable to finalize the Zstandard dictionary: %s";
+ break;
+
+ default:
+ Py_UNREACHABLE();
}
PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret));
}
@@ -183,7 +184,7 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes,
chunks_number = Py_SIZE(samples_sizes);
if ((size_t) chunks_number > UINT32_MAX) {
PyErr_Format(PyExc_ValueError,
- "The number of samples should be <= %u.", UINT32_MAX);
+ "The number of samples should be <= %u.", UINT32_MAX);
return -1;
}
@@ -200,8 +201,8 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes,
(*chunk_sizes)[i] = PyLong_AsSize_t(size);
if ((*chunk_sizes)[i] == (size_t)-1 && PyErr_Occurred()) {
PyErr_Format(PyExc_ValueError,
- "Items in samples_sizes should be an int "
- "object, with a value between 0 and %u.", SIZE_MAX);
+ "Items in samples_sizes should be an int "
+ "object, with a value between 0 and %u.", SIZE_MAX);
return -1;
}
sizes_sum += (*chunk_sizes)[i];
@@ -209,7 +210,8 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes,
if (sizes_sum != Py_SIZE(samples_bytes)) {
PyErr_SetString(PyExc_ValueError,
- "The samples size tuple doesn't match the concatenation's size.");
+ "The samples size tuple doesn't match the "
+ "concatenation's size.");
return -1;
}
return chunks_number;
@@ -242,15 +244,15 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
/* Check arguments */
if (dict_size <= 0) {
- PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number.");
+ PyErr_SetString(PyExc_ValueError,
+ "dict_size argument should be positive number.");
return NULL;
}
/* Check that the samples are valid and get their sizes */
chunks_number = calculate_samples_stats(samples_bytes, samples_sizes,
&chunk_sizes);
- if (chunks_number < 0)
- {
+ if (chunks_number < 0) {
goto error;
}
@@ -271,7 +273,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
/* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret);
goto error;
}
@@ -324,15 +326,15 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
/* Check arguments */
if (dict_size <= 0) {
- PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number.");
+ PyErr_SetString(PyExc_ValueError,
+ "dict_size argument should be positive number.");
return NULL;
}
/* Check that the samples are valid and get their sizes */
chunks_number = calculate_samples_stats(samples_bytes, samples_sizes,
&chunk_sizes);
- if (chunks_number < 0)
- {
+ if (chunks_number < 0) {
goto error;
}
@@ -355,14 +357,15 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
Py_BEGIN_ALLOW_THREADS
zstd_ret = ZDICT_finalizeDictionary(
PyBytes_AS_STRING(dst_dict_bytes), dict_size,
- PyBytes_AS_STRING(custom_dict_bytes), Py_SIZE(custom_dict_bytes),
+ PyBytes_AS_STRING(custom_dict_bytes),
+ Py_SIZE(custom_dict_bytes),
PyBytes_AS_STRING(samples_bytes), chunk_sizes,
(uint32_t)chunks_number, params);
Py_END_ALLOW_THREADS
/* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret);
goto error;
}
@@ -402,7 +405,7 @@ _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
if (is_compress) {
bound = ZSTD_cParam_getBounds(parameter);
if (ZSTD_isError(bound.error)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_GET_C_BOUNDS, bound.error);
return NULL;
}
@@ -410,7 +413,7 @@ _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
else {
bound = ZSTD_dParam_getBounds(parameter);
if (ZSTD_isError(bound.error)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_GET_D_BOUNDS, bound.error);
return NULL;
}
@@ -435,9 +438,10 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
{
size_t frame_size;
- frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, frame_buffer->len);
+ frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf,
+ frame_buffer->len);
if (ZSTD_isError(frame_size)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
PyErr_Format(mod_state->ZstdError,
"Error when finding the compressed size of a Zstandard frame. "
"Ensure the frame_buffer argument starts from the "
@@ -473,7 +477,7 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
/* #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) */
if (decompressed_size == ZSTD_CONTENTSIZE_ERROR) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
PyErr_SetString(mod_state->ZstdError,
"Error when getting information from the header of "
"a Zstandard frame. Ensure the frame_buffer argument "
@@ -508,7 +512,7 @@ _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
PyObject *d_parameter_type)
/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/
{
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) {
PyErr_SetString(PyExc_ValueError,
@@ -568,7 +572,7 @@ do { \
Py_DECREF(v); \
} while (0)
- _zstd_state* const mod_state = get_zstd_state(m);
+ _zstd_state* mod_state = get_zstd_state(m);
/* Reusable objects & variables */
mod_state->CParameter_type = NULL;
@@ -674,7 +678,7 @@ do { \
static int
_zstd_traverse(PyObject *module, visitproc visit, void *arg)
{
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
Py_VISIT(mod_state->ZstdDict_type);
Py_VISIT(mod_state->ZstdCompressor_type);
@@ -691,7 +695,7 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg)
static int
_zstd_clear(PyObject *module)
{
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
Py_CLEAR(mod_state->ZstdDict_type);
Py_CLEAR(mod_state->ZstdCompressor_type);
diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h
index bff3a81d8aa..4c885fa0d72 100644
--- a/Modules/_zstd/buffer.h
+++ b/Modules/_zstd/buffer.h
@@ -19,7 +19,8 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
/* Ensure .list was set to NULL */
assert(buffer->list == NULL);
- Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, &ob->dst);
+ Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length,
+ &ob->dst);
if (res < 0) {
return -1;
}
@@ -34,8 +35,7 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
Return -1 on failure */
static inline int
_OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
- Py_ssize_t max_length,
- Py_ssize_t init_size)
+ Py_ssize_t max_length, Py_ssize_t init_size)
{
Py_ssize_t block_size;
@@ -50,7 +50,8 @@ _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
block_size = init_size;
}
- Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, &ob->dst);
+ Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size,
+ &ob->dst);
if (res < 0) {
return -1;
}
diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h
index 4ecb19e9bde..c6fdae74ab0 100644
--- a/Modules/_zstd/clinic/decompressor.c.h
+++ b/Modules/_zstd/clinic/decompressor.c.h
@@ -7,7 +7,6 @@ preserve
# include "pycore_runtime.h" // _Py_ID()
#endif
#include "pycore_abstract.h" // _PyNumber_Index()
-#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__,
@@ -114,13 +113,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self);
static PyObject *
_zstd_ZstdDecompressor_unused_data_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self);
}
PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__,
@@ -227,4 +220,4 @@ exit:
return return_value;
}
-/*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h
index 34e0e4b3ecf..810befdaf71 100644
--- a/Modules/_zstd/clinic/zstddict.c.h
+++ b/Modules/_zstd/clinic/zstddict.c.h
@@ -6,7 +6,6 @@ preserve
# include "pycore_gc.h" // PyGC_Head
# include "pycore_runtime.h" // _Py_ID()
#endif
-#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_zstd_ZstdDict_new__doc__,
@@ -90,7 +89,9 @@ exit:
PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__,
"Load as a digested dictionary to compressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_digested_dict)\n"
+"\n"
"1. Some advanced compression parameters of compressor may be overridden\n"
" by parameters of digested dictionary.\n"
"2. ZstdDict has a digested dictionaries cache for each compression level.\n"
@@ -118,19 +119,15 @@ _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self);
}
PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__,
"Load as an undigested dictionary to compressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_undigested_dict)\n"
+"\n"
"1. The advanced compression parameters of compressor will not be overridden.\n"
"2. Loading an undigested dictionary is costly. If load an undigested dictionary\n"
" multiple times, consider reusing a compressor object.\n"
@@ -156,19 +153,15 @@ _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self);
}
PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__,
"Load as a prefix to compressor/decompressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_prefix)\n"
+"\n"
"1. Prefix is compatible with long distance matching, while dictionary is not.\n"
"2. It only works for the first frame, then the compressor/decompressor will\n"
" return to no prefix state.\n"
@@ -194,12 +187,6 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self);
}
-/*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/
+/*[clinic end generated code: output=47b12b5848b53ed8 input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c
index 38baee2be1e..31cb8c535c0 100644
--- a/Modules/_zstd/compressor.c
+++ b/Modules/_zstd/compressor.c
@@ -17,6 +17,7 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
#include "_zstdmodule.h"
#include "buffer.h"
#include "zstddict.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
#include <stddef.h> // offsetof()
#include <zstd.h> // ZSTD_*()
@@ -38,6 +39,9 @@ typedef struct {
/* Compression level */
int compression_level;
+
+ /* Lock to protect the compression context */
+ PyMutex lock;
} ZstdCompressor;
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
@@ -49,7 +53,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
const char *arg_name, const char* arg_type)
{
size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state == NULL) {
return -1;
}
@@ -59,8 +63,8 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
int level = PyLong_AsInt(level_or_options);
if (level == -1 && PyErr_Occurred()) {
PyErr_Format(PyExc_ValueError,
- "Compression level should be an int value between %d and %d.",
- ZSTD_minCLevel(), ZSTD_maxCLevel());
+ "Compression level should be an int value between "
+ "%d and %d.", ZSTD_minCLevel(), ZSTD_maxCLevel());
return -1;
}
@@ -89,24 +93,23 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
/* Check key type */
if (Py_TYPE(key) == mod_state->DParameter_type) {
PyErr_SetString(PyExc_TypeError,
- "Key of compression option dict should "
- "NOT be DecompressionParameter.");
+ "Key of compression options dict should "
+ "NOT be a DecompressionParameter attribute.");
return -1;
}
int key_v = PyLong_AsInt(key);
if (key_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
- "Key of options dict should be a CompressionParameter attribute.");
+ "Key of options dict should be either a "
+ "CompressionParameter attribute or an int.");
return -1;
}
- // TODO(emmatyping): check bounds when there is a value error here for better
- // error message?
int value_v = PyLong_AsInt(value);
if (value_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
- "Value of option dict should be an int.");
+ "Value of options dict should be an int.");
return -1;
}
@@ -135,7 +138,8 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
}
return 0;
}
- PyErr_Format(PyExc_TypeError, "Invalid type for %s. Expected %s", arg_name, arg_type);
+ PyErr_Format(PyExc_TypeError,
+ "Invalid type for %s. Expected %s", arg_name, arg_type);
return -1;
}
@@ -149,12 +153,12 @@ capsule_free_cdict(PyObject *capsule)
ZSTD_CDict *
_get_CDict(ZstdDict *self, int compressionLevel)
{
+ assert(PyMutex_IsLocked(&self->lock));
PyObject *level = NULL;
- PyObject *capsule;
+ PyObject *capsule = NULL;
ZSTD_CDict *cdict;
+ int ret;
- // TODO(emmatyping): refactor critical section code into a lock_held function
- Py_BEGIN_CRITICAL_SECTION(self);
/* int level object */
level = PyLong_FromLong(compressionLevel);
@@ -163,12 +167,11 @@ _get_CDict(ZstdDict *self, int compressionLevel)
}
/* Get PyCapsule object from self->c_dicts */
- capsule = PyDict_GetItemWithError(self->c_dicts, level);
+ ret = PyDict_GetItemRef(self->c_dicts, level, &capsule);
+ if (ret < 0) {
+ goto error;
+ }
if (capsule == NULL) {
- if (PyErr_Occurred()) {
- goto error;
- }
-
/* Create ZSTD_CDict instance */
char *dict_buffer = PyBytes_AS_STRING(self->dict_content);
Py_ssize_t dict_len = Py_SIZE(self->dict_content);
@@ -179,7 +182,7 @@ _get_CDict(ZstdDict *self, int compressionLevel)
Py_END_ALLOW_THREADS
if (cdict == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Failed to create a ZSTD_CDict instance from "
@@ -196,11 +199,10 @@ _get_CDict(ZstdDict *self, int compressionLevel)
}
/* Add PyCapsule object to self->c_dicts */
- if (PyDict_SetItem(self->c_dicts, level, capsule) < 0) {
- Py_DECREF(capsule);
+ ret = PyDict_SetItem(self->c_dicts, level, capsule);
+ if (ret < 0) {
goto error;
}
- Py_DECREF(capsule);
}
else {
/* ZSTD_CDict instance already exists */
@@ -212,16 +214,56 @@ error:
cdict = NULL;
success:
Py_XDECREF(level);
- Py_END_CRITICAL_SECTION();
+ Py_XDECREF(capsule);
return cdict;
}
static int
-_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
+_zstd_load_impl(ZstdCompressor *self, ZstdDict *zd,
+ _zstd_state *mod_state, int type)
{
-
size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ if (type == DICT_TYPE_DIGESTED) {
+ /* Get ZSTD_CDict */
+ ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level);
+ if (c_dict == NULL) {
+ return -1;
+ }
+ /* Reference a prepared dictionary.
+ It overrides some compression context's parameters. */
+ zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict);
+ }
+ else if (type == DICT_TYPE_UNDIGESTED) {
+ /* Load a dictionary.
+ It doesn't override compression context's parameters. */
+ zstd_ret = ZSTD_CCtx_loadDictionary(
+ self->cctx,
+ PyBytes_AS_STRING(zd->dict_content),
+ Py_SIZE(zd->dict_content));
+ }
+ else if (type == DICT_TYPE_PREFIX) {
+ /* Load a prefix */
+ zstd_ret = ZSTD_CCtx_refPrefix(
+ self->cctx,
+ PyBytes_AS_STRING(zd->dict_content),
+ Py_SIZE(zd->dict_content));
+ }
+ else {
+ Py_UNREACHABLE();
+ }
+
+ /* Check error */
+ if (ZSTD_isError(zstd_ret)) {
+ set_zstd_error(mod_state, ERR_LOAD_C_DICT, zstd_ret);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
+{
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state == NULL) {
return -1;
}
@@ -237,7 +279,10 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
/* When compressing, use undigested dictionary by default. */
zd = (ZstdDict*)dict;
type = DICT_TYPE_UNDIGESTED;
- goto load;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
}
/* Check (ZstdDict, type) */
@@ -251,13 +296,16 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
else if (ret > 0) {
/* type == -1 may indicate an error. */
type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1));
- if (type == DICT_TYPE_DIGESTED ||
- type == DICT_TYPE_UNDIGESTED ||
- type == DICT_TYPE_PREFIX)
+ if (type == DICT_TYPE_DIGESTED
+ || type == DICT_TYPE_UNDIGESTED
+ || type == DICT_TYPE_PREFIX)
{
assert(type >= 0);
zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0);
- goto load;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
}
}
}
@@ -266,49 +314,6 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
PyErr_SetString(PyExc_TypeError,
"zstd_dict argument should be ZstdDict object.");
return -1;
-
-load:
- if (type == DICT_TYPE_DIGESTED) {
- /* Get ZSTD_CDict */
- ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level);
- if (c_dict == NULL) {
- return -1;
- }
- /* Reference a prepared dictionary.
- It overrides some compression context's parameters. */
- Py_BEGIN_CRITICAL_SECTION(self);
- zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict);
- Py_END_CRITICAL_SECTION();
- }
- else if (type == DICT_TYPE_UNDIGESTED) {
- /* Load a dictionary.
- It doesn't override compression context's parameters. */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_CCtx_loadDictionary(
- self->cctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
- }
- else if (type == DICT_TYPE_PREFIX) {
- /* Load a prefix */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_CCtx_refPrefix(
- self->cctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
- }
- else {
- Py_UNREACHABLE();
- }
-
- /* Check error */
- if (ZSTD_isError(zstd_ret)) {
- set_zstd_error(mod_state, ERR_LOAD_C_DICT, zstd_ret);
- return -1;
- }
- return 0;
}
/*[clinic input]
@@ -339,11 +344,12 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
self->use_multithread = 0;
self->dict = NULL;
+ self->lock = (PyMutex){0};
/* Compression context */
self->cctx = ZSTD_createCCtx();
if (self->cctx == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Unable to create ZSTD_CCtx instance.");
@@ -355,7 +361,8 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
self->last_mode = ZSTD_e_end;
if (level != Py_None && options != Py_None) {
- PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
+ PyErr_SetString(PyExc_RuntimeError,
+ "Only one of level or options should be used.");
goto error;
}
@@ -403,6 +410,8 @@ ZstdCompressor_dealloc(PyObject *ob)
ZSTD_freeCCtx(self->cctx);
}
+ assert(!PyMutex_IsLocked(&self->lock));
+
/* Py_XDECREF the dict after free the compression context */
Py_CLEAR(self->dict);
@@ -412,9 +421,10 @@ ZstdCompressor_dealloc(PyObject *ob)
}
static PyObject *
-compress_impl(ZstdCompressor *self, Py_buffer *data,
- ZSTD_EndDirective end_directive)
+compress_lock_held(ZstdCompressor *self, Py_buffer *data,
+ ZSTD_EndDirective end_directive)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
ZSTD_outBuffer out;
_BlocksOutputBuffer buffer = {.list = NULL};
@@ -441,7 +451,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
}
if (_OutputBuffer_InitWithSize(&buffer, &out, -1,
- (Py_ssize_t) output_buffer_size) < 0) {
+ (Py_ssize_t) output_buffer_size) < 0) {
goto error;
}
@@ -454,7 +464,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
}
@@ -495,8 +505,9 @@ mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
#endif
static PyObject *
-compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
+compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
ZSTD_outBuffer out;
_BlocksOutputBuffer buffer = {.list = NULL};
@@ -516,20 +527,23 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
while (1) {
Py_BEGIN_ALLOW_THREADS
do {
- zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, ZSTD_e_continue);
- } while (out.pos != out.size && in.pos != in.size && !ZSTD_isError(zstd_ret));
+ zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in,
+ ZSTD_e_continue);
+ } while (out.pos != out.size
+ && in.pos != in.size
+ && !ZSTD_isError(zstd_ret));
Py_END_ALLOW_THREADS
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
}
goto error;
}
- /* Like compress_impl(), output as much as possible. */
+ /* Like compress_lock_held(), output as much as possible. */
if (out.pos == out.size) {
if (_OutputBuffer_Grow(&buffer, &out) < 0) {
goto error;
@@ -588,14 +602,14 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data,
}
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
+ PyMutex_Lock(&self->lock);
/* Compress */
if (self->use_multithread && mode == ZSTD_e_continue) {
- ret = compress_mt_continue_impl(self, data);
+ ret = compress_mt_continue_lock_held(self, data);
}
else {
- ret = compress_impl(self, data, mode);
+ ret = compress_lock_held(self, data, mode);
}
if (ret) {
@@ -607,7 +621,7 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data,
/* Resetting cctx's session never fail */
ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
}
- Py_END_CRITICAL_SECTION();
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -642,8 +656,9 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
}
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
- ret = compress_impl(self, NULL, mode);
+ PyMutex_Lock(&self->lock);
+
+ ret = compress_lock_held(self, NULL, mode);
if (ret) {
self->last_mode = mode;
@@ -654,7 +669,7 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
/* Resetting cctx's session never fail */
ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
}
- Py_END_CRITICAL_SECTION();
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -668,12 +683,12 @@ static PyMethodDef ZstdCompressor_methods[] = {
PyDoc_STRVAR(ZstdCompressor_last_mode_doc,
"The last mode used to this compressor object, its value can be .CONTINUE,\n"
".FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\n"
-"It can be used to get the current state of a compressor, such as, data flushed,\n"
-"a frame ended.");
+"It can be used to get the current state of a compressor, such as, data\n"
+"flushed, or a frame ended.");
static PyMemberDef ZstdCompressor_members[] = {
{"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode),
- Py_READONLY, ZstdCompressor_last_mode_doc},
+ Py_READONLY, ZstdCompressor_last_mode_doc},
{NULL}
};
diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c
index 58f9c9f804e..d084f0847c7 100644
--- a/Modules/_zstd/decompressor.c
+++ b/Modules/_zstd/decompressor.c
@@ -17,6 +17,7 @@ class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
#include "_zstdmodule.h"
#include "buffer.h"
#include "zstddict.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
#include <stdbool.h> // bool
#include <stddef.h> // offsetof()
@@ -45,6 +46,9 @@ typedef struct {
/* For ZstdDecompressor, 0 or 1.
1 means the end of the first frame has been reached. */
bool eof;
+
+ /* Lock to protect the decompression context */
+ PyMutex lock;
} ZstdDecompressor;
#define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
@@ -54,6 +58,7 @@ typedef struct {
static inline ZSTD_DDict *
_get_DDict(ZstdDict *self)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_DDict *ret;
/* Already created */
@@ -61,18 +66,17 @@ _get_DDict(ZstdDict *self)
return self->d_dict;
}
- Py_BEGIN_CRITICAL_SECTION(self);
if (self->d_dict == NULL) {
/* Create ZSTD_DDict instance from dictionary content */
char *dict_buffer = PyBytes_AS_STRING(self->dict_content);
Py_ssize_t dict_len = Py_SIZE(self->dict_content);
Py_BEGIN_ALLOW_THREADS
- self->d_dict = ZSTD_createDDict(dict_buffer,
- dict_len);
+ ret = ZSTD_createDDict(dict_buffer, dict_len);
Py_END_ALLOW_THREADS
+ self->d_dict = ret;
if (self->d_dict == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Failed to create a ZSTD_DDict instance from "
@@ -81,11 +85,7 @@ _get_DDict(ZstdDict *self)
}
}
- /* Don't lose any exception */
- ret = self->d_dict;
- Py_END_CRITICAL_SECTION();
-
- return ret;
+ return self->d_dict;
}
/* Set decompression parameters to decompression context */
@@ -95,7 +95,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
size_t zstd_ret;
PyObject *key, *value;
Py_ssize_t pos;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state == NULL) {
return -1;
}
@@ -112,7 +112,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
if (Py_TYPE(key) == mod_state->CParameter_type) {
PyErr_SetString(PyExc_TypeError,
"Key of decompression options dict should "
- "NOT be CompressionParameter.");
+ "NOT be a CompressionParameter attribute.");
return -1;
}
@@ -120,12 +120,11 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
int key_v = PyLong_AsInt(key);
if (key_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
- "Key of options dict should be a DecompressionParameter attribute.");
+ "Key of options dict should be either a "
+ "DecompressionParameter attribute or an int.");
return -1;
}
- // TODO(emmatyping): check bounds when there is a value error here for better
- // error message?
int value_v = PyLong_AsInt(value);
if (value_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
@@ -134,9 +133,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
}
/* Set parameter to compression context */
- Py_BEGIN_CRITICAL_SECTION(self);
zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
- Py_END_CRITICAL_SECTION();
/* Check error */
if (ZSTD_isError(zstd_ret)) {
@@ -147,12 +144,54 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
return 0;
}
+static int
+_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd,
+ _zstd_state *mod_state, int type)
+{
+ size_t zstd_ret;
+ if (type == DICT_TYPE_DIGESTED) {
+ /* Get ZSTD_DDict */
+ ZSTD_DDict *d_dict = _get_DDict(zd);
+ if (d_dict == NULL) {
+ return -1;
+ }
+ /* Reference a prepared dictionary */
+ zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
+ }
+ else if (type == DICT_TYPE_UNDIGESTED) {
+ /* Load a dictionary */
+ zstd_ret = ZSTD_DCtx_loadDictionary(
+ self->dctx,
+ PyBytes_AS_STRING(zd->dict_content),
+ Py_SIZE(zd->dict_content));
+ }
+ else if (type == DICT_TYPE_PREFIX) {
+ /* Load a prefix */
+ zstd_ret = ZSTD_DCtx_refPrefix(
+ self->dctx,
+ PyBytes_AS_STRING(zd->dict_content),
+ Py_SIZE(zd->dict_content));
+ }
+ else {
+ /* Impossible code path */
+ PyErr_SetString(PyExc_SystemError,
+ "load_d_dict() impossible code path");
+ return -1;
+ }
+
+ /* Check error */
+ if (ZSTD_isError(zstd_ret)) {
+ set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret);
+ return -1;
+ }
+ return 0;
+}
+
/* Load dictionary or prefix to decompression context */
static int
_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
{
- size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state == NULL) {
return -1;
}
@@ -168,7 +207,10 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
/* When decompressing, use digested dictionary by default. */
zd = (ZstdDict*)dict;
type = DICT_TYPE_DIGESTED;
- goto load;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
}
/* Check (ZstdDict, type) */
@@ -182,13 +224,16 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
else if (ret > 0) {
/* type == -1 may indicate an error. */
type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1));
- if (type == DICT_TYPE_DIGESTED ||
- type == DICT_TYPE_UNDIGESTED ||
- type == DICT_TYPE_PREFIX)
+ if (type == DICT_TYPE_DIGESTED
+ || type == DICT_TYPE_UNDIGESTED
+ || type == DICT_TYPE_PREFIX)
{
assert(type >= 0);
zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0);
- goto load;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
}
}
}
@@ -197,50 +242,6 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
PyErr_SetString(PyExc_TypeError,
"zstd_dict argument should be ZstdDict object.");
return -1;
-
-load:
- if (type == DICT_TYPE_DIGESTED) {
- /* Get ZSTD_DDict */
- ZSTD_DDict *d_dict = _get_DDict(zd);
- if (d_dict == NULL) {
- return -1;
- }
- /* Reference a prepared dictionary */
- Py_BEGIN_CRITICAL_SECTION(self);
- zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
- Py_END_CRITICAL_SECTION();
- }
- else if (type == DICT_TYPE_UNDIGESTED) {
- /* Load a dictionary */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_DCtx_loadDictionary(
- self->dctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
- }
- else if (type == DICT_TYPE_PREFIX) {
- /* Load a prefix */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_DCtx_refPrefix(
- self->dctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
- }
- else {
- /* Impossible code path */
- PyErr_SetString(PyExc_SystemError,
- "load_d_dict() impossible code path");
- return -1;
- }
-
- /* Check error */
- if (ZSTD_isError(zstd_ret)) {
- set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret);
- return -1;
- }
- return 0;
}
/*
@@ -268,8 +269,8 @@ load:
Note, decompressing "an empty input" in any case will make it > 0.
*/
static PyObject *
-decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
- Py_ssize_t max_length)
+decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
+ Py_ssize_t max_length)
{
size_t zstd_ret;
ZSTD_outBuffer out;
@@ -290,7 +291,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
}
@@ -339,10 +340,9 @@ error:
}
static void
-decompressor_reset_session(ZstdDecompressor *self)
+decompressor_reset_session_lock_held(ZstdDecompressor *self)
{
- // TODO(emmatyping): use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here
- // and ensure lock is always held
+ assert(PyMutex_IsLocked(&self->lock));
/* Reset variables */
self->in_begin = 0;
@@ -359,15 +359,18 @@ decompressor_reset_session(ZstdDecompressor *self)
}
static PyObject *
-stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length)
+stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data,
+ Py_ssize_t max_length)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
PyObject *ret = NULL;
int use_input_buffer;
/* Check .eof flag */
if (self->eof) {
- PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame.");
+ PyErr_SetString(PyExc_EOFError,
+ "Already at the end of a Zstandard frame.");
assert(ret == NULL);
return NULL;
}
@@ -456,7 +459,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
assert(in.pos == 0);
/* Decompress */
- ret = decompress_impl(self, &in, max_length);
+ ret = decompress_lock_held(self, &in, max_length);
if (ret == NULL) {
goto error;
}
@@ -484,8 +487,8 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
if (!use_input_buffer) {
/* Discard buffer if it's too small
(resizing it may needlessly copy the current contents) */
- if (self->input_buffer != NULL &&
- self->input_buffer_size < data_size)
+ if (self->input_buffer != NULL
+ && self->input_buffer_size < data_size)
{
PyMem_Free(self->input_buffer);
self->input_buffer = NULL;
@@ -517,7 +520,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
error:
/* Reset decompressor's states/session */
- decompressor_reset_session(self);
+ decompressor_reset_session_lock_held(self);
Py_CLEAR(ret);
return NULL;
@@ -555,6 +558,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
self->unused_data = NULL;
self->eof = 0;
self->dict = NULL;
+ self->lock = (PyMutex){0};
/* needs_input flag */
self->needs_input = 1;
@@ -562,7 +566,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
/* Decompression context */
self->dctx = ZSTD_createDCtx();
if (self->dctx == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Unable to create ZSTD_DCtx instance.");
@@ -608,6 +612,8 @@ ZstdDecompressor_dealloc(PyObject *ob)
ZSTD_freeDCtx(self->dctx);
}
+ assert(!PyMutex_IsLocked(&self->lock));
+
/* Py_CLEAR the dict after free decompression context */
Py_CLEAR(self->dict);
@@ -623,7 +629,6 @@ ZstdDecompressor_dealloc(PyObject *ob)
}
/*[clinic input]
-@critical_section
@getter
_zstd.ZstdDecompressor.unused_data
@@ -635,11 +640,14 @@ decompressed, unused input data after the frame. Otherwise this will be b''.
static PyObject *
_zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
-/*[clinic end generated code: output=f3a20940f11b6b09 input=5233800bef00df04]*/
+/*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/
{
PyObject *ret;
+ PyMutex_Lock(&self->lock);
+
if (!self->eof) {
+ PyMutex_Unlock(&self->lock);
return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
}
else {
@@ -656,6 +664,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
}
}
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -693,10 +702,9 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
{
PyObject *ret;
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
-
- ret = stream_decompress(self, data, max_length);
- Py_END_CRITICAL_SECTION();
+ PyMutex_Lock(&self->lock);
+ ret = stream_decompress_lock_held(self, data, max_length);
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -710,9 +718,10 @@ PyDoc_STRVAR(ZstdDecompressor_eof_doc,
"after that, an EOFError exception will be raised.");
PyDoc_STRVAR(ZstdDecompressor_needs_input_doc,
-"If the max_length output limit in .decompress() method has been reached, and\n"
-"the decompressor has (or may has) unconsumed input data, it will be set to\n"
-"False. In this case, pass b'' to .decompress() method may output further data.");
+"If the max_length output limit in .decompress() method has been reached,\n"
+"and the decompressor has (or may has) unconsumed input data, it will be set\n"
+"to False. In this case, passing b'' to the .decompress() method may output\n"
+"further data.");
static PyMemberDef ZstdDecompressor_members[] = {
{"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof),
diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c
index 7df187a6fa6..e3e9e5d0645 100644
--- a/Modules/_zstd/zstddict.c
+++ b/Modules/_zstd/zstddict.c
@@ -17,6 +17,7 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
#include "_zstdmodule.h"
#include "zstddict.h"
#include "clinic/zstddict.c.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
@@ -53,6 +54,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
self->dict_content = NULL;
self->d_dict = NULL;
self->dict_id = 0;
+ self->lock = (PyMutex){0};
/* ZSTD_CDict dict */
self->c_dicts = PyDict_New();
@@ -72,13 +74,15 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
at least 8 bytes */
if (Py_SIZE(self->dict_content) < 8) {
PyErr_SetString(PyExc_ValueError,
- "Zstandard dictionary content should at least 8 bytes.");
+ "Zstandard dictionary content should at least "
+ "8 bytes.");
goto error;
}
/* Get dict_id, 0 means "raw content" dictionary. */
- self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
- Py_SIZE(self->dict_content));
+ self->dict_id = ZSTD_getDictID_fromDict(
+ PyBytes_AS_STRING(self->dict_content),
+ Py_SIZE(self->dict_content));
/* Check validity for ordinary dictionary */
if (!is_raw && self->dict_id == 0) {
@@ -109,6 +113,8 @@ ZstdDict_dealloc(PyObject *ob)
ZSTD_freeDDict(self->d_dict);
}
+ assert(!PyMutex_IsLocked(&self->lock));
+
/* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */
Py_CLEAR(self->dict_content);
Py_CLEAR(self->c_dicts);
@@ -137,19 +143,22 @@ ZstdDict_str(PyObject *ob)
}
static PyMemberDef ZstdDict_members[] = {
- {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
- {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc},
+ {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY,
+ ZstdDict_dictid_doc},
+ {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content),
+ Py_READONLY, ZstdDict_dictcontent_doc},
{NULL}
};
/*[clinic input]
-@critical_section
@getter
_zstd.ZstdDict.as_digested_dict
Load as a digested dictionary to compressor.
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_digested_dict)
+
1. Some advanced compression parameters of compressor may be overridden
by parameters of digested dictionary.
2. ZstdDict has a digested dictionaries cache for each compression level.
@@ -160,19 +169,20 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digeste
static PyObject *
_zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=09b086e7a7320dbb input=585448c79f31f74a]*/
+/*[clinic end generated code: output=09b086e7a7320dbb input=ee45e1b4a48f6f2c]*/
{
return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
}
/*[clinic input]
-@critical_section
@getter
_zstd.ZstdDict.as_undigested_dict
Load as an undigested dictionary to compressor.
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_undigested_dict)
+
1. The advanced compression parameters of compressor will not be overridden.
2. Loading an undigested dictionary is costly. If load an undigested dictionary
multiple times, consider reusing a compressor object.
@@ -181,19 +191,20 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undiges
static PyObject *
_zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=43c7a989e6d4253a input=022b0829ffb1c220]*/
+/*[clinic end generated code: output=43c7a989e6d4253a input=d39210eedec76fed]*/
{
return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
}
/*[clinic input]
-@critical_section
@getter
_zstd.ZstdDict.as_prefix
Load as a prefix to compressor/decompressor.
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_prefix)
+
1. Prefix is compatible with long distance matching, while dictionary is not.
2. It only works for the first frame, then the compressor/decompressor will
return to no prefix state.
@@ -202,7 +213,7 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)
static PyObject *
_zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=6f7130c356595a16 input=09fb82a6a5407e87]*/
+/*[clinic end generated code: output=6f7130c356595a16 input=d59757b0b5a9551a]*/
{
return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
}
diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h
index e8a55a3670b..dcba0f21852 100644
--- a/Modules/_zstd/zstddict.h
+++ b/Modules/_zstd/zstddict.h
@@ -19,6 +19,9 @@ typedef struct {
PyObject *dict_content;
/* Dictionary id */
uint32_t dict_id;
+
+ /* Lock to protect the digested dictionaries */
+ PyMutex lock;
} ZstdDict;
#endif // !ZSTD_DICT_H
diff --git a/Modules/main.c b/Modules/main.c
index 2be194bdadf..2d7ed25f5f9 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -269,13 +269,14 @@ error:
static int
-pymain_start_pyrepl_no_main(void)
+pymain_start_pyrepl(int pythonstartup)
{
int res = 0;
PyObject *console = NULL;
PyObject *empty_tuple = NULL;
PyObject *kwargs = NULL;
PyObject *console_result = NULL;
+ PyObject *main_module = NULL;
PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main");
if (pyrepl == NULL) {
@@ -299,7 +300,13 @@ pymain_start_pyrepl_no_main(void)
res = pymain_exit_err_print();
goto done;
}
- if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) {
+ main_module = PyImport_AddModuleRef("__main__");
+ if (main_module == NULL) {
+ res = pymain_exit_err_print();
+ goto done;
+ }
+ if (!PyDict_SetItemString(kwargs, "mainmodule", main_module)
+ && !PyDict_SetItemString(kwargs, "pythonstartup", pythonstartup ? Py_True : Py_False)) {
console_result = PyObject_Call(console, empty_tuple, kwargs);
if (console_result == NULL) {
res = pymain_exit_err_print();
@@ -311,6 +318,7 @@ done:
Py_XDECREF(empty_tuple);
Py_XDECREF(console);
Py_XDECREF(pyrepl);
+ Py_XDECREF(main_module);
return res;
}
@@ -562,7 +570,7 @@ pymain_run_stdin(PyConfig *config)
int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, &cf);
return (run != 0);
}
- return pymain_run_module(L"_pyrepl", 0);
+ return pymain_start_pyrepl(0);
}
@@ -595,7 +603,7 @@ pymain_repl(PyConfig *config, int *exitcode)
*exitcode = (run != 0);
return;
}
- int run = pymain_start_pyrepl_no_main();
+ int run = pymain_start_pyrepl(1);
*exitcode = (run != 0);
return;
}
diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c
index 2240e2078b2..c5a8cead19a 100644
--- a/Modules/pwdmodule.c
+++ b/Modules/pwdmodule.c
@@ -301,18 +301,33 @@ pwd_getpwall_impl(PyObject *module)
struct passwd *p;
if ((d = PyList_New(0)) == NULL)
return NULL;
+
+#ifdef Py_GIL_DISABLED
+ static PyMutex getpwall_mutex = {0};
+ PyMutex_Lock(&getpwall_mutex);
+#endif
+ int failure = 0;
+ PyObject *v = NULL;
setpwent();
while ((p = getpwent()) != NULL) {
- PyObject *v = mkpwent(module, p);
+ v = mkpwent(module, p);
if (v == NULL || PyList_Append(d, v) != 0) {
- Py_XDECREF(v);
- Py_DECREF(d);
- endpwent();
- return NULL;
+ /* NOTE: cannot dec-ref here, while holding the mutex. */
+ failure = 1;
+ goto done;
}
Py_DECREF(v);
}
+
+done:
endpwent();
+#ifdef Py_GIL_DISABLED
+ PyMutex_Unlock(&getpwall_mutex);
+#endif
+ if (failure) {
+ Py_XDECREF(v);
+ Py_CLEAR(d);
+ }
return d;
}
#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-25 03:43:56 +0000