Skip to content

Commit d61541f

Browse files
vstinnervstinner
committed
WIP: [C API] Add PyTupleWriter API
1 parent e3262d9 commit d61541f

9 files changed

Lines changed: 649 additions & 128 deletions

File tree

‎Doc/c-api/tuple.rst‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,3 +293,101 @@ type.
293293
294294
.. versionchanged:: 3.13
295295
Now implemented as an alias to :c:func:`PyStructSequence_SetItem`.
296+
297+
298+
.. _pytuplewriter:
299+
300+
PyTupleWriter
301+
-------------
302+
303+
The ``PyTupleWriter`` API can be used to create a Python :class:`tuple` object
304+
when the final size is not known in advance.
305+
306+
.. versionadded:: next
307+
308+
.. c:type:: PyTupleWriter
309+
310+
A tuple writer instance.
311+
312+
The instance must be destroyed by :c:func:`PyTupleWriter_Finish` on
313+
success, or :c:func:`PyTupleWriter_Discard` on error.
314+
315+
The API is **not thread safe**. To share a writer with multiple threads, a
316+
critical section or a lock is needed.
317+
318+
.. c:function:: PyTupleWriter* PyTupleWriter_Create(Py_ssize_t size)
319+
320+
Create a tuple writer instance.
321+
322+
*size* must be greater than or equal to ``0``.
323+
324+
If *size* is greater than ``0``, allocate an internal buffer of *size* items
325+
and all items must be set with :c:func:`PyTupleWriter_SetItem` or
326+
:c:func:`PyTupleWriter_SetItems`.
327+
328+
Set an exception and return ``NULL`` on error.
329+
330+
.. c:function:: PyObject* PyTupleWriter_Finish(PyTupleWriter *writer)
331+
332+
Return the final Python :class:`str` object and destroy the writer instance.
333+
334+
Set an exception and return ``NULL`` on error.
335+
336+
The writer instance is invalid after this call.
337+
338+
.. c:function:: PyObject* PyTupleWriter_FinishWithSize(PyTupleWriter *writer, Py_ssize_t final_size)
339+
340+
Similar to :c:func:`PyTupleWriter_FinishWithSize`, but first truncates
341+
the tuple to *final_size* items.
342+
343+
On success, return a new tuple object.
344+
On error, set an exception and return ``NULL``.
345+
346+
.. c:function:: void PyTupleWriter_Discard(PyTupleWriter *writer)
347+
348+
Discard the internal Tuple buffer and destroy the writer instance.
349+
350+
If *writer* is ``NULL``, no operation is performed.
351+
352+
The writer instance is invalid after this call.
353+
354+
.. c:function:: void PyTupleWriter_Discard(PyTupleWriter *writer)
355+
356+
Discard the internal tuple buffer and destroy the writer instance.
357+
358+
If *writer* is ``NULL``, no operation is performed.
359+
360+
The writer instance is invalid after this call.
361+
362+
.. c:function:: Py_ssize_t PyTupleWriter_GetSize(PyTupleWriter *writer)
363+
364+
Get the writer size.
365+
366+
The function cannot fail.
367+
368+
.. c:function:: int PyTupleWriter_SetItem(PyTupleWriter *writer, Py_ssize_t index, PyObject *item)
369+
370+
Set an item of a tuple writer start at position *index*.
371+
Copy *item* reference.
372+
373+
An item must be set exactly once.
374+
375+
On success, return ``0``.
376+
On error, set an exception and return ``-1``.
377+
378+
.. c:function:: int PyTupleWriter_SetItems(PyTupleWriter *writer, Py_ssize_t index, PyObject *const *items, Py_ssize_t nitem)
379+
380+
Set *nitem* items of a tuple writer start at position *index*.
381+
Copy *items* references.
382+
383+
Items must be set exactly once.
384+
385+
On success, return ``0``.
386+
On error, set an exception and return ``-1``.
387+
388+
.. c:function:: int PyTupleWriter_Resize(PyTupleWriter **writer, Py_ssize_t new_size)
389+
390+
Resize the tuple writer to *new_size* items.
391+
392+
On success, return ``0``.
393+
On error, set an exception and return ``-1``.

‎Doc/whatsnew/3.16.rst‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1398,6 +1398,20 @@ New features
13981398
API.
13991399
(Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.)
14001400

1401+
* Add a new :ref:`PyTupleWriter API <pytuplewriter>` to create a :class:`tuple`
1402+
when the final size is not known in advance. Add functions:
1403+
1404+
* :c:func:`PyTupleWriter_Create`
1405+
* :c:func:`PyTupleWriter_Finish`
1406+
* :c:func:`PyTupleWriter_FinishWithSize`
1407+
* :c:func:`PyTupleWriter_Discard`
1408+
* :c:func:`PyTupleWriter_GetSize`
1409+
* :c:func:`PyTupleWriter_SetItem`
1410+
* :c:func:`PyTupleWriter_SetItems`
1411+
* :c:func:`PyTupleWriter_Resize`
1412+
1413+
(Contributed by Victor Stinner in :gh:`xxx`.)
1414+
14011415
* Add the :ref:`PyUnicodeWriter API <pyunicodewriter>` to the limited C API.
14021416
(Contributed by Victor Stinner in :gh:`161172`.)
14031417

‎Include/cpython/tupleobject.h‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,12 @@ static inline Py_ssize_t PyTuple_GET_SIZE(PyObject *op) {
3030

3131
/* Function *only* to be used to fill in brand new tuples */
3232
static inline void
33-
PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
33+
PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value)
34+
{
35+
// For backward compatibility, accept mutating a tuple even if
36+
// _PyObject_IsUniquelyReferenced(op) is false and/or tuple->ob_hash != -1.
37+
// See for example _PyTuple_Recycle(). Accept also setting an item which
38+
// was already set: the caller must Py_DECREF() the old value in this case.
3439
PyTupleObject *tuple = _PyTuple_CAST(op);
3540
assert(0 <= index);
3641
assert(index < Py_SIZE(tuple));
@@ -42,3 +47,29 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
4247
PyAPI_FUNC(PyObject*) PyTuple_FromArray(
4348
PyObject *const *array,
4449
Py_ssize_t size);
50+
51+
52+
/* --- PyTupleWriter API -------------------------------------------------- */
53+
54+
struct PyTupleWriter;
55+
typedef struct PyTupleWriter PyTupleWriter;
56+
57+
PyAPI_FUNC(PyTupleWriter*) PyTupleWriter_Create(Py_ssize_t size);
58+
PyAPI_FUNC(PyObject*) PyTupleWriter_Finish(PyTupleWriter *writer);
59+
PyAPI_FUNC(PyObject*) PyTupleWriter_FinishWithSize(
60+
PyTupleWriter *writer,
61+
Py_ssize_t final_size);
62+
PyAPI_FUNC(void) PyTupleWriter_Discard(PyTupleWriter *writer);
63+
PyAPI_FUNC(Py_ssize_t) PyTupleWriter_GetSize(PyTupleWriter *writer);
64+
PyAPI_FUNC(int) PyTupleWriter_SetItem(
65+
PyTupleWriter *writer,
66+
Py_ssize_t index,
67+
PyObject *item);
68+
PyAPI_FUNC(int) PyTupleWriter_SetItems(
69+
PyTupleWriter *writer,
70+
Py_ssize_t index,
71+
PyObject *const *items,
72+
Py_ssize_t nitem);
73+
PyAPI_FUNC(int) PyTupleWriter_Resize(
74+
PyTupleWriter **writer,
75+
Py_ssize_t new_size);

‎Include/internal/pycore_tuple.h‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,84 @@ _PyTuple_Recycle(PyObject *op)
7676
#endif
7777
#define _PyTuple_HASH_EMPTY (_PyTuple_HASH_XXPRIME_5 + (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL))
7878

79+
80+
#ifndef NDEBUG
81+
// Check that a tuple can be mutated in-place.
82+
// Usage: assert(_PyTuple_CanMutate(v));
83+
static int
84+
_PyTuple_CanMutate(PyTupleObject *tuple)
85+
{
86+
assert(PyTuple_Check(tuple));
87+
88+
// Do not check _PyObject_IsUniquelyReferenced(): the function is called
89+
// by _PyTupleWriter_CheckConsistency() and a tuple writer can be used by
90+
// multiple threads using external locking.
91+
assert(Py_REFCNT(tuple) == 1);
92+
93+
// The empty tuple must not be modified
94+
assert(1 <= Py_SIZE(tuple));
95+
assert(tuple != &_Py_SINGLETON(tuple_empty));
96+
97+
// The hash value must not be cached yet. Otherwise, it means that the
98+
// tuple object was already used in Python somehow (ex: as a dictionary
99+
// key).
100+
assert(tuple->ob_hash == -1);
101+
102+
return 1;
103+
}
104+
#endif
105+
106+
107+
/* --- PyTupleWriter API -------------------------------------------------- */
108+
109+
#ifndef NDEBUG
110+
static inline int
111+
_PyTupleWriter_CheckConsistency(PyTupleWriter *writer)
112+
{
113+
assert(writer != NULL);
114+
assert(PyTuple_CheckExact(_PyObject_CAST(writer)));
115+
PyTupleObject *tuple = (PyTupleObject*)writer;
116+
assert(0 <= Py_SIZE(tuple));
117+
if (Py_SIZE(tuple) != 0) {
118+
// Do not use _PyObject_IsUniquelyReferenced(): the caller can have
119+
// its own lock to prevent a writer from being used by two threads
120+
// at the same time.
121+
assert(Py_REFCNT(tuple) == 1);
122+
123+
assert(!_PyObject_GC_IS_TRACKED(tuple));
124+
}
125+
else {
126+
assert(tuple == &_Py_SINGLETON(tuple_empty));
127+
}
128+
return 1;
129+
}
130+
#endif
131+
132+
133+
static inline void
134+
_PyTupleWriter_SET_ITEM_STEAL(PyTupleWriter *writer,
135+
Py_ssize_t index, PyObject *item)
136+
{
137+
assert(_PyTupleWriter_CheckConsistency(writer));
138+
PyTupleObject *tuple = _PyTuple_CAST(writer);
139+
140+
assert(_PyTuple_CanMutate(tuple));
141+
assert(0 <= index && index < Py_SIZE(tuple));
142+
assert(tuple->ob_item[index] == NULL);
143+
assert(item != NULL);
144+
145+
tuple->ob_item[index] = item;
146+
}
147+
148+
149+
static inline void
150+
_PyTupleWriter_SET_ITEM(PyTupleWriter *writer,
151+
Py_ssize_t index, PyObject *item)
152+
{
153+
assert(item != NULL);
154+
return _PyTupleWriter_SET_ITEM_STEAL(writer, index, Py_NewRef(item));
155+
}
156+
79157
#ifdef __cplusplus
80158
}
81159
#endif

‎Modules/_tkinter.c‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ Copyright (C) 1994 Steen Lumholt.
3131
#endif
3232

3333
#include "pycore_long.h" // _PyLong_IsNegative()
34+
#include "pycore_tuple.h" // _PyTupleWriter_SET_ITEM()
3435
#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8String
3536

3637
#include <signal.h> // SIGINT
@@ -3401,7 +3402,7 @@ Tkapp_Dealloc(PyObject *op)
34013402
/**** Tkinter Module ****/
34023403

34033404
typedef struct {
3404-
PyObject* tuple;
3405+
PyTupleWriter *writer;
34053406
Py_ssize_t size; /* current size */
34063407
Py_ssize_t maxsize; /* allocated size */
34073408
} FlattenContext;
@@ -3419,7 +3420,7 @@ _bump(FlattenContext* context, Py_ssize_t size)
34193420

34203421
context->maxsize = maxsize;
34213422

3422-
return _PyTuple_Resize(&context->tuple, maxsize) >= 0;
3423+
return PyTupleWriter_Resize(&context->writer, maxsize) >= 0;
34233424
}
34243425

34253426
static int
@@ -3449,8 +3450,7 @@ _flatten1(FlattenContext* context, PyObject* item, int depth)
34493450
if (context->size + 1 > context->maxsize &&
34503451
!_bump(context, 1))
34513452
return 0;
3452-
PyTuple_SET_ITEM(context->tuple,
3453-
context->size++, Py_NewRef(o));
3453+
_PyTupleWriter_SET_ITEM(context->writer, context->size++, o);
34543454
}
34553455
}
34563456
} else {
@@ -3480,21 +3480,18 @@ _tkinter__flatten(PyObject *module, PyObject *item)
34803480
if (context.maxsize == 0)
34813481
return PyTuple_New(0);
34823482

3483-
context.tuple = PyTuple_New(context.maxsize);
3484-
if (!context.tuple)
3483+
context.writer = PyTupleWriter_Create(context.maxsize);
3484+
if (!context.writer) {
34853485
return NULL;
3486-
3486+
}
34873487
context.size = 0;
34883488

34893489
if (!_flatten1(&context, item, 0)) {
3490-
Py_XDECREF(context.tuple);
3490+
PyTupleWriter_Discard(context.writer);
34913491
return NULL;
34923492
}
34933493

3494-
if (_PyTuple_Resize(&context.tuple, context.size))
3495-
return NULL;
3496-
3497-
return context.tuple;
3494+
return PyTupleWriter_FinishWithSize(context.writer, context.size);
34983495
}
34993496

35003497
/*[clinic input]

‎Modules/itertoolsmodule.c‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -207,25 +207,24 @@ batched_next(PyObject *op)
207207
Py_ssize_t n = FT_ATOMIC_LOAD_SSIZE_RELAXED(bo->batch_size);
208208
PyObject *it = bo->it;
209209
PyObject *item;
210-
PyObject *result;
211210

212211
if (n < 0) {
213212
return NULL;
214213
}
215-
result = PyTuple_New(n);
216-
if (result == NULL) {
214+
215+
PyTupleWriter *writer = PyTupleWriter_Create(n);
216+
if (writer == NULL) {
217217
return NULL;
218218
}
219219
iternextfunc iternext = *Py_TYPE(it)->tp_iternext;
220-
PyObject **items = _PyTuple_ITEMS(result);
221220
for (i=0 ; i < n ; i++) {
222221
item = iternext(it);
223222
if (item == NULL) {
224223
goto null_item;
225224
}
226-
items[i] = item;
225+
_PyTupleWriter_SET_ITEM_STEAL(writer, i, item);
227226
}
228-
return result;
227+
return PyTupleWriter_Finish(writer);
229228

230229
null_item:
231230
if (PyErr_Occurred()) {
@@ -235,7 +234,7 @@ batched_next(PyObject *op)
235234
#ifndef Py_GIL_DISABLED
236235
Py_CLEAR(bo->it);
237236
#endif
238-
Py_DECREF(result);
237+
PyTupleWriter_Discard(writer);
239238
return NULL;
240239
}
241240
PyErr_Clear();
@@ -245,20 +244,19 @@ batched_next(PyObject *op)
245244
#ifndef Py_GIL_DISABLED
246245
Py_CLEAR(bo->it);
247246
#endif
248-
Py_DECREF(result);
247+
PyTupleWriter_Discard(writer);
249248
return NULL;
250249
}
251250
if (bo->strict) {
252251
FT_ATOMIC_STORE_SSIZE_RELAXED(bo->batch_size, -1);
253252
#ifndef Py_GIL_DISABLED
254253
Py_CLEAR(bo->it);
255254
#endif
256-
Py_DECREF(result);
255+
PyTupleWriter_Discard(writer);
257256
PyErr_SetString(PyExc_ValueError, "batched(): incomplete batch");
258257
return NULL;
259258
}
260-
_PyTuple_Resize(&result, i);
261-
return result;
259+
return PyTupleWriter_FinishWithSize(writer, i);
262260
}
263261

264262
static PyType_Slot batched_slots[] = {

0 commit comments

Comments
 (0)