Skip to content

Commit 0997ab6

Browse files
author
James William Pye
committed
Add process_chunk function.
Provides a performance increase by pre-allocating the whole list for a given chunk and filling it in a C-loop. Future improvements will include a tuple to be shaped before setting it in the list. (To accommodate for pg_type.Row.from_sequence())
1 parent 9d136ed commit 0997ab6

3 files changed

Lines changed: 117 additions & 20 deletions

File tree

‎postgresql/driver/pq3.py‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -238,24 +238,19 @@ def _process_tuple_chunk_Row(self, x):
238238
Process the Tuple messages in `x`.
239239
"""
240240
return [
241-
pg_types.Row.from_sequence(
242-
self._output_attmap,
243-
pg_typio.process_tuple(
244-
self._output_io, y,
245-
self._raise_column_tuple_error
246-
),
247-
) for y in x
241+
pg_types.Row.from_sequence(self._output_attmap, y)
242+
for y in pg_typio.process_chunk(
243+
self._output_io, x, self._raise_column_tuple_error
244+
)
248245
]
249246

250247
def _process_tuple_chunk(self, x):
251248
"""
252249
Process the Tuple messages in `x`.
253250
"""
254-
return [
255-
pg_typio.process_tuple(
256-
self._output_io, y, self._raise_column_tuple_error
257-
) for y in x
258-
]
251+
return pg_typio.process_chunk(
252+
self._output_io, x, self._raise_column_tuple_error
253+
)
259254

260255
def _raise_column_tuple_error(self, procs, tup, itemnum):
261256
'for column processing'
@@ -407,7 +402,7 @@ def _process_chunk(self, x):
407402
))
408403

409404
class MultiXactStream(Chunks):
410-
chunksize = 256
405+
chunksize = 512
411406
# only tuple streams
412407
_process_chunk = Output._process_tuple_chunk_Row
413408

‎postgresql/protocol/optimized/typio.c‎

Lines changed: 103 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
mFUNC(process_tuple, METH_VARARGS, \
77
"process the items in the second argument " \
88
"with the corresponding items in the first argument.") \
9+
mFUNC(process_chunk, METH_VARARGS, \
10+
"process the items of the chunk given as the second argument " \
11+
"with the corresponding items in the first argument.") \
912
mFUNC(int2_pack, METH_O, "PyInt to serialized, int2") \
1013
mFUNC(int2_unpack, METH_O, "PyInt from serialized, int2") \
1114
mFUNC(int4_pack, METH_O, "PyInt to serialized, int4") \
@@ -24,7 +27,7 @@
2427
mFUNC(swap_uint4_unpack, METH_O, "PyInt from swapped serialized, int4") \
2528

2629
/*
27-
* Define the swap functionality.
30+
* Define the swap functionality for those endians.
2831
*/
2932
#define swap2(CP) do{register char c; \
3033
c=CP[1];CP[1]=CP[0];CP[0]=c;\
@@ -387,14 +390,11 @@ swap_uint4_unpack(PyObject *self, PyObject *arg)
387390
* calling the third object in cases of failure to generalize the exception.
388391
*/
389392
static PyObject *
390-
process_tuple(PyObject *self, PyObject *args)
393+
_process_tuple(PyObject *procs, PyObject *tup, PyObject *fail)
391394
{
392-
PyObject *tup, *procs, *fail, *rob;
395+
PyObject *rob;
393396
Py_ssize_t len, i;
394397

395-
if (!PyArg_ParseTuple(args, "OOO", &procs, &tup, &fail))
396-
return(NULL);
397-
398398
if (!PyTuple_CheckExact(procs))
399399
{
400400
PyErr_SetString(
@@ -539,3 +539,100 @@ process_tuple(PyObject *self, PyObject *args)
539539

540540
return(rob);
541541
}
542+
543+
/*
544+
* process the tuple with the associated callables while
545+
* calling the third object in cases of failure to generalize the exception.
546+
*/
547+
static PyObject *
548+
process_tuple(PyObject *self, PyObject *args)
549+
{
550+
PyObject *tup, *procs, *fail;
551+
552+
if (!PyArg_ParseTuple(args, "OOO", &procs, &tup, &fail))
553+
return(NULL);
554+
555+
return(_process_tuple(procs, tup, fail));
556+
}
557+
558+
static PyObject *
559+
_process_chunk_new_list(PyObject *procs, PyObject *tupc, PyObject *fail)
560+
{
561+
PyObject *rob;
562+
Py_ssize_t i, len;
563+
564+
rob = PyObject_CallFunctionObjArgs((PyObject *) &PyList_Type, tupc, NULL);
565+
if (rob == NULL)
566+
return(NULL);
567+
len = PyList_Size(rob);
568+
569+
for (i = 0; i < len; ++i)
570+
{
571+
PyObject *tup, *r;
572+
/*
573+
* If it's Py_None, that means it's NULL. No processing necessary.
574+
*/
575+
tup = PyList_GetItem(rob, i);
576+
r = _process_tuple(procs, tup, fail);
577+
if (r == NULL)
578+
{
579+
Py_DECREF(rob);
580+
return(NULL);
581+
}
582+
PyList_SetItem(rob, i, r);
583+
}
584+
585+
return(rob);
586+
}
587+
588+
static PyObject *
589+
_process_chunk_from_list(PyObject *procs, PyObject *tupc, PyObject *fail)
590+
{
591+
PyObject *rob;
592+
Py_ssize_t i, len;
593+
594+
len = PyList_GET_SIZE(tupc);
595+
rob = PyList_New(len);
596+
if (rob == NULL)
597+
return(NULL);
598+
599+
for (i = 0; i < len; ++i)
600+
{
601+
PyObject *tup, *r;
602+
/*
603+
* If it's Py_None, that means it's NULL. No processing necessary.
604+
*/
605+
tup = PyList_GET_ITEM(tupc, i);
606+
r = _process_tuple(procs, tup, fail);
607+
if (r == NULL)
608+
{
609+
Py_DECREF(rob);
610+
return(NULL);
611+
}
612+
PyList_SET_ITEM(rob, i, r);
613+
}
614+
615+
return(rob);
616+
}
617+
618+
/*
619+
* process the chunk of tuples with the associated callables while
620+
* calling the third object in cases of failure to generalize the exception.
621+
*/
622+
static PyObject *
623+
process_chunk(PyObject *self, PyObject *args)
624+
{
625+
PyObject *tupc, *procs, *fail;
626+
627+
if (!PyArg_ParseTuple(args, "OOO", &procs, &tupc, &fail))
628+
return(NULL);
629+
630+
if (PyList_Check(tupc))
631+
{
632+
return(_process_chunk_from_list(procs, tupc, fail));
633+
}
634+
else
635+
{
636+
return(_process_chunk_new_list(procs, tupc, fail));
637+
}
638+
}

‎postgresql/protocol/typio.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,13 @@ def process_tuple(procs, tup, exception_handler):
484484
raise RuntimeError("process_tuple exception handler failed to raise")
485485
return r
486486

487+
def process_chunk(procs, tupc, fail):
488+
return [
489+
process_tuple(procs, x, fail) for x in tupc
490+
]
491+
487492
try:
488-
from .optimized import process_tuple
493+
from .optimized import process_tuple, process_chunk
489494
except ImportError:
490495
pass
491496

0 commit comments

Comments
 (0)