Skip to content

Commit fa94e80

Browse files
committed
Manual py3k backport: [svn r74158] Issue python#6218: Make io.BytesIO and io.StringIO picklable.
1 parent fd42f30 commit fa94e80

5 files changed

Lines changed: 415 additions & 18 deletions

File tree

‎Lib/_pyio.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,11 @@ def __init__(self, initial_bytes=None):
773773
self._buffer = buf
774774
self._pos = 0
775775

776+
def __getstate__(self):
777+
if self.closed:
778+
raise ValueError("__getstate__ on closed file")
779+
return self.__dict__.copy()
780+
776781
def getvalue(self):
777782
"""Return the bytes value (contents) of the buffer
778783
"""

‎Lib/test/test_memoryio.py‎

Lines changed: 147 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import io
1313
import _pyio as pyio
1414
import sys
15+
import pickle
1516

1617
class MemorySeekTestMixin:
1718

@@ -352,6 +353,42 @@ def test_instance_dict_leak(self):
352353
memio = self.ioclass()
353354
memio.foo = 1
354355

356+
def test_pickling(self):
357+
buf = self.buftype("1234567890")
358+
memio = self.ioclass(buf)
359+
memio.foo = 42
360+
memio.seek(2)
361+
362+
class PickleTestMemIO(self.ioclass):
363+
def __init__(me, initvalue, foo):
364+
self.ioclass.__init__(me, initvalue)
365+
me.foo = foo
366+
# __getnewargs__ is undefined on purpose. This checks that PEP 307
367+
# is used to provide pickling support.
368+
369+
# Pickle expects the class to be on the module level. Here we use a
370+
# little hack to allow the PickleTestMemIO class to derive from
371+
# self.ioclass without having to define all combinations explictly on
372+
# the module-level.
373+
import __main__
374+
PickleTestMemIO.__module__ = '__main__'
375+
__main__.PickleTestMemIO = PickleTestMemIO
376+
submemio = PickleTestMemIO(buf, 80)
377+
submemio.seek(2)
378+
379+
# We only support pickle protocol 2 and onward since we use extended
380+
# __reduce__ API of PEP 307 to provide pickling support.
381+
for proto in range(2, pickle.HIGHEST_PROTOCOL):
382+
for obj in (memio, submemio):
383+
obj2 = pickle.loads(pickle.dumps(obj, protocol=proto))
384+
self.assertEqual(obj.getvalue(), obj2.getvalue())
385+
self.assertEqual(obj.__class__, obj2.__class__)
386+
self.assertEqual(obj.foo, obj2.foo)
387+
self.assertEqual(obj.tell(), obj2.tell())
388+
obj.close()
389+
self.assertRaises(ValueError, pickle.dumps, obj, proto)
390+
del __main__.PickleTestMemIO
391+
355392

356393
class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
357394

@@ -431,13 +468,26 @@ def test_bytes_array(self):
431468
self.assertEqual(memio.getvalue(), buf)
432469

433470

434-
class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
435-
buftype = unicode
436-
ioclass = pyio.StringIO
437-
UnsupportedOperation = pyio.UnsupportedOperation
438-
EOF = ""
471+
class TextIOTestMixin:
439472

440-
# TextIO-specific behaviour.
473+
def test_relative_seek(self):
474+
memio = self.ioclass()
475+
476+
self.assertRaises(IOError, memio.seek, -1, 1)
477+
self.assertRaises(IOError, memio.seek, 3, 1)
478+
self.assertRaises(IOError, memio.seek, -3, 1)
479+
self.assertRaises(IOError, memio.seek, -1, 2)
480+
self.assertRaises(IOError, memio.seek, 1, 1)
481+
self.assertRaises(IOError, memio.seek, 1, 2)
482+
483+
def test_textio_properties(self):
484+
memio = self.ioclass()
485+
486+
# These are just dummy values but we nevertheless check them for fear
487+
# of unexpected breakage.
488+
self.assertTrue(memio.encoding is None)
489+
self.assertEqual(memio.errors, "strict")
490+
self.assertEqual(memio.line_buffering, False)
441491

442492
def test_newlines_property(self):
443493
memio = self.ioclass(newline=None)
@@ -519,15 +569,13 @@ def test_newline_lf(self):
519569
def test_newline_cr(self):
520570
# newline="\r"
521571
memio = self.ioclass("a\nb\r\nc\rd", newline="\r")
522-
memio.seek(0)
523572
self.assertEqual(memio.read(), "a\rb\r\rc\rd")
524573
memio.seek(0)
525574
self.assertEqual(list(memio), ["a\r", "b\r", "\r", "c\r", "d"])
526575

527576
def test_newline_crlf(self):
528577
# newline="\r\n"
529578
memio = self.ioclass("a\nb\r\nc\rd", newline="\r\n")
530-
memio.seek(0)
531579
self.assertEqual(memio.read(), "a\r\nb\r\r\nc\rd")
532580
memio.seek(0)
533581
self.assertEqual(list(memio), ["a\r\n", "b\r\r\n", "c\rd"])
@@ -538,6 +586,28 @@ def test_issue5265(self):
538586
self.assertEqual(memio.read(5), "a\nb\n")
539587

540588

589+
class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin,
590+
TextIOTestMixin, unittest.TestCase):
591+
buftype = unicode
592+
ioclass = pyio.StringIO
593+
UnsupportedOperation = pyio.UnsupportedOperation
594+
EOF = ""
595+
596+
597+
class PyStringIOPickleTest(TextIOTestMixin, unittest.TestCase):
598+
"""Test if pickle restores properly the internal state of StringIO.
599+
"""
600+
buftype = unicode
601+
UnsupportedOperation = pyio.UnsupportedOperation
602+
EOF = ""
603+
604+
class ioclass(pyio.StringIO):
605+
def __new__(cls, *args, **kwargs):
606+
return pickle.loads(pickle.dumps(pyio.StringIO(*args, **kwargs)))
607+
def __init__(self, *args, **kwargs):
608+
pass
609+
610+
541611
class CBytesIOTest(PyBytesIOTest):
542612
ioclass = io.BytesIO
543613
UnsupportedOperation = io.UnsupportedOperation
@@ -547,6 +617,33 @@ class CBytesIOTest(PyBytesIOTest):
547617
)(PyBytesIOTest.test_bytes_array)
548618

549619

620+
def test_getstate(self):
621+
memio = self.ioclass()
622+
state = memio.__getstate__()
623+
self.assertEqual(len(state), 3)
624+
bytearray(state[0]) # Check if state[0] supports the buffer interface.
625+
self.assert_(isinstance(state[1], int))
626+
self.assert_(isinstance(state[2], dict) or state[2] is None)
627+
memio.close()
628+
self.assertRaises(ValueError, memio.__getstate__)
629+
630+
def test_setstate(self):
631+
# This checks whether __setstate__ does proper input validation.
632+
memio = self.ioclass()
633+
memio.__setstate__((b"no error", 0, None))
634+
memio.__setstate__((bytearray(b"no error"), 0, None))
635+
memio.__setstate__((b"no error", 0, {'spam': 3}))
636+
self.assertRaises(ValueError, memio.__setstate__, (b"", -1, None))
637+
self.assertRaises(TypeError, memio.__setstate__, ("unicode", 0, None))
638+
self.assertRaises(TypeError, memio.__setstate__, (b"", 0.0, None))
639+
self.assertRaises(TypeError, memio.__setstate__, (b"", 0, 0))
640+
self.assertRaises(TypeError, memio.__setstate__, (b"len-test", 0))
641+
self.assertRaises(TypeError, memio.__setstate__)
642+
self.assertRaises(TypeError, memio.__setstate__, 0)
643+
memio.close()
644+
self.assertRaises(ValueError, memio.__setstate__, (b"closed", 0, None))
645+
646+
550647
class CStringIOTest(PyStringIOTest):
551648
ioclass = io.StringIO
552649
UnsupportedOperation = io.UnsupportedOperation
@@ -565,9 +662,50 @@ def test_widechar(self):
565662
self.assertEqual(memio.tell(), len(buf) * 2)
566663
self.assertEqual(memio.getvalue(), buf + buf)
567664

665+
def test_getstate(self):
666+
memio = self.ioclass()
667+
state = memio.__getstate__()
668+
self.assertEqual(len(state), 4)
669+
self.assert_(isinstance(state[0], unicode))
670+
self.assert_(isinstance(state[1], str))
671+
self.assert_(isinstance(state[2], int))
672+
self.assert_(isinstance(state[3], dict) or state[3] is None)
673+
memio.close()
674+
self.assertRaises(ValueError, memio.__getstate__)
675+
676+
def test_setstate(self):
677+
# This checks whether __setstate__ does proper input validation.
678+
memio = self.ioclass()
679+
memio.__setstate__(("no error", "\n", 0, None))
680+
memio.__setstate__(("no error", "", 0, {'spam': 3}))
681+
self.assertRaises(ValueError, memio.__setstate__, ("", "f", 0, None))
682+
self.assertRaises(ValueError, memio.__setstate__, ("", "", -1, None))
683+
self.assertRaises(TypeError, memio.__setstate__, (b"", "", 0, None))
684+
# trunk is more tolerant than py3k on the type of the newline param
685+
#self.assertRaises(TypeError, memio.__setstate__, ("", b"", 0, None))
686+
self.assertRaises(TypeError, memio.__setstate__, ("", "", 0.0, None))
687+
self.assertRaises(TypeError, memio.__setstate__, ("", "", 0, 0))
688+
self.assertRaises(TypeError, memio.__setstate__, ("len-test", 0))
689+
self.assertRaises(TypeError, memio.__setstate__)
690+
self.assertRaises(TypeError, memio.__setstate__, 0)
691+
memio.close()
692+
self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None))
693+
694+
695+
class CStringIOPickleTest(PyStringIOPickleTest):
696+
UnsupportedOperation = io.UnsupportedOperation
697+
698+
class ioclass(io.StringIO):
699+
def __new__(cls, *args, **kwargs):
700+
return pickle.loads(pickle.dumps(io.StringIO(*args, **kwargs),
701+
protocol=2))
702+
def __init__(self, *args, **kwargs):
703+
pass
704+
568705

569706
def test_main():
570-
tests = [PyBytesIOTest, PyStringIOTest, CBytesIOTest, CStringIOTest]
707+
tests = [PyBytesIOTest, PyStringIOTest, CBytesIOTest, CStringIOTest,
708+
PyStringIOPickleTest, CStringIOPickleTest]
571709
support.run_unittest(*tests)
572710

573711
if __name__ == '__main__':

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,9 @@ Core and Builtins
418418
Library
419419
-------
420420

421+
- Issue #6218: io.StringIO and io.BytesIO instances are now picklable with
422+
protocol 2.
423+
421424
- Issue #7077: logging: SysLogHandler now treats Unicode as per RFC 5424.
422425

423426
- Issue #7099: Decimal.is_normal now returns True for numbers with exponent

‎Modules/_io/bytesio.c‎

Lines changed: 119 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,120 @@ bytesio_close(bytesio *self)
613613
Py_RETURN_NONE;
614614
}
615615

616+
/* Pickling support.
617+
618+
Note that only pickle protocol 2 and onward are supported since we use
619+
extended __reduce__ API of PEP 307 to make BytesIO instances picklable.
620+
621+
Providing support for protocol < 2 would require the __reduce_ex__ method
622+
which is notably long-winded when defined properly.
623+
624+
For BytesIO, the implementation would similar to one coded for
625+
object.__reduce_ex__, but slightly less general. To be more specific, we
626+
could call bytesio_getstate directly and avoid checking for the presence of
627+
a fallback __reduce__ method. However, we would still need a __newobj__
628+
function to use the efficient instance representation of PEP 307.
629+
*/
630+
631+
static PyObject *
632+
bytesio_getstate(bytesio *self)
633+
{
634+
PyObject *initvalue = bytesio_getvalue(self);
635+
PyObject *dict;
636+
PyObject *state;
637+
638+
if (initvalue == NULL)
639+
return NULL;
640+
if (self->dict == NULL) {
641+
Py_INCREF(Py_None);
642+
dict = Py_None;
643+
}
644+
else {
645+
dict = PyDict_Copy(self->dict);
646+
if (dict == NULL)
647+
return NULL;
648+
}
649+
650+
state = Py_BuildValue("(OnN)", initvalue, self->pos, dict);
651+
Py_DECREF(initvalue);
652+
return state;
653+
}
654+
655+
static PyObject *
656+
bytesio_setstate(bytesio *self, PyObject *state)
657+
{
658+
PyObject *result;
659+
PyObject *position_obj;
660+
PyObject *dict;
661+
Py_ssize_t pos;
662+
663+
assert(state != NULL);
664+
665+
/* We allow the state tuple to be longer than 3, because we may need
666+
someday to extend the object's state without breaking
667+
backward-compatibility. */
668+
if (!PyTuple_Check(state) || Py_SIZE(state) < 3) {
669+
PyErr_Format(PyExc_TypeError,
670+
"%.200s.__setstate__ argument should be 3-tuple, got %.200s",
671+
Py_TYPE(self)->tp_name, Py_TYPE(state)->tp_name);
672+
return NULL;
673+
}
674+
/* Reset the object to its default state. This is only needed to handle
675+
the case of repeated calls to __setstate__. */
676+
self->string_size = 0;
677+
self->pos = 0;
678+
679+
/* Set the value of the internal buffer. If state[0] does not support the
680+
buffer protocol, bytesio_write will raise the appropriate TypeError. */
681+
result = bytesio_write(self, PyTuple_GET_ITEM(state, 0));
682+
if (result == NULL)
683+
return NULL;
684+
Py_DECREF(result);
685+
686+
/* Set carefully the position value. Alternatively, we could use the seek
687+
method instead of modifying self->pos directly to better protect the
688+
object internal state against errneous (or malicious) inputs. */
689+
position_obj = PyTuple_GET_ITEM(state, 1);
690+
if (!PyIndex_Check(position_obj)) {
691+
PyErr_Format(PyExc_TypeError,
692+
"second item of state must be an integer, not %.200s",
693+
Py_TYPE(position_obj)->tp_name);
694+
return NULL;
695+
}
696+
pos = PyNumber_AsSsize_t(position_obj, PyExc_OverflowError);
697+
if (pos == -1 && PyErr_Occurred())
698+
return NULL;
699+
if (pos < 0) {
700+
PyErr_SetString(PyExc_ValueError,
701+
"position value cannot be negative");
702+
return NULL;
703+
}
704+
self->pos = pos;
705+
706+
/* Set the dictionary of the instance variables. */
707+
dict = PyTuple_GET_ITEM(state, 2);
708+
if (dict != Py_None) {
709+
if (!PyDict_Check(dict)) {
710+
PyErr_Format(PyExc_TypeError,
711+
"third item of state should be a dict, got a %.200s",
712+
Py_TYPE(dict)->tp_name);
713+
return NULL;
714+
}
715+
if (self->dict) {
716+
/* Alternatively, we could replace the internal dictionary
717+
completely. However, it seems more practical to just update it. */
718+
if (PyDict_Update(self->dict, dict) < 0)
719+
return NULL;
720+
}
721+
else {
722+
Py_INCREF(dict);
723+
self->dict = dict;
724+
}
725+
}
726+
727+
Py_RETURN_NONE;
728+
}
729+
616730
static void
617731
bytesio_dealloc(bytesio *self)
618732
{
@@ -637,9 +751,9 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
637751
if (self == NULL)
638752
return NULL;
639753

640-
self->string_size = 0;
641-
self->pos = 0;
642-
self->buf_size = 0;
754+
/* tp_alloc initializes all the fields to zero. So we don't have to
755+
initialize them here. */
756+
643757
self->buf = (char *)PyMem_Malloc(0);
644758
if (self->buf == NULL) {
645759
Py_DECREF(self);
@@ -712,6 +826,8 @@ static struct PyMethodDef bytesio_methods[] = {
712826
{"getvalue", (PyCFunction)bytesio_getvalue, METH_VARARGS, getval_doc},
713827
{"seek", (PyCFunction)bytesio_seek, METH_VARARGS, seek_doc},
714828
{"truncate", (PyCFunction)bytesio_truncate, METH_VARARGS, truncate_doc},
829+
{"__getstate__", (PyCFunction)bytesio_getstate, METH_NOARGS, NULL},
830+
{"__setstate__", (PyCFunction)bytesio_setstate, METH_O, NULL},
715831
{NULL, NULL} /* sentinel */
716832
};
717833

0 commit comments

Comments
 (0)