Skip to content

Commit 485fb56

Browse files
author
Victor Stinner
committed
Issue #8383: pickle and pickletools use surrogatepass error handler when
encoding unicode as utf8 to support lone surrogates and stay compatible with Python 2.x and 3.0
1 parent 3606760 commit 485fb56

5 files changed

Lines changed: 14 additions & 6 deletions

File tree

‎Lib/pickle.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@ def save_bytes(self, obj, pack=struct.pack):
499499

500500
def save_str(self, obj, pack=struct.pack):
501501
if self.bin:
502-
encoded = obj.encode('utf-8')
502+
encoded = obj.encode('utf-8', 'surrogatepass')
503503
n = len(encoded)
504504
self.write(BINUNICODE + pack("<i", n) + encoded)
505505
else:
@@ -966,7 +966,7 @@ def load_unicode(self):
966966

967967
def load_binunicode(self):
968968
len = mloads(b'i' + self.read(4))
969-
self.append(str(self.read(len), 'utf-8'))
969+
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
970970
dispatch[BINUNICODE[0]] = load_binunicode
971971

972972
def load_short_binstring(self):

‎Lib/pickletools.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -469,7 +469,7 @@ def read_unicodestring4(f):
469469
raise ValueError("unicodestring4 byte count < 0: %d" % n)
470470
data = f.read(n)
471471
if len(data) == n:
472-
return str(data, 'utf-8')
472+
return str(data, 'utf-8', 'surrogatepass')
473473
raise ValueError("expected %d bytes in a unicodestring4, but only %d "
474474
"remain" % (n, len(data)))
475475

‎Lib/test/pickletester.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -515,7 +515,9 @@ def test_insecure_strings(self):
515515

516516
def test_unicode(self):
517517
endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
518-
'<\\>', '<\\\U00012345>']
518+
'<\\>', '<\\\U00012345>',
519+
# surrogates
520+
'<\udc80>']
519521
for proto in protocols:
520522
for u in endcases:
521523
p = self.dumps(u, proto)

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,10 @@ C-API
312312
Library
313313
-------
314314

315+
- Issue #8383: pickle and pickletools use surrogatepass error handler when
316+
encoding unicode as utf8 to support lone surrogates and stay compatible with
317+
Python 2.x and 3.0
318+
315319
- Issue #7585: difflib context and unified diffs now place a tab between
316320
filename and date, conforming to the 'standards' they were originally
317321
designed to follow. This improves compatibility with patch tools.

‎Modules/_pickle.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1227,7 +1227,9 @@ save_unicode(PicklerObject *self, PyObject *obj)
12271227
if (self->bin) {
12281228
char pdata[5];
12291229

1230-
encoded = PyUnicode_AsUTF8String(obj);
1230+
encoded = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(obj),
1231+
PyUnicode_GET_SIZE(obj),
1232+
"surrogatepass");
12311233
if (encoded == NULL)
12321234
goto error;
12331235

@@ -3352,7 +3354,7 @@ load_binunicode(UnpicklerObject *self)
33523354
if (unpickler_read(self, &s, size) < 0)
33533355
return -1;
33543356

3355-
str = PyUnicode_DecodeUTF8(s, size, NULL);
3357+
str = PyUnicode_DecodeUTF8(s, size, "surrogatepass");
33563358
if (str == NULL)
33573359
return -1;
33583360

0 commit comments

Comments
 (0)