Skip to content

Commit dafda9b

Browse files
Issue python#11489: JSON decoder now accepts lone surrogates.
1 parent 60e361f commit dafda9b

4 files changed

Lines changed: 84 additions & 57 deletions

File tree

‎Lib/json/decoder.py‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@ def errmsg(msg, doc, pos, end=None):
6262

6363
DEFAULT_ENCODING = "utf-8"
6464

65+
def _decode_uXXXX(s, pos):
66+
esc = s[pos + 1:pos + 5]
67+
if len(esc) == 4 and esc[1] not in 'xX':
68+
try:
69+
return int(esc, 16)
70+
except ValueError:
71+
pass
72+
msg = "Invalid \\uXXXX escape"
73+
raise ValueError(errmsg(msg, s, pos))
74+
6575
def py_scanstring(s, end, encoding=None, strict=True,
6676
_b=BACKSLASH, _m=STRINGCHUNK.match):
6777
"""Scan the string s for a JSON string. End is the index of the
@@ -116,25 +126,16 @@ def py_scanstring(s, end, encoding=None, strict=True,
116126
end += 1
117127
else:
118128
# Unicode escape sequence
119-
esc = s[end + 1:end + 5]
120-
next_end = end + 5
121-
if len(esc) != 4:
122-
msg = "Invalid \\uXXXX escape"
123-
raise ValueError(errmsg(msg, s, end))
124-
uni = int(esc, 16)
129+
uni = _decode_uXXXX(s, end)
130+
end += 5
125131
# Check for surrogate pair on UCS-4 systems
126-
if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
127-
msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
128-
if not s[end + 5:end + 7] == '\\u':
129-
raise ValueError(errmsg(msg, s, end))
130-
esc2 = s[end + 7:end + 11]
131-
if len(esc2) != 4:
132-
raise ValueError(errmsg(msg, s, end))
133-
uni2 = int(esc2, 16)
134-
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
135-
next_end += 6
132+
if sys.maxunicode > 65535 and \
133+
0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u':
134+
uni2 = _decode_uXXXX(s, end + 1)
135+
if 0xdc00 <= uni2 <= 0xdfff:
136+
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
137+
end += 6
136138
char = unichr(uni)
137-
end = next_end
138139
# Append the unescaped character
139140
_append(char)
140141
return u''.join(chunks), end

‎Lib/json/tests/test_scanstring.py‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
class TestScanstring(object):
66
def test_scanstring(self):
77
scanstring = self.json.decoder.scanstring
8-
self.assertEqual(
9-
scanstring('"z\\ud834\\udd20x"', 1, None, True),
10-
(u'z\U0001d120x', 16))
11-
128
if sys.maxunicode == 65535:
139
self.assertEqual(
1410
scanstring(u'"z\U0001d120x"', 1, None, True),
@@ -94,6 +90,57 @@ def test_scanstring(self):
9490
scanstring('["Bad value", truth]', 2, None, True),
9591
(u'Bad value', 12))
9692

93+
def test_surrogates(self):
94+
scanstring = self.json.decoder.scanstring
95+
def assertScan(given, expect):
96+
self.assertEqual(scanstring(given, 1, None, True),
97+
(expect, len(given)))
98+
if not isinstance(given, unicode):
99+
given = unicode(given)
100+
self.assertEqual(scanstring(given, 1, None, True),
101+
(expect, len(given)))
102+
103+
assertScan('"z\\ud834\\u0079x"', u'z\ud834yx')
104+
assertScan('"z\\ud834\\udd20x"', u'z\U0001d120x')
105+
assertScan('"z\\ud834\\ud834\\udd20x"', u'z\ud834\U0001d120x')
106+
assertScan('"z\\ud834x"', u'z\ud834x')
107+
assertScan(u'"z\\ud834\udd20x12345"', u'z\ud834\udd20x12345')
108+
assertScan('"z\\udd20x"', u'z\udd20x')
109+
assertScan(u'"z\ud834\udd20x"', u'z\ud834\udd20x')
110+
assertScan(u'"z\ud834\\udd20x"', u'z\ud834\udd20x')
111+
assertScan(u'"z\ud834x"', u'z\ud834x')
112+
113+
def test_bad_escapes(self):
114+
scanstring = self.json.decoder.scanstring
115+
bad_escapes = [
116+
'"\\"',
117+
'"\\x"',
118+
'"\\u"',
119+
'"\\u0"',
120+
'"\\u01"',
121+
'"\\u012"',
122+
'"\\uz012"',
123+
'"\\u0z12"',
124+
'"\\u01z2"',
125+
'"\\u012z"',
126+
'"\\u0x12"',
127+
'"\\u0X12"',
128+
'"\\ud834\\"',
129+
'"\\ud834\\u"',
130+
'"\\ud834\\ud"',
131+
'"\\ud834\\udd"',
132+
'"\\ud834\\udd2"',
133+
'"\\ud834\\uzdd2"',
134+
'"\\ud834\\udzd2"',
135+
'"\\ud834\\uddz2"',
136+
'"\\ud834\\udd2z"',
137+
'"\\ud834\\u0x20"',
138+
'"\\ud834\\u0X20"',
139+
]
140+
for s in bad_escapes:
141+
with self.assertRaises(ValueError):
142+
scanstring(s, 1, None, True)
143+
97144
def test_issue3623(self):
98145
self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1,
99146
"xxx")

‎Misc/NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ Core and Builtins
1515
Library
1616
-------
1717

18+
- Issue #11489: JSON decoder now accepts lone surrogates.
19+
1820
- Fix test.test_support.bind_port() to not cause an error when Python was
1921
compiled on a system with SO_REUSEPORT defined in the headers but run on
2022
a system with an OS kernel that does not support that new socket option.

‎Modules/_json.c‎

Lines changed: 13 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -524,16 +524,10 @@ scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_s
524524
}
525525
#ifdef Py_UNICODE_WIDE
526526
/* Surrogate pair */
527-
if ((c & 0xfc00) == 0xd800) {
527+
if ((c & 0xfc00) == 0xd800 && end + 6 < len &&
528+
buf[next++] == '\\' &&
529+
buf[next++] == 'u') {
528530
Py_UNICODE c2 = 0;
529-
if (end + 6 >= len) {
530-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
531-
goto bail;
532-
}
533-
if (buf[next++] != '\\' || buf[next++] != 'u') {
534-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
535-
goto bail;
536-
}
537531
end += 6;
538532
/* Decode 4 hex digits */
539533
for (; next < end; next++) {
@@ -554,15 +548,10 @@ scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_s
554548
goto bail;
555549
}
556550
}
557-
if ((c2 & 0xfc00) != 0xdc00) {
558-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
559-
goto bail;
560-
}
561-
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
562-
}
563-
else if ((c & 0xfc00) == 0xdc00) {
564-
raise_errmsg("Unpaired low surrogate", pystr, end - 5);
565-
goto bail;
551+
if ((c2 & 0xfc00) == 0xdc00)
552+
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
553+
else
554+
end -= 6;
566555
}
567556
#endif
568557
}
@@ -703,16 +692,9 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
703692
}
704693
#ifdef Py_UNICODE_WIDE
705694
/* Surrogate pair */
706-
if ((c & 0xfc00) == 0xd800) {
695+
if ((c & 0xfc00) == 0xd800 && end + 6 < len &&
696+
buf[next++] == '\\' && buf[next++] == 'u') {
707697
Py_UNICODE c2 = 0;
708-
if (end + 6 >= len) {
709-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
710-
goto bail;
711-
}
712-
if (buf[next++] != '\\' || buf[next++] != 'u') {
713-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
714-
goto bail;
715-
}
716698
end += 6;
717699
/* Decode 4 hex digits */
718700
for (; next < end; next++) {
@@ -733,15 +715,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
733715
goto bail;
734716
}
735717
}
736-
if ((c2 & 0xfc00) != 0xdc00) {
737-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
738-
goto bail;
739-
}
740-
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
741-
}
742-
else if ((c & 0xfc00) == 0xdc00) {
743-
raise_errmsg("Unpaired low surrogate", pystr, end - 5);
744-
goto bail;
718+
if ((c2 & 0xfc00) == 0xdc00)
719+
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
720+
else
721+
end -= 6;
745722
}
746723
#endif
747724
}

0 commit comments

Comments
 (0)