Skip to content

Commit aeaae1e

Browse files
changjoon-parkchangjoon-park
committed
Accept surrogates in _json.encode_basestring{,_ascii}
encode_basestring/encode_basestring_ascii took PyUtf8StrRef, so json.dumps(str_with_lone_surrogate) raised UnicodeEncodeError at the Python/Rust boundary before write_json_string ran. CPython's encoder emits \uXXXX under ensure_ascii=True and passes raw WTF-8 otherwise. Switch to PyStrRef + s.as_wtf8(), matching scanstring in the same file. Rewrite write_json_string to accept &Wtf8 and iterate code_point_indices, emitting \uXXXX for surrogates in ascii mode and passing their bytes through otherwise. Stop escaping 0x7F in the ensure_ascii=False path (matches py_encode_basestring). Return Wtf8Buf via the checked from_bytes so invariant breaks panic instead of UB. Fuzzing also exposed two pre-existing ESCAPE_CHARS typos: 0x0B was "\u000" and 0x1B was "\u001" (both missing trailing 'b'). Fixed here. Verified byte-identical with CPython 3.13.4 over 16 manual + 10,000 random fuzz cases. Full test.test_json: 214 tests, 0 failures, 0 unexpected successes. Unmasks test_ascii_non_printable_encode and test_single_surrogate_encode. Decoder path is a follow-up.
1 parent 43ef2ea commit aeaae1e

3 files changed

Lines changed: 53 additions & 28 deletions

File tree

‎Lib/test/test_json/test_unicode.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,14 +138,12 @@ def test_object_pairs_hook_with_unicode(self):
138138

139139
class TestPyUnicode(TestUnicode, PyTest): pass
140140
class TestCUnicode(TestUnicode, CTest):
141-
@unittest.expectedFailure # TODO: RUSTPYTHON
142141
def test_ascii_non_printable_encode(self):
143142
return super().test_ascii_non_printable_encode()
144143

145-
@unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'")
144+
@unittest.skip("TODO: RUSTPYTHON; decode path still uses PyUtf8StrRef")
146145
def test_single_surrogate_decode(self):
147146
return super().test_single_surrogate_decode()
148147

149-
@unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'")
150148
def test_single_surrogate_encode(self):
151149
return super().test_single_surrogate_encode()

‎crates/stdlib/src/json.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -667,24 +667,30 @@ mod _json {
667667
}
668668
}
669669

670-
fn encode_string(s: &str, ascii_only: bool) -> String {
670+
fn encode_string(wtf8: &rustpython_common::wtf8::Wtf8, ascii_only: bool) -> Wtf8Buf {
671671
flame_guard!("_json::encode_string");
672-
let mut buf = Vec::<u8>::with_capacity(s.len() + 2);
673-
machinery::write_json_string(s, ascii_only, &mut buf)
672+
let mut buf = Vec::<u8>::with_capacity(wtf8.len() + 2);
673+
machinery::write_json_string(wtf8, ascii_only, &mut buf)
674674
// SAFETY: writing to a vec can't fail
675675
.unwrap_or_else(|_| unsafe { core::hint::unreachable_unchecked() });
676-
// SAFETY: we only output valid utf8 from write_json_string
677-
unsafe { String::from_utf8_unchecked(buf) }
676+
// write_json_string is designed to produce valid WTF-8 bytes:
677+
// - ASCII control characters and JSON-specials are written as ASCII escapes
678+
// - Valid Unicode scalars are written as UTF-8 (a subset of WTF-8)
679+
// - Lone surrogates (ascii_only=false branch only) pass through as the
680+
// input's WTF-8 byte sequences unchanged
681+
// Use the checked constructor so any violation of that invariant
682+
// surfaces as a panic during testing instead of undefined behavior.
683+
Wtf8Buf::from_bytes(buf).expect("write_json_string produced invalid WTF-8")
678684
}
679685

680686
#[pyfunction]
681-
fn encode_basestring(s: PyUtf8StrRef) -> String {
682-
encode_string(s.as_str(), false)
687+
fn encode_basestring(s: PyStrRef) -> Wtf8Buf {
688+
encode_string(s.as_wtf8(), false)
683689
}
684690

685691
#[pyfunction]
686-
fn encode_basestring_ascii(s: PyUtf8StrRef) -> String {
687-
encode_string(s.as_str(), true)
692+
fn encode_basestring_ascii(s: PyStrRef) -> Wtf8Buf {
693+
encode_string(s.as_wtf8(), true)
688694
}
689695

690696
fn py_decode_error(

‎crates/stdlib/src/json/machinery.rs‎

Lines changed: 37 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf};
3535

3636
static ESCAPE_CHARS: [&str; 0x20] = [
3737
"\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b",
38-
"\\t", "\\n", "\\u000", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012",
38+
"\\t", "\\n", "\\u000b", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012",
3939
"\\u0013", "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a",
40-
"\\u001", "\\u001c", "\\u001d", "\\u001e", "\\u001f",
40+
"\\u001b", "\\u001c", "\\u001d", "\\u001e", "\\u001f",
4141
];
4242

4343
// This bitset represents which bytes can be copied as-is to a JSON string (0)
@@ -72,30 +72,51 @@ fn json_escaped_char(c: u8) -> Option<&'static str> {
7272
}
7373
}
7474

75-
pub fn write_json_string<W: io::Write>(s: &str, ascii_only: bool, w: &mut W) -> io::Result<()> {
75+
pub fn write_json_string<W: io::Write>(wtf8: &Wtf8, ascii_only: bool, w: &mut W) -> io::Result<()> {
7676
w.write_all(b"\"")?;
7777
let mut write_start_idx = 0;
78-
let bytes = s.as_bytes();
78+
let bytes = wtf8.as_bytes();
7979
if ascii_only {
80-
for (idx, c) in s.char_indices() {
81-
if c.is_ascii() {
82-
if let Some(escaped) = json_escaped_char(c as u8) {
80+
for (idx, cp) in wtf8.code_point_indices() {
81+
if let Some(c) = cp.to_char() {
82+
// Valid Unicode scalar.
83+
if c.is_ascii() {
84+
if let Some(escaped) = json_escaped_char(c as u8) {
85+
w.write_all(&bytes[write_start_idx..idx])?;
86+
w.write_all(escaped.as_bytes())?;
87+
write_start_idx = idx + 1;
88+
}
89+
} else {
8390
w.write_all(&bytes[write_start_idx..idx])?;
84-
w.write_all(escaped.as_bytes())?;
85-
write_start_idx = idx + 1;
91+
write_start_idx = idx + c.len_utf8();
92+
// codepoints outside the BMP get 2 '\uxxxx' sequences to represent them
93+
for point in c.encode_utf16(&mut [0; 2]) {
94+
write!(w, "\\u{point:04x}")?;
95+
}
8696
}
8797
} else {
98+
// Lone surrogate code point (U+D800..U+DFFF).
99+
// WTF-8 encodes these as 3-byte sequences; skip those raw bytes
100+
// and emit a \uXXXX escape with the surrogate value.
88101
w.write_all(&bytes[write_start_idx..idx])?;
89-
write_start_idx = idx + c.len_utf8();
90-
// codepoints outside the BMP get 2 '\uxxxx' sequences to represent them
91-
for point in c.encode_utf16(&mut [0; 2]) {
92-
write!(w, "\\u{point:04x}")?;
93-
}
102+
write_start_idx = idx + 3;
103+
write!(w, "\\u{:04x}", cp.to_u32())?;
94104
}
95105
}
96106
} else {
97-
for (idx, c) in s.bytes().enumerate() {
98-
if let Some(escaped) = json_escaped_char(c) {
107+
// ensure_ascii is false: only JSON-required escapes (< 0x20, \, ")
108+
// are applied. 0x7F (DEL) is NOT escaped here, matching CPython's
109+
// py_encode_basestring. Multi-byte UTF-8 characters and WTF-8
110+
// surrogate sequences flow through unchanged via the trailing flush,
111+
// so surrogates round-trip as-is (matching CPython behavior).
112+
for (idx, c) in wtf8.as_bytes().iter().enumerate() {
113+
let escaped_opt: Option<&'static str> = match *c {
114+
x if x < 0x20 => Some(ESCAPE_CHARS[x as usize]),
115+
b'\\' => Some("\\\\"),
116+
b'\"' => Some("\\\""),
117+
_ => None,
118+
};
119+
if let Some(escaped) = escaped_opt {
99120
w.write_all(&bytes[write_start_idx..idx])?;
100121
w.write_all(escaped.as_bytes())?;
101122
write_start_idx = idx + 1;

0 commit comments

Comments
 (0)