Skip to content

Commit 65f30fd

Browse files
authored
Return character counts from the UTF-8 encoder fast path (#9205)
Assisted-by: Codex:GPT-6
1 parent 3a0ef4d commit 65f30fd

2 files changed

Lines changed: 5 additions & 1 deletion

File tree

‎crates/vm/src/stdlib/_codecs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ mod _codecs {
241241
.as_ref()
242242
.is_some_and(|s| s.is(identifier!(vm, surrogatepass)))
243243
{
244-
return Ok((args.str.as_bytes().to_vec(), args.str.byte_len()));
244+
return Ok((args.str.as_bytes().to_vec(), args.str.char_len()));
245245
}
246246
do_codec!(utf8::encode, args, vm)
247247
}

‎extra_tests/snippets/stdlib_codecs_wide.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121

2222
import _codecs
2323

24+
# Encoder counts are characters consumed, not the size of the encoded bytes.
25+
assert _codecs.utf_8_encode("é😀") == (b"\xc3\xa9\xf0\x9f\x98\x80", 2)
26+
assert _codecs.utf_8_encode("a\ud800b", "surrogatepass") == (b"a\xed\xa0\x80b", 3)
27+
2428
assert _codecs.escape_encode(b"a\tb") == (b"a\\tb", 3)
2529
assert _codecs.escape_decode(rb"a\tb") == (b"a\tb", 4)
2630
assert _codecs.utf_16_ex_decode(b"\xff\xfeh\x00i\x00", "strict", 0, True)[0] == "hi"

0 commit comments

Comments
 (0)