Skip to content

Commit aaeab4f

Browse files
youknowoneyouknowone
authored
Fix malformed Unicode error constructors (RustPython#8497)
* Fix malformed Unicode error constructors Replace message-only Unicode decode and encode errors with fully initialized exceptions, preserve source objects and failure ranges, and remove the obsolete constructors. Assisted-by: Codex:gpt-5 * Fix Windows Unicode CI regressions Assisted-by: Codex:gpt-5
1 parent c85b83f commit aaeab4f

14 files changed

Lines changed: 149 additions & 95 deletions

File tree

‎Lib/test/test_codeccallbacks.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,8 +1067,7 @@ def test_decodehelper_bug36819(self):
10671067
decoded = input.decode(enc, "test.bug36819")
10681068
self.assertEqual(decoded, 'abcdx' * 51)
10691069

1070-
# TODO: RUSTPYTHON
1071-
@unittest.expectedFailure
1070+
@unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON")
10721071
def test_encodehelper_bug36819(self):
10731072
handler = RepeatedPosReturn()
10741073
codecs.register_error("test.bug36819", handler.handle)

‎crates/capi/src/pyerrors.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create(
347347
unsafe { slice::from_raw_parts(object.cast::<u8>(), length) }.to_vec()
348348
};
349349

350-
let exc = vm.new_unicode_decode_error_real(
350+
let exc = vm.new_unicode_decode_error(
351351
vm.ctx.new_str(encoding),
352352
vm.ctx.new_bytes(bytes),
353353
start,

‎crates/host_env/src/posix.rs‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
use alloc::ffi::CString;
2-
#[cfg(all(unix, not(target_os = "redox")))]
32
use alloc::vec::Vec;
43
use core::ffi::CStr;
54
#[cfg(all(unix, not(target_os = "redox")))]
@@ -22,6 +21,12 @@ pub struct UnameInfo {
2221
pub machine: String,
2322
}
2423

24+
#[derive(Debug)]
25+
pub struct UnameDecodeError {
26+
pub bytes: Vec<u8>,
27+
pub error: core::str::Utf8Error,
28+
}
29+
2530
#[cfg(all(unix, not(target_os = "redox")))]
2631
#[derive(Clone, Copy, Debug)]
2732
pub struct StatVfsInfo {
@@ -354,14 +359,23 @@ pub fn fchownat(
354359
.map_err(std::io::Error::from)
355360
}
356361

357-
pub fn uname_info() -> Result<UnameInfo, core::str::Utf8Error> {
362+
pub fn uname_info() -> Result<UnameInfo, UnameDecodeError> {
363+
fn decode(value: &CStr) -> Result<String, UnameDecodeError> {
364+
core::str::from_utf8(value.to_bytes())
365+
.map(str::to_owned)
366+
.map_err(|error| UnameDecodeError {
367+
bytes: value.to_bytes().to_vec(),
368+
error,
369+
})
370+
}
371+
358372
let info = rustix::system::uname();
359373
Ok(UnameInfo {
360-
sysname: info.sysname().to_str()?.into(),
361-
nodename: info.nodename().to_str()?.into(),
362-
release: info.release().to_str()?.into(),
363-
version: info.version().to_str()?.into(),
364-
machine: info.machine().to_str()?.into(),
374+
sysname: decode(info.sysname())?,
375+
nodename: decode(info.nodename())?,
376+
release: decode(info.release())?,
377+
version: decode(info.version())?,
378+
machine: decode(info.machine())?,
365379
})
366380
}
367381

‎crates/stdlib/src/array.rs‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -657,7 +657,7 @@ pub mod array {
657657
impl ToPyResult for WideChar {
658658
fn to_pyresult(self, vm: &VirtualMachine) -> PyResult {
659659
Ok(CodePoint::try_from(self)
660-
.map_err(|e| vm.new_unicode_encode_error(e))?
660+
.map_err(|e| vm.new_value_error(e))?
661661
.to_pyobject(vm))
662662
}
663663
}
@@ -1755,8 +1755,17 @@ pub mod array {
17551755
})?,
17561756
MachineFormatCode::Utf16 { big_endian } => {
17571757
let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect();
1758-
let s = String::from_utf16(&utf16)
1759-
.map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?;
1758+
let s = String::from_utf16(&utf16).map_err(|_| {
1759+
let (index, reason) = invalid_utf16(&utf16).unwrap();
1760+
vm.new_unicode_decode_error(
1761+
vm.ctx
1762+
.new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }),
1763+
args.items.clone(),
1764+
index * 2,
1765+
index * 2 + 2,
1766+
vm.ctx.new_str(reason),
1767+
)
1768+
})?;
17601769
let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize());
17611770
array.frombytes_move(bytes);
17621771
}
@@ -1772,6 +1781,25 @@ pub mod array {
17721781
PyArray::from(array).into_ref_with_type(vm, cls)
17731782
}
17741783

1784+
fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> {
1785+
let mut index = 0;
1786+
while index < units.len() {
1787+
let unit = units[index];
1788+
if (0xd800..=0xdbff).contains(&unit) {
1789+
match units.get(index + 1) {
1790+
Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2,
1791+
Some(_) => return Some((index, "illegal UTF-16 surrogate")),
1792+
None => return Some((index, "unexpected end of data")),
1793+
}
1794+
} else if (0xdc00..=0xdfff).contains(&unit) {
1795+
return Some((index, "illegal encoding"));
1796+
} else {
1797+
index += 1;
1798+
}
1799+
}
1800+
None
1801+
}
1802+
17751803
// Register array.array as collections.abc.MutableSequence
17761804
pub(crate) fn module_exec(
17771805
vm: &VirtualMachine,

‎crates/stdlib/src/csv.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ mod _csv {
6464
bytes: &[u8],
6565
err: core::str::Utf8Error,
6666
) -> PyBaseExceptionRef {
67-
vm.new_unicode_decode_error_real(
67+
vm.new_unicode_decode_error(
6868
vm.ctx.new_str("utf-8"),
6969
vm.ctx.new_bytes(bytes.to_vec()),
7070
err.valid_up_to(),

‎crates/stdlib/src/socket.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2624,7 +2624,7 @@ mod _socket {
26242624
Some(ArgStrOrBytesLike::Buf(b)) => {
26252625
let bytes = b.borrow_buf();
26262626
let host_str = core::str::from_utf8(&bytes).map_err(|e| {
2627-
vm.new_unicode_decode_error_real(
2627+
vm.new_unicode_decode_error(
26282628
vm.ctx.new_str("utf-8"),
26292629
vm.ctx.new_bytes(bytes.to_vec()),
26302630
e.valid_up_to(),
@@ -2666,7 +2666,7 @@ mod _socket {
26662666
let bytes = b.borrow_buf();
26672667
core::str::from_utf8(&bytes)
26682668
.map_err(|e| {
2669-
vm.new_unicode_decode_error_real(
2669+
vm.new_unicode_decode_error(
26702670
vm.ctx.new_str("utf-8"),
26712671
vm.ctx.new_bytes(bytes.to_vec()),
26722672
e.valid_up_to(),

‎crates/stdlib/src/tkinter.rs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,18 @@ mod _tkinter {
162162

163163
if let Some(tcl_obj) = obj.downcast_ref::<TclObject>() {
164164
let c_str = unsafe { tk_sys::Tcl_GetString(tcl_obj.value) };
165-
let varname = unsafe { ffi::CStr::from_ptr(c_str as _) }
166-
.to_str()
167-
.map_err(|e| vm.new_unicode_decode_error(e.to_string()))?
165+
let bytes = unsafe { ffi::CStr::from_ptr(c_str as _) }.to_bytes();
166+
let varname = core::str::from_utf8(bytes)
167+
.map_err(|e| {
168+
vm.new_unicode_decode_error(
169+
vm.ctx.new_str("utf-8"),
170+
vm.ctx.new_bytes(bytes.to_vec()),
171+
e.valid_up_to(),
172+
e.error_len()
173+
.map_or(bytes.len(), |len| e.valid_up_to() + len),
174+
vm.ctx.new_str(e.to_string()),
175+
)
176+
})?
168177
.to_owned();
169178
return Ok(varname);
170179
}

‎crates/vm/src/codecs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -802,7 +802,7 @@ impl DecodeContext for PyDecodeContext<'_> {
802802
} else {
803803
vm.ctx.new_bytes(self.data.to_vec())
804804
};
805-
vm.new_unicode_decode_error_real(
805+
vm.new_unicode_decode_error(
806806
vm.ctx.new_str(self.encoding),
807807
data,
808808
byte_range.start,

‎crates/vm/src/function/fspath.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ impl FsPath {
125125

126126
pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
127127
rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
128-
vm.new_unicode_decode_error_real(
128+
vm.new_unicode_decode_error(
129129
vm.ctx.new_str("utf-8"),
130130
vm.ctx.new_bytes(b.to_vec()),
131131
e.valid_up_to(),

‎crates/vm/src/stdlib/_codecs.rs‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,23 @@ mod _codecs_windows {
382382
use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike};
383383
use rustpython_host_env::windows as host_windows;
384384

385+
fn string_from_utf16(
386+
encoding: &str,
387+
data: &[u8],
388+
wide: &[u16],
389+
vm: &VirtualMachine,
390+
) -> PyResult<String> {
391+
String::from_utf16(wide).map_err(|err| {
392+
vm.new_unicode_decode_error(
393+
vm.ctx.new_str(encoding),
394+
vm.ctx.new_bytes(data.to_vec()),
395+
0,
396+
data.len(),
397+
vm.ctx.new_str(format!("{encoding}_decode failed: {err}")),
398+
)
399+
})
400+
}
401+
385402
#[derive(FromArgs)]
386403
struct MbcsEncodeArgs {
387404
#[pyarg(positional)]
@@ -399,9 +416,7 @@ mod _codecs_windows {
399416
Some(s) => s,
400417
None => {
401418
// String contains surrogates - not encodable with mbcs
402-
return Err(vm.new_unicode_encode_error(
403-
"'mbcs' codec can't encode character: surrogates not allowed",
404-
));
419+
return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
405420
}
406421
};
407422
let char_len = args.s.char_len();
@@ -433,9 +448,7 @@ mod _codecs_windows {
433448
.map_err(|err| vm.new_os_error(format!("mbcs_encode failed: {err}")))?;
434449

435450
if errors == "strict" && used_default_char {
436-
return Err(vm.new_unicode_encode_error(
437-
"'mbcs' codec can't encode characters: invalid character",
438-
));
451+
return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
439452
}
440453

441454
buffer.truncate(result);
@@ -484,8 +497,7 @@ mod _codecs_windows {
484497
)
485498
.map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
486499
buffer.truncate(result);
487-
let s = String::from_utf16(&buffer)
488-
.map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
500+
let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
489501
return Ok((s, len));
490502
}
491503

@@ -500,8 +512,7 @@ mod _codecs_windows {
500512
)
501513
.map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
502514
buffer.truncate(result);
503-
let s = String::from_utf16(&buffer)
504-
.map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
515+
let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
505516

506517
Ok((s, len))
507518
}
@@ -523,9 +534,7 @@ mod _codecs_windows {
523534
Some(s) => s,
524535
None => {
525536
// String contains surrogates - not encodable with oem
526-
return Err(vm.new_unicode_encode_error(
527-
"'oem' codec can't encode character: surrogates not allowed",
528-
));
537+
return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
529538
}
530539
};
531540
let char_len = args.s.char_len();
@@ -557,9 +566,7 @@ mod _codecs_windows {
557566
.map_err(|err| vm.new_os_error(format!("oem_encode failed: {err}")))?;
558567

559568
if errors == "strict" && used_default_char {
560-
return Err(vm.new_unicode_encode_error(
561-
"'oem' codec can't encode characters: invalid character",
562-
));
569+
return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
563570
}
564571

565572
buffer.truncate(result);
@@ -609,8 +616,7 @@ mod _codecs_windows {
609616
)
610617
.map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
611618
buffer.truncate(result);
612-
let s = String::from_utf16(&buffer)
613-
.map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
619+
let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
614620
return Ok((s, len));
615621
}
616622

@@ -625,8 +631,7 @@ mod _codecs_windows {
625631
)
626632
.map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
627633
buffer.truncate(result);
628-
let s = String::from_utf16(&buffer)
629-
.map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
634+
let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
630635

631636
Ok((s, len))
632637
}
@@ -1024,7 +1029,7 @@ mod _codecs_windows {
10241029
}
10251030
}
10261031
let object = vm.ctx.new_bytes(data.to_vec());
1027-
return Err(vm.new_unicode_decode_error_real(
1032+
return Err(vm.new_unicode_decode_error(
10281033
encoding_str,
10291034
object,
10301035
fail_pos,
@@ -1115,7 +1120,7 @@ mod _codecs_windows {
11151120
}
11161121
"strict" => {
11171122
let object = vm.ctx.new_bytes(data.to_vec());
1118-
return Err(vm.new_unicode_decode_error_real(
1123+
return Err(vm.new_unicode_decode_error(
11191124
encoding_str,
11201125
object,
11211126
pos,
@@ -1126,7 +1131,7 @@ mod _codecs_windows {
11261131
_ => {
11271132
// Custom error handler
11281133
let object = vm.ctx.new_bytes(data.to_vec());
1129-
let exc = vm.new_unicode_decode_error_real(
1134+
let exc = vm.new_unicode_decode_error(
11301135
encoding_str.clone(),
11311136
object,
11321137
pos,

0 commit comments

Comments
 (0)