Skip to content

Replace malformed unicode error helper calls with the _real variants #8359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 2 commits into from
Jul 26, 2026
Merged

Replace malformed unicode error helper calls with the _real variants #8359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02c70c4
Replace malformed unicode error helper calls with the _real variants
devyubin Jul 24, 2026
b63c408
Report the full invalid UTF-8 span in the converted decode errors
devyubin Jul 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions crates/stdlib/src/csv.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,21 @@ mod _csv {
vm.new_exception_msg(super::_csv::error(vm), msg.into())
}

fn new_not_utf8_error(
vm: &VirtualMachine,
bytes: &[u8],
err: core::str::Utf8Error,
) -> PyBaseExceptionRef {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
err.valid_up_to(),
err.error_len()
.map_or(bytes.len(), |n| err.valid_up_to() + n),
vm.ctx.new_str("csv not utf8"),
)
}

Comment thread
coderabbitai[bot] marked this conversation as resolved.
#[pyattr]
#[pyclass(module = "csv", name = "Dialect")]
#[derive(Debug, PyPayload, Clone, Copy)]
Expand Down Expand Up @@ -1111,8 +1126,8 @@ mod _csv {
{
return Ok(vm.ctx.none());
}
let field = core::str::from_utf8(&field)
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
let field =
core::str::from_utf8(&field).map_err(|e| new_not_utf8_error(vm, &field, e))?;
Ok(vm.ctx.new_str(field).into())
})
.collect()
Expand Down Expand Up @@ -1248,7 +1263,7 @@ mod _csv {
prev_end = end;
let s = core::str::from_utf8(&buffer[range.clone()])
// not sure if this is possible - the input was all strings
.map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?;
.map_err(|e| new_not_utf8_error(vm, &buffer[range.clone()], e))?;

// TODO: RUSTPYTHON; Incomplete implementation
if let QuoteStyle::Nonnumeric = zelf.dialect.quoting {
Expand Down Expand Up @@ -1423,8 +1438,8 @@ mod _csv {
}

write_lineterminator(&mut output, self.dialect.lineterminator);
let s = core::str::from_utf8(&output)
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
let s =
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
self.write.call((s,), vm)
}

Expand Down Expand Up @@ -1469,8 +1484,8 @@ mod _csv {

write_lineterminator(&mut output, self.dialect.lineterminator);

let s = core::str::from_utf8(&output)
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
let s =
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;

self.write.call((s,), vm)
}
Expand Down Expand Up @@ -1518,8 +1533,8 @@ mod _csv {

write_lineterminator(&mut output, self.dialect.lineterminator);

let s = core::str::from_utf8(&output)
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
let s =
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;

self.write.call((s,), vm)
}
Expand Down Expand Up @@ -1593,7 +1608,7 @@ mod _csv {
}

let s = core::str::from_utf8(&buffer[..buffer_offset])
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
.map_err(|e| new_not_utf8_error(vm, &buffer[..buffer_offset], e))?;

self.write.call((s,), vm)
}
Expand Down
36 changes: 32 additions & 4 deletions crates/stdlib/src/socket.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2612,8 +2612,15 @@ mod _socket {
}
Some(ArgStrOrBytesLike::Buf(b)) => {
let bytes = b.borrow_buf();
let host_str = core::str::from_utf8(&bytes)
.map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?;
let host_str = core::str::from_utf8(&bytes).map_err(|e| {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
e.valid_up_to(),
e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
vm.ctx.new_str("host bytes is not utf8"),
)
})?;
Some(host_str.to_owned())
}
None => None,
Expand All @@ -2627,14 +2634,35 @@ mod _socket {
ArgStrOrBytesLike::Str(s) => {
// For str, check for surrogates and raise UnicodeEncodeError if found
s.to_str()
.ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))?
.ok_or_else(|| {
let start = s
.as_wtf8()
.code_points()
.position(|c| c.to_char().is_none())
.unwrap();
Comment on lines +2639 to +2642

@youknowone youknowone Jul 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
.as_wtf8()
.code_points()
.position(|c| c.to_char().is_none())
.unwrap();
.as_bytes()
// match surrogate bytes directly using bytes may also work

vm.new_unicode_encode_error_real(
vm.ctx.new_str("utf-8"),
(*s).clone(),
start,
start + 1,
vm.ctx.new_str("surrogates not allowed"),
)
})?
.to_owned()
}
ArgStrOrBytesLike::Buf(b) => {
// For bytes, check if it's valid UTF-8
let bytes = b.borrow_buf();
core::str::from_utf8(&bytes)
.map_err(|_| vm.new_unicode_decode_error("port is not utf8"))?
.map_err(|e| {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
e.valid_up_to(),
e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
vm.ctx.new_str("port is not utf8"),
)
})?
.to_owned()
}
};
Expand Down
11 changes: 9 additions & 2 deletions crates/vm/src/function/fspath.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,15 @@ impl FsPath {
}

pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
rustpython_host_env::os::bytes_as_os_str(b)
.map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(b.to_vec()),
e.valid_up_to(),
e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
vm.ctx.new_str("can't decode path for utf-8"),
)
})
}
}

Expand Down
11 changes: 9 additions & 2 deletions crates/vm/src/stdlib/os.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,15 @@ pub(super) struct FollowSymlinks(

#[cfg(not(windows))]
fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
rustpython_host_env::os::bytes_as_os_str(b)
.map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(b.to_vec()),
e.valid_up_to(),
e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
vm.ctx.new_str("can't decode path for utf-8"),
)
})
}

pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
Expand Down
14 changes: 10 additions & 4 deletions crates/vm/src/stdlib/posix.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1731,10 +1731,16 @@ pub mod module {
let Some(login) = rustpython_host_env::posix::getlogin() else {
return Err(vm.new_os_error("unable to determine login name"));
};
login
.to_str()
.map(|s| s.to_owned())
.map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}")))
login.to_str().map(|s| s.to_owned()).map_err(|e| {
vm.new_unicode_decode_error_real(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(login.as_bytes().to_vec()),
e.valid_up_to(),
e.error_len()
.map_or(login.as_bytes().len(), |n| e.valid_up_to() + n),
vm.ctx.new_str("unable to decode login name"),
)
})
}

// cfg from nix
Expand Down
Loading