Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions crates/host_env/src/locale.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,103 @@ pub fn strxfrm(string: &CStr, _initial_len: usize) -> Vec<u8> {
buff
}

#[cfg(target_os = "linux")]

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Add the required AI-assistance commit trailer

The commit message explicitly states that implementation, source review, and validation were AI-assisted, but it contains no Assisted-by: trailer. AGENTS.md requires an Assisted-by: AGENT_NAME:MODEL_VERSION trailer for each AI tool; the free-form “AI assistance” section does not satisfy that attribution requirement. Add the required structured trailer to the commit message.

AGENTS.md reference: AGENTS.md:L23-L23

Useful? React with 👍 / 👎.

pub use wide_collation::{CollationString, CollationTransformError};

#[cfg(target_os = "linux")]
mod wide_collation {
use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf};
use std::io;

unsafe extern "C" {
fn wcscoll(first: *const libc::wchar_t, second: *const libc::wchar_t) -> libc::c_int;
fn wcsxfrm(
destination: *mut libc::wchar_t,
source: *const libc::wchar_t,
capacity: usize,
) -> usize;
}

/// A NUL-terminated wide string for the locale collation APIs.
pub struct CollationString(Vec<libc::wchar_t>);

pub enum CollationTransformError {
Memory,
Os(io::Error),
}

impl CollationString {
/// Preserve Python's lone surrogates and reject embedded NULs.
pub fn new(string: &Wtf8) -> Option<Self> {
let mut wide = Vec::with_capacity(string.len() + 1);
for point in string.code_points() {
let value = point.to_u32();
if value == 0 {
return None;
}
wide.push(value as libc::wchar_t);
}
wide.push(0);
Some(Self(wide))
}

pub fn compare(&self, other: &Self) -> libc::c_int {
// Both buffers retain their terminator and cannot contain interior NULs.
unsafe { wcscoll(self.0.as_ptr(), other.0.as_ptr()) }
}

pub fn transform(&self) -> Result<Wtf8Buf, CollationTransformError> {
let mut dummy = [0];
crate::os::clear_errno();
let needed = unsafe { wcsxfrm(dummy.as_mut_ptr(), self.0.as_ptr(), 1) };
let errno = crate::os::get_errno();
if errno != 0 && errno != libc::ERANGE {
return Err(CollationTransformError::Os(io::Error::from_raw_os_error(
errno,
)));
}
let capacity = needed
.checked_add(1)
.ok_or(CollationTransformError::Memory)?;
let mut wide = Vec::new();
wide.try_reserve_exact(capacity)
.map_err(|_| CollationTransformError::Memory)?;
wide.resize(capacity, 0);
crate::os::clear_errno();
let written = unsafe { wcsxfrm(wide.as_mut_ptr(), self.0.as_ptr(), wide.len()) };
let errno = crate::os::get_errno();
if errno != 0 {
return Err(CollationTransformError::Os(io::Error::from_raw_os_error(
errno,
)));
}
// A concurrent locale change can invalidate the size query. Never
// interpret a truncated or unterminated result as a collation key.
if written >= wide.len() {
return Err(CollationTransformError::Os(io::Error::from_raw_os_error(
libc::ERANGE,
)));
}
wide.truncate(written);

// wcsxfrm emits integer weights, not necessarily Unicode. CPython
// 3.15 splits weights above 0x10000 into two ordered code points.
// Keep individual surrogates rather than decoding as UTF-16.
let mut key = Wtf8Buf::new();
for weight in wide {
let value = weight as u32;
if value > 0x10000 {
key.push(CodePoint::from_u32((value >> 16) | 0x10000).unwrap());
key.push(CodePoint::from_u32(value & 0xffff).unwrap());
} else {
key.push(CodePoint::from_u32(value).unwrap());
}
}
Ok(key)
}
}
}

pub fn setlocale(category: i32, locale: Option<&CStr>) -> Option<Vec<u8>> {
let result = unsafe {
match locale {
Expand Down
35 changes: 33 additions & 2 deletions crates/stdlib/src/locale.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ pub(crate) use _locale::module_def;
mod _locale {
use alloc::ffi::CString;
use rustpython_host_env::locale as host_locale;
#[cfg(target_os = "linux")]
use rustpython_vm::builtins::{PyStr, PyStrRef};
use rustpython_vm::{
PyObjectRef, PyResult, VirtualMachine,
builtins::{PyDictRef, PyIntRef, PyListRef, PyTypeRef, PyUtf8StrRef},
Expand Down Expand Up @@ -90,7 +92,25 @@ mod _locale {
Ok(vm.new_pyobj(host_locale::wcscoll(&w1, &w2)))
}

#[cfg(not(windows))]
#[cfg(target_os = "linux")]
fn collation_string(
string: &PyStr,
vm: &VirtualMachine,
) -> PyResult<host_locale::CollationString> {
host_locale::CollationString::new(string.as_wtf8())
.ok_or_else(|| vm.new_value_error("embedded null character"))
}

#[cfg(target_os = "linux")]
#[pyfunction]
fn strcoll(os1: PyStrRef, os2: PyStrRef, vm: &VirtualMachine) -> PyResult {
// Bind both strings before checking for NULs, as Argument Clinic does.
let first = collation_string(&os1, vm)?;
let second = collation_string(&os2, vm)?;
Ok(vm.new_pyobj(first.compare(&second)))
}

#[cfg(not(any(windows, target_os = "linux")))]
#[pyfunction]
fn strcoll(os1: PyUtf8StrRef, os2: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult {
let cstr1 = CString::new(os1.as_str()).map_err(|e| e.to_pyexception(vm))?;
Expand All @@ -109,7 +129,18 @@ mod _locale {
Ok(vm.new_pyobj(String::from_utf16_lossy(&transformed)))
}

#[cfg(not(windows))]
#[cfg(target_os = "linux")]
#[pyfunction]
fn strxfrm(string: PyStrRef, vm: &VirtualMachine) -> PyResult {
let string = collation_string(&string, vm)?;
let key = string.transform().map_err(|err| match err {
host_locale::CollationTransformError::Memory => vm.new_memory_error(""),
host_locale::CollationTransformError::Os(err) => err.to_pyexception(vm),
})?;
Ok(vm.ctx.new_str(key).into())
}

#[cfg(not(any(windows, target_os = "linux")))]
#[pyfunction]
fn strxfrm(string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult {
// https://github.com/python/cpython/blob/eaae563b6878aa050b4ad406b67728b6b066220e/Modules/_localemodule.c#L390-L442
Expand Down
Loading