Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions Lib/test/test_decimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -854,10 +854,6 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase):
class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase):
decimal = P

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_unicode_digits(self):
return super().test_unicode_digits()

class ImplicitConstructionTest:
'''Unit tests for Implicit Construction cases of Decimal.'''

Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_int.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,6 @@ def test_invalid_signs(self):
with self.assertRaises(ValueError):
int(' + 1 ')

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_unicode(self):
self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890)
self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890)
Expand Down
54 changes: 54 additions & 0 deletions crates/common/src/str.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
use crate::atomic::{PyAtomic, Radium};
use crate::format::CharLen;
use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
use alloc::borrow::Cow;
use ascii::{AsciiChar, AsciiStr, AsciiString};
use core::fmt;
use core::ops::{Bound, RangeBounds};
Expand Down Expand Up @@ -658,10 +659,63 @@ pub fn char_to_decimal(ch: char) -> Option<u8> {
.map(|i| (i % 10) as u8)
}

/// Replace Unicode decimal digits with their ASCII equivalents and any Unicode
/// whitespace with a plain space, so the byte-oriented numeric parsers can read
/// them. Mirrors CPython's `_PyUnicode_TransformDecimalAndSpaceToASCII`.
///
/// The result is always ASCII. Any other non-ASCII character cannot appear in a
/// numeric literal, so it becomes a `?` and the rest of the string is dropped:
/// `?` is rejected by every parser at every base, which leaves the caller — the
/// one that knows the base and owns the original string — to raise the error.
#[must_use]
pub fn transform_decimal_and_space_to_ascii(s: &str) -> Cow<'_, str> {
if s.is_ascii() {
return Cow::Borrowed(s);
}
let mut out = String::with_capacity(s.len());
for c in s.chars() {
if (c as u32) < 127 {
out.push(c);
} else if c.is_whitespace() {
out.push(' ');
} else if let Some(n) = char_to_decimal(c) {
out.push(char::from_digit(n.into(), 10).unwrap());
} else {
out.push('?');
break;
}
}
debug_assert!(out.is_ascii());
Cow::Owned(out)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn transform_decimal_and_space() {
// ASCII input is passed through untouched, without allocating.
assert!(matches!(
transform_decimal_and_space_to_ascii("123"),
Cow::Borrowed("123")
));
// Decimal digits from any script fold to ASCII.
assert_eq!(transform_decimal_and_space_to_ascii("١٢٣"), "123");
assert_eq!(transform_decimal_and_space_to_ascii("12३"), "123");
assert_eq!(transform_decimal_and_space_to_ascii("1٢3"), "123");
// Unicode whitespace folds to a plain space.
assert_eq!(transform_decimal_and_space_to_ascii("\u{3000}٣"), " 3");
// ASCII characters ride through untouched, whatever they are.
assert_eq!(transform_decimal_and_space_to_ascii("0x١f"), "0x1f");
assert_eq!(transform_decimal_and_space_to_ascii("-١_٢"), "-1_2");
// Anything else poisons the literal and truncates it, so the result stays
// ASCII and the caller's parser is guaranteed to reject it.
assert_eq!(transform_decimal_and_space_to_ascii("½가"), "?");
assert_eq!(transform_decimal_and_space_to_ascii("١٢가٣"), "12?");
assert_eq!(transform_decimal_and_space_to_ascii("١\u{7f}"), "1?");
}

#[test]
fn get_chars_basic() {
let s = "0123456789";
Expand Down
8 changes: 4 additions & 4 deletions crates/vm/src/builtins/complex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,10 @@ impl Constructor for PyComplex {
"complex() can't take second arg if first is a string",
));
}
let (re, im) = s
.to_str()
.and_then(rustpython_literal::complex::parse_str)
.ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?;
let (re, im) = rustpython_literal::complex::parse_str(
&crate::protocol::numeric_literal_from_str(s),
)
.ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?;
return Ok(Self::from(Complex64 { re, im }));
} else {
return Err(vm.new_type_error(format!(
Expand Down
25 changes: 2 additions & 23 deletions crates/vm/src/builtins/float.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,29 +208,8 @@ impl Constructor for PyFloat {
pub fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> {
let (bytearray, buffer, buffer_lock, mapped_string);
let b = if let Some(s) = val.downcast_ref::<PyStr>() {
use crate::common::str::PyKindStr;
match s.as_str_kind() {
PyKindStr::Ascii(s) => s.trim().as_bytes(),
PyKindStr::Utf8(s) => {
mapped_string = s
.trim()
.chars()
.map(|c| {
if let Some(n) = rustpython_common::str::char_to_decimal(c) {
char::from_digit(n.into(), 10).unwrap()
} else if c.is_whitespace() {
' '
} else {
c
}
})
.collect::<String>();
mapped_string.as_bytes()
}
// if there are surrogates, it's not gonna parse anyway,
// so we can just choose a known bad value
PyKindStr::Wtf8(_) => b"",
}
mapped_string = crate::protocol::numeric_literal_from_str(s);
mapped_string.as_bytes()
} else if let Some(bytes) = val.downcast_ref::<PyBytes>() {
bytes.as_bytes()
} else if let Some(buf) = val.downcast_ref::<PyByteArray>() {
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use crate::{
ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue,
PyComparisonValue,
},
protocol::{PyNumberMethods, handle_bytes_to_int_err},
protocol::{PyNumberMethods, handle_bytes_to_int_err, numeric_literal_from_str},
types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable},
};
use alloc::fmt;
Expand Down Expand Up @@ -804,7 +804,7 @@ struct IntToByteArgs {
fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult<BigInt> {
match_class!(match obj.to_owned() {
string @ PyStr => {
let s = string.as_wtf8().trim();
let s = numeric_literal_from_str(&string);
bytes_to_int(s.as_bytes(), base, vm.state.int_max_str_digits.load())
.map_err(|e| handle_bytes_to_int_err(e, obj, vm))
}
Expand Down
1 change: 1 addition & 0 deletions crates/vm/src/protocol/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@ pub use mapping::{PyMapping, PyMappingMethods, PyMappingSlots};
pub use number::{
PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots,
PyNumberTernaryFunc, PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err,
numeric_literal_from_str,
};
pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots};
27 changes: 25 additions & 2 deletions crates/vm/src/protocol/number.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,34 @@ use crate::{
builtins::{
PyBaseExceptionRef, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr, int,
},
common::int::{BytesToIntError, bytes_to_int},
common::{
int::{BytesToIntError, bytes_to_int},
str::{PyKindStr, transform_decimal_and_space_to_ascii},
},
function::ArgBytesLike,
object::{Traverse, TraverseFn},
stdlib::_warnings,
};
use alloc::borrow::Cow;

/// Normalize a `str` for the byte-oriented numeric parsers: Unicode decimal digits
/// and whitespace fold to their ASCII equivalents, the way CPython runs every
/// numeric constructor's string argument through
/// `_PyUnicode_TransformDecimalAndSpaceToASCII` first.
///
/// `int`, `float` and `complex` share this step and nothing else — only `int` takes
/// a base, and only `int` and `float` accept bytes-like input, so each keeps its own
/// entry point around this one.
///
/// A string holding surrogates can never be a valid literal, so it folds to an
/// empty — and therefore invalid — one.
pub fn numeric_literal_from_str(s: &PyStr) -> Cow<'_, str> {
match s.as_str_kind() {
PyKindStr::Ascii(s) => Cow::Borrowed(s.trim().as_str()),
PyKindStr::Utf8(s) => transform_decimal_and_space_to_ascii(s.trim()),
PyKindStr::Wtf8(_) => Cow::Borrowed(""),
}
}

pub type PyNumberUnaryFunc<R = PyObjectRef> = fn(PyNumber<'_>, &VirtualMachine) -> PyResult<R>;
pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult;
Expand Down Expand Up @@ -59,7 +82,7 @@ impl PyObject {
} else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) {
i
} else if let Some(s) = self.downcast_ref::<PyStr>() {
try_convert(self, s.as_wtf8().trim().as_bytes(), vm)
try_convert(self, numeric_literal_from_str(s).as_bytes(), vm)
} else if let Some(bytes) = self.downcast_ref::<PyBytes>() {
try_convert(self, bytes, vm)
} else if let Some(bytearray) = self.downcast_ref::<PyByteArray>() {
Expand Down
Loading