Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
flamescope = { version = "0.1.2", optional = true }

rustls = { workspace = true, optional = true }
rustls-graviola = { workspace = true, optional = true }

Check warning on line 52 in Cargo.toml

View workflow job for this annotation

GitHub Actions / cargo shear

shear/misplaced_optional_dependency

misplaced optional dependency `rustls-graviola` (remove the `optional` flag and move to `[dev-dependencies]`)

[target.'cfg(windows)'.dependencies]
libc = { workspace = true }
Expand Down Expand Up @@ -306,6 +306,7 @@
timsort = "0.1.2"
tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" }
icu_casemap = "2"
icu_decimal = "2"
icu_locale = "2"
icu_properties = "2"
icu_normalizer = "2"
Expand Down
44 changes: 0 additions & 44 deletions crates/common/src/str.rs
Original file line number Diff line number Diff line change
Expand Up @@ -615,50 +615,6 @@ macro_rules! ascii {
}
pub use ascii;

// TODO: this should probably live in a crate like unic or unicode-properties
const UNICODE_DECIMAL_VALUES: &[char] = &[
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '٠', '١', '٢', '٣', '٤', '٥', '٦', '٧', '٨',
'٩', '۰', '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '߀', '߁', '߂', '߃', '߄', '߅', '߆', '߇',
'߈', '߉', '०', '१', '२', '३', '४', '५', '६', '७', '८', '९', '০', '১', '২', '৩', '৪', '৫', '৬',
'৭', '৮', '৯', '੦', '੧', '੨', '੩', '੪', '੫', '੬', '੭', '੮', '੯', '૦', '૧', '૨', '૩', '૪', '૫',
'૬', '૭', '૮', '૯', '୦', '୧', '୨', '୩', '୪', '୫', '୬', '୭', '୮', '୯', '௦', '௧', '௨', '௩', '௪',
'௫', '௬', '௭', '௮', '௯', '౦', '౧', '౨', '౩', '౪', '౫', '౬', '౭', '౮', '౯', '೦', '೧', '೨', '೩',
'೪', '೫', '೬', '೭', '೮', '೯', '൦', '൧', '൨', '൩', '൪', '൫', '൬', '൭', '൮', '൯', '෦', '෧', '෨',
'෩', '෪', '෫', '෬', '෭', '෮', '෯', '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', '๘', '๙', '໐', '໑',
'໒', '໓', '໔', '໕', '໖', '໗', '໘', '໙', '༠', '༡', '༢', '༣', '༤', '༥', '༦', '༧', '༨', '༩', '၀',
'၁', '၂', '၃', '၄', '၅', '၆', '၇', '၈', '၉', '႐', '႑', '႒', '႓', '႔', '႕', '႖', '႗', '႘', '႙',
'០', '១', '២', '៣', '៤', '៥', '៦', '៧', '៨', '៩', '᠐', '᠑', '᠒', '᠓', '᠔', '᠕', '᠖', '᠗', '᠘',
'᠙', '᥆', '᥇', '᥈', '᥉', '᥊', '᥋', '᥌', '᥍', '᥎', '᥏', '᧐', '᧑', '᧒', '᧓', '᧔', '᧕', '᧖', '᧗',
'᧘', '᧙', '᪀', '᪁', '᪂', '᪃', '᪄', '᪅', '᪆', '᪇', '᪈', '᪉', '᪐', '᪑', '᪒', '᪓', '᪔', '᪕', '᪖',
'᪗', '᪘', '᪙', '᭐', '᭑', '᭒', '᭓', '᭔', '᭕', '᭖', '᭗', '᭘', '᭙', '᮰', '᮱', '᮲', '᮳', '᮴', '᮵',
'᮶', '᮷', '᮸', '᮹', '᱀', '᱁', '᱂', '᱃', '᱄', '᱅', '᱆', '᱇', '᱈', '᱉', '᱐', '᱑', '᱒', '᱓', '᱔',
'᱕', '᱖', '᱗', '᱘', '᱙', '꘠', '꘡', '꘢', '꘣', '꘤', '꘥', '꘦', '꘧', '꘨', '꘩', '꣐', '꣑', '꣒', '꣓',
'꣔', '꣕', '꣖', '꣗', '꣘', '꣙', '꤀', '꤁', '꤂', '꤃', '꤄', '꤅', '꤆', '꤇', '꤈', '꤉', '꧐', '꧑', '꧒',
'꧓', '꧔', '꧕', '꧖', '꧗', '꧘', '꧙', '꧰', '꧱', '꧲', '꧳', '꧴', '꧵', '꧶', '꧷', '꧸', '꧹', '꩐', '꩑',
'꩒', '꩓', '꩔', '꩕', '꩖', '꩗', '꩘', '꩙', '꯰', '꯱', '꯲', '꯳', '꯴', '꯵', '꯶', '꯷', '꯸', '꯹', '0',
'1', '2', '3', '4', '5', '6', '7', '8', '9', '𐒠', '𐒡', '𐒢', '𐒣', '𐒤', '𐒥', '𐒦', '𐒧',
'𐒨', '𐒩', '𑁦', '𑁧', '𑁨', '𑁩', '𑁪', '𑁫', '𑁬', '𑁭', '𑁮', '𑁯', '𑃰', '𑃱', '𑃲', '𑃳', '𑃴', '𑃵', '𑃶',
'𑃷', '𑃸', '𑃹', '𑄶', '𑄷', '𑄸', '𑄹', '𑄺', '𑄻', '𑄼', '𑄽', '𑄾', '𑄿', '𑇐', '𑇑', '𑇒', '𑇓', '𑇔', '𑇕',
'𑇖', '𑇗', '𑇘', '𑇙', '𑋰', '𑋱', '𑋲', '𑋳', '𑋴', '𑋵', '𑋶', '𑋷', '𑋸', '𑋹', '𑑐', '𑑑', '𑑒', '𑑓', '𑑔',
'𑑕', '𑑖', '𑑗', '𑑘', '𑑙', '𑓐', '𑓑', '𑓒', '𑓓', '𑓔', '𑓕', '𑓖', '𑓗', '𑓘', '𑓙', '𑙐', '𑙑', '𑙒', '𑙓',
'𑙔', '𑙕', '𑙖', '𑙗', '𑙘', '𑙙', '𑛀', '𑛁', '𑛂', '𑛃', '𑛄', '𑛅', '𑛆', '𑛇', '𑛈', '𑛉', '𑜰', '𑜱', '𑜲',
'𑜳', '𑜴', '𑜵', '𑜶', '𑜷', '𑜸', '𑜹', '𑣠', '𑣡', '𑣢', '𑣣', '𑣤', '𑣥', '𑣦', '𑣧', '𑣨', '𑣩', '𑱐', '𑱑',
'𑱒', '𑱓', '𑱔', '𑱕', '𑱖', '𑱗', '𑱘', '𑱙', '𑵐', '𑵑', '𑵒', '𑵓', '𑵔', '𑵕', '𑵖', '𑵗', '𑵘', '𑵙', '𖩠',
'𖩡', '𖩢', '𖩣', '𖩤', '𖩥', '𖩦', '𖩧', '𖩨', '𖩩', '𖭐', '𖭑', '𖭒', '𖭓', '𖭔', '𖭕', '𖭖', '𖭗', '𖭘', '𖭙',
'𝟎', '𝟏', '𝟐', '𝟑', '𝟒', '𝟓', '𝟔', '𝟕', '𝟖', '𝟗', '𝟘', '𝟙', '𝟚', '𝟛', '𝟜', '𝟝', '𝟞', '𝟟', '𝟠',
'𝟡', '𝟢', '𝟣', '𝟤', '𝟥', '𝟦', '𝟧', '𝟨', '𝟩', '𝟪', '𝟫', '𝟬', '𝟭', '𝟮', '𝟯', '𝟰', '𝟱', '𝟲', '𝟳',
'𝟴', '𝟵', '𝟶', '𝟷', '𝟸', '𝟹', '𝟺', '𝟻', '𝟼', '𝟽', '𝟾', '𝟿', '𞥐', '𞥑', '𞥒', '𞥓', '𞥔', '𞥕', '𞥖',
'𞥗', '𞥘', '𞥙',
];

#[must_use]
pub fn char_to_decimal(ch: char) -> Option<u8> {
UNICODE_DECIMAL_VALUES
.binary_search(&ch)
.ok()
.map(|i| (i % 10) as u8)
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
1 change: 1 addition & 0 deletions crates/vm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ timsort = { workspace = true }

## unicode stuff
icu_casemap = { workspace = true }
icu_decimal = { workspace = true }
icu_locale = { workspace = true }
icu_properties = { workspace = true }
writeable = { workspace = true }
Expand Down
31 changes: 18 additions & 13 deletions crates/vm/src/builtins/float.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,16 @@ use crate::{
},
protocol::PyNumberMethods,
types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable},
utils::SliceFmtWriter,
};
use core::cell::Cell;
use core::ptr::NonNull;
use icu_decimal::input::Decimal;
use malachite_bigint::{BigInt, ToBigInt};
use num_complex::Complex64;
use num_traits::{Signed, ToPrimitive, Zero};
use rustpython_common::int::float_to_ratio;
use writeable::Writeable;

#[pyclass(module = false, name = "float")]
#[derive(Debug, Copy, Clone, PartialEq)]
Expand Down Expand Up @@ -206,26 +209,28 @@ impl Constructor for PyFloat {
}

fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> {
let (bytearray, buffer, buffer_lock, mapped_string);
let (bytearray, buffer, buffer_lock);
let mut mapped_str_buf = [0u8; 1024];
let b = if let Some(s) = val.downcast_ref::<PyStr>() {
use crate::common::str::PyKindStr;
match s.as_str_kind() {
PyKindStr::Ascii(s) => s.trim().as_bytes(),
PyKindStr::Utf8(s) => {
mapped_string = s
.trim()
.chars()
.map(|c| {
if let Some(n) = rustpython_common::str::char_to_decimal(c) {
char::from_digit(n.into(), 10).unwrap()
} else if c.is_whitespace() {
' '
match Decimal::try_from_str(s.trim()) {
Ok(decimal) => {
let mut writer = SliceFmtWriter::new(&mut mapped_str_buf);
if decimal.write_to(&mut writer).is_err() {
s.as_bytes()
} else {
c
let written = writer.written();
&mapped_str_buf[..written]
}
})
.collect::<String>();
mapped_string.as_bytes()
}
Err(_) => {
// Let the string fail parsing below
s.as_bytes()
}
}
}
// if there are surrogates, it's not gonna parse anyway,
// so we can just choose a known bad value
Expand Down
40 changes: 40 additions & 0 deletions crates/vm/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,43 @@ impl fmt::Write for VecFmtWriter {
Ok(())
}
}

/// Wrapper around a bytes slice that implements [`fmt::Write`].
///
/// # Errors
/// [`fmt::Error`] is returned if the string doesn't fit into the internal buffer. This
/// implementation writes as many bytes as can fit before returning an error.
/// Check [`Self::remainder`] if the amount of bytes that can be written is important.
pub(crate) struct SliceFmtWriter<'slice> {
buf: &'slice mut [u8],
written: usize,
}

impl<'slice> SliceFmtWriter<'slice> {
pub(crate) const fn new(buf: &'slice mut [u8]) -> Self {
Self { buf, written: 0 }
}

/// Amount of space left in the internal buffer.
pub(crate) const fn remainder(&self) -> usize {
self.buf.len() - self.written
}

/// Amount of bytes written to the internal buffer.
pub(crate) const fn written(&self) -> usize {
self.written
}
}

impl fmt::Write for SliceFmtWriter<'_> {
fn write_str(&mut self, s: &str) -> fmt::Result {
let to_copy = self.remainder().min(s.len());
self.buf[self.written..self.written + to_copy].copy_from_slice(&s.as_bytes()[..to_copy]);
self.written += to_copy;
if to_copy == s.len() {
Ok(())
} else {
Err(fmt::Error)
}
}
Comment thread
joshuamegnauth54 marked this conversation as resolved.
}
Loading