Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .cspell.json
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@
"implicits",
"inity",
"interps",
"isxidcontinue",
"isxidstart",
"jitted",
"jitting",
"kwonly",
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_unicodedata.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,7 +769,6 @@ def parse_cp(s):
for cp in range(0, sys.maxunicode + 1):
self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
def test_isxidstart(self):
self.assertTrue(self.db.isxidstart('S'))
self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM
Expand All @@ -783,7 +782,6 @@ def test_isxidstart(self):
self.assertRaises(TypeError, self.db.isxidstart)
self.assertRaises(TypeError, self.db.isxidstart, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
def test_isxidcontinue(self):
self.assertTrue(self.db.isxidcontinue('S'))
self.assertTrue(self.db.isxidcontinue('_'))
Expand Down Expand Up @@ -1013,7 +1011,6 @@ def graphemes(*args):
'a\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
['a', '\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
def test_block(self):
self.assertEqual(self.db.block('\u0000'), 'Basic Latin')
self.assertEqual(self.db.block('\u0041'), 'Basic Latin')
Expand Down Expand Up @@ -1098,7 +1095,6 @@ def test_block(self):
self.assertEqual(self.db.block('\U00100000'), 'Supplementary Private Use Area-B')
self.assertEqual(self.db.block('\U0010FFFF'), 'Supplementary Private Use Area-B')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
def test_block_invalid_input(self):
self.assertRaises(TypeError, self.db.block)
self.assertRaises(TypeError, self.db.block, b'x')
Expand Down
52 changes: 51 additions & 1 deletion crates/stdlib/src/unicodedata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeFormArg {
mod unicodedata {
use super::{NormalizeFormArg, unicode_core};
use crate::vm::{
Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyModule, PyStr, PyStrRef},
common::lock::PyMutex,
function::OptionalArg,
Expand All @@ -52,6 +52,56 @@ mod unicodedata {
.map_err(|_| vm.new_type_error("argument must be a unicode character, not str"))
}

fn property_char(
character: &PyObject,
function: &str,
vm: &VirtualMachine,
) -> PyResult<CodePoint> {
let Some(character_str) = character.downcast_ref::<PyStr>() else {
let kind = if vm.is_none(character) {
"None".to_owned()
} else {
character.class().slot_name().to_string()
};
let kind = &kind[..kind.floor_char_boundary(50.min(kind.len()))];
return Err(vm.new_type_error(format!(
"{function}() argument must be a unicode character, not {kind}"
)));
};
character_str
.as_wtf8()
.code_points()
.exactly_one()
.map_err(|_| {
vm.new_type_error(format!(
"{function}(): argument must be a unicode character, not a string of length {}",
character_str.char_len()
))
})
}

/// Return True if the character has the XID_Start property, else False.
#[pyfunction]
fn isxidstart(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> {
Ok(property_char(&chr, "isxidstart", vm)?
.to_char()
.is_some_and(unicode_core::identifier::is_xid_start))
}

/// Return True if the character has the XID_Continue property, else False.
#[pyfunction]
fn isxidcontinue(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> {
Ok(property_char(&chr, "isxidcontinue", vm)?
.to_char()
.is_some_and(unicode_core::identifier::is_xid_continue))
}

/// Return block assigned to the character chr.
#[pyfunction]
fn block(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<&'static str> {
property_char(&chr, "block", vm).map(unicode_core::data::block)
}

#[pyfunction]
fn grapheme_cluster_break(chr: PyStrRef, vm: &VirtualMachine) -> PyResult<&'static str> {
grapheme_char(&chr, vm).map(grapheme::grapheme_cluster_break)
Expand Down
12 changes: 12 additions & 0 deletions crates/unicode/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,17 @@ fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
);
}

/// Block names and ranges from the same Unicode release as the other latest tables.
fn generate_blocks() {
let mut writer = open_writer("blocks.rs");
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
}
write_slice_post(&mut writer);
}

/// Formal name aliases (`NameAliases.txt`) and named sequences
/// (`NamedSequences.txt`). `unicodedata.lookup` consults both; `\N{}`
/// accepts only a single character, so named sequences stay out of
Expand Down Expand Up @@ -892,6 +903,7 @@ fn drive_parsers() {
binary_props_3_2,
combining_class_3_2,
generate_name_lookups,
generate_blocks,
];

let mut handles = Vec::with_capacity(parsers.len() + 2);
Expand Down
18 changes: 18 additions & 0 deletions crates/unicode/src/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use icu_properties::props::{
use rustpython_wtf8::CodePoint;

// Latest Unicode
include!(concat!(env!("OUT_DIR"), "/generated/blocks.rs"));
include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
include!(concat!(env!("OUT_DIR"), "/generated/name_lookups.rs"));
include!(concat!(env!("OUT_DIR"), "/generated/decomp.rs"));
Expand Down Expand Up @@ -268,6 +269,23 @@ pub fn character_name(ch: char) -> Option<String> {
.or_else(|| lookup_table(ALGO_NAMES, ch).map(|v| v.to_name(ch)))
}

/// Unicode block name, including ranges containing surrogates or unassigned points.
#[must_use]
pub fn block(c: CodePoint) -> &'static str {
let c = c.to_u32();
BLOCKS
.binary_search_by(|&(start, end, _)| {
if c > end {
Ordering::Less
} else if c < start {
Ordering::Greater
} else {
Ordering::Equal
}
})
.map_or("No_Block", |i| BLOCKS[i].2)
}

/// A view over the Unicode character database at a fixed version.
///
/// `modern` selects the latest bundled UCD; otherwise the Unicode 3.2.0 tables
Expand Down
2 changes: 2 additions & 0 deletions crates/unicode/unicode/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ The files in `latest` need to be periodically bumped to match `icu4x` and Rust.
`NameAliases.txt` and `NamedSequences.txt` feed `unicodedata.lookup` (formal aliases and named sequences). They are not used by `unicodedata.name`, which still reports the official character name.

RustPython vendors [Unicode 3.2.0](https://www.unicode.org/reports/tr28/tr28-3.html) to match CPython. CPython uses 3.2.0 to ensure backwards compatibility with a few older modules. Unicode 3.2.0 was released in 2002 and does not need periodic refreshes.

`Blocks.txt` supplies the ranges and original block names for `unicodedata.block`. The current copy is from [Unicode 17.0.0](https://www.unicode.org/Public/17.0.0/ucd/Blocks.txt).
Loading
Loading