Skip to content

Commit f98543d

Browse files
youknowdotcodex
andauthored
Implement unicodedata XID and block properties (#9458)
Follow CPython 3.15 character conversion and Unicode 17 block data; retire four verified test markers via update_lib. Assisted-by: OpenAI Codex:GPT-6.1 Co-authored-by: Codex <codex@openai.com>
1 parent bcaa07a commit f98543d

7 files changed

Lines changed: 468 additions & 5 deletions

File tree

‎.cspell.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@
9797
"implicits",
9898
"inity",
9999
"interps",
100+
"isxidcontinue",
101+
"isxidstart",
100102
"jitted",
101103
"jitting",
102104
"kwonly",

‎Lib/test/test_unicodedata.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -769,7 +769,6 @@ def parse_cp(s):
769769
for cp in range(0, sys.maxunicode + 1):
770770
self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))
771771

772-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
773772
def test_isxidstart(self):
774773
self.assertTrue(self.db.isxidstart('S'))
775774
self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM
@@ -783,7 +782,6 @@ def test_isxidstart(self):
783782
self.assertRaises(TypeError, self.db.isxidstart)
784783
self.assertRaises(TypeError, self.db.isxidstart, 'xx')
785784

786-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
787785
def test_isxidcontinue(self):
788786
self.assertTrue(self.db.isxidcontinue('S'))
789787
self.assertTrue(self.db.isxidcontinue('_'))
@@ -1013,7 +1011,6 @@ def graphemes(*args):
10131011
'a\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
10141012
['a', '\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])
10151013

1016-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
10171014
def test_block(self):
10181015
self.assertEqual(self.db.block('\u0000'), 'Basic Latin')
10191016
self.assertEqual(self.db.block('\u0041'), 'Basic Latin')
@@ -1098,7 +1095,6 @@ def test_block(self):
10981095
self.assertEqual(self.db.block('\U00100000'), 'Supplementary Private Use Area-B')
10991096
self.assertEqual(self.db.block('\U0010FFFF'), 'Supplementary Private Use Area-B')
11001097

1101-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
11021098
def test_block_invalid_input(self):
11031099
self.assertRaises(TypeError, self.db.block)
11041100
self.assertRaises(TypeError, self.db.block, b'x')

‎crates/stdlib/src/unicodedata.rs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeFormArg {
3333
mod unicodedata {
3434
use super::{NormalizeFormArg, unicode_core};
3535
use crate::vm::{
36-
Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
36+
Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
3737
builtins::{PyModule, PyStr, PyStrRef},
3838
common::lock::PyMutex,
3939
function::OptionalArg,
@@ -52,6 +52,56 @@ mod unicodedata {
5252
.map_err(|_| vm.new_type_error("argument must be a unicode character, not str"))
5353
}
5454

55+
fn property_char(
56+
character: &PyObject,
57+
function: &str,
58+
vm: &VirtualMachine,
59+
) -> PyResult<CodePoint> {
60+
let Some(character_str) = character.downcast_ref::<PyStr>() else {
61+
let kind = if vm.is_none(character) {
62+
"None".to_owned()
63+
} else {
64+
character.class().slot_name().to_string()
65+
};
66+
let kind = &kind[..kind.floor_char_boundary(50.min(kind.len()))];
67+
return Err(vm.new_type_error(format!(
68+
"{function}() argument must be a unicode character, not {kind}"
69+
)));
70+
};
71+
character_str
72+
.as_wtf8()
73+
.code_points()
74+
.exactly_one()
75+
.map_err(|_| {
76+
vm.new_type_error(format!(
77+
"{function}(): argument must be a unicode character, not a string of length {}",
78+
character_str.char_len()
79+
))
80+
})
81+
}
82+
83+
/// Return True if the character has the XID_Start property, else False.
84+
#[pyfunction]
85+
fn isxidstart(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> {
86+
Ok(property_char(&chr, "isxidstart", vm)?
87+
.to_char()
88+
.is_some_and(unicode_core::identifier::is_xid_start))
89+
}
90+
91+
/// Return True if the character has the XID_Continue property, else False.
92+
#[pyfunction]
93+
fn isxidcontinue(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> {
94+
Ok(property_char(&chr, "isxidcontinue", vm)?
95+
.to_char()
96+
.is_some_and(unicode_core::identifier::is_xid_continue))
97+
}
98+
99+
/// Return block assigned to the character chr.
100+
#[pyfunction]
101+
fn block(chr: PyObjectRef, vm: &VirtualMachine) -> PyResult<&'static str> {
102+
property_char(&chr, "block", vm).map(unicode_core::data::block)
103+
}
104+
55105
#[pyfunction]
56106
fn grapheme_cluster_break(chr: PyStrRef, vm: &VirtualMachine) -> PyResult<&'static str> {
57107
grapheme_char(&chr, vm).map(grapheme::grapheme_cluster_break)

‎crates/unicode/build.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,17 @@ fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
443443
);
444444
}
445445

446+
/// Block names and ranges from the same Unicode release as the other latest tables.
447+
fn generate_blocks() {
448+
let mut writer = open_writer("blocks.rs");
449+
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
450+
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
451+
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
452+
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
453+
}
454+
write_slice_post(&mut writer);
455+
}
456+
446457
/// Formal name aliases (`NameAliases.txt`) and named sequences
447458
/// (`NamedSequences.txt`). `unicodedata.lookup` consults both; `\N{}`
448459
/// accepts only a single character, so named sequences stay out of
@@ -892,6 +903,7 @@ fn drive_parsers() {
892903
binary_props_3_2,
893904
combining_class_3_2,
894905
generate_name_lookups,
906+
generate_blocks,
895907
];
896908

897909
let mut handles = Vec::with_capacity(parsers.len() + 2);

‎crates/unicode/src/data.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use icu_properties::props::{
2020
use rustpython_wtf8::CodePoint;
2121

2222
// Latest Unicode
23+
include!(concat!(env!("OUT_DIR"), "/generated/blocks.rs"));
2324
include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
2425
include!(concat!(env!("OUT_DIR"), "/generated/name_lookups.rs"));
2526
include!(concat!(env!("OUT_DIR"), "/generated/decomp.rs"));
@@ -268,6 +269,23 @@ pub fn character_name(ch: char) -> Option<String> {
268269
.or_else(|| lookup_table(ALGO_NAMES, ch).map(|v| v.to_name(ch)))
269270
}
270271

272+
/// Unicode block name, including ranges containing surrogates or unassigned points.
273+
#[must_use]
274+
pub fn block(c: CodePoint) -> &'static str {
275+
let c = c.to_u32();
276+
BLOCKS
277+
.binary_search_by(|&(start, end, _)| {
278+
if c > end {
279+
Ordering::Less
280+
} else if c < start {
281+
Ordering::Greater
282+
} else {
283+
Ordering::Equal
284+
}
285+
})
286+
.map_or("No_Block", |i| BLOCKS[i].2)
287+
}
288+
271289
/// A view over the Unicode character database at a fixed version.
272290
///
273291
/// `modern` selects the latest bundled UCD; otherwise the Unicode 3.2.0 tables

‎crates/unicode/unicode/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ The files in `latest` need to be periodically bumped to match `icu4x` and Rust.
77
`NameAliases.txt` and `NamedSequences.txt` feed `unicodedata.lookup` (formal aliases and named sequences). They are not used by `unicodedata.name`, which still reports the official character name.
88

99
RustPython vendors [Unicode 3.2.0](https://www.unicode.org/reports/tr28/tr28-3.html) to match CPython. CPython uses 3.2.0 to ensure backwards compatibility with a few older modules. Unicode 3.2.0 was released in 2002 and does not need periodic refreshes.
10+
11+
`Blocks.txt` supplies the ranges and original block names for `unicodedata.block`. The current copy is from [Unicode 17.0.0](https://www.unicode.org/Public/17.0.0/ucd/Blocks.txt).

0 commit comments

Comments
 (0)