Skip to content

Commit 0b8fcb4

Browse files
joshuamegnauth54joshuamegnauth54
authored
unicodedata: Fix name, unmask test_name (#8599)
This commit fixes most of UCD's names. The two main fixes are: * Handling chars missing in 3.2.0 by checking for membership #8548 * Handling names not present in unicodenames AI disclosure: I used AI for the initial research of this patch to determine why the Tangut characters are missing. AI also assisted with a bug fix where I neglected to parse the Tangut u32 as hex. I wrote all of the code. Assisted-by: GPT 5.6 Luna
1 parent a14f582 commit 0b8fcb4

5 files changed

Lines changed: 115 additions & 14 deletions

File tree

‎Lib/test/test_unicodedata.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ def test_function_checksum(self):
131131
result = h.hexdigest()
132132
self.assertEqual(result, self.expectedchecksum)
133133

134-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
135134
def test_name(self):
136135
name = self.db.name
137136
self.assertRaises(ValueError, name, '\0')
@@ -732,7 +731,6 @@ class UnicodeFunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
732731
'ebfc9dd281c2226998fd435744dd2e9321899beb')
733732

734733
@requires_resource('network')
735-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
736734
def test_all_names(self):
737735
TESTDATAFILE = "DerivedName.txt"
738736
testdata = download_test_data_file(TESTDATAFILE)
@@ -1141,10 +1139,6 @@ class Unicode_3_2_0_FunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
11411139
def test_normalization(self):
11421140
return super().test_normalization()
11431141

1144-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'LATIN SMALL LETTER D WITH CURL' != None
1145-
def test_name(self):
1146-
return super().test_name()
1147-
11481142
@unittest.expectedSuccess # TODO: RUSTPYTHON
11491143
def test_combining(self):
11501144
return super().test_combining()

‎crates/stdlib/src/unicodedata.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ mod unicodedata {
102102
fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
103103
if let Some(name_str) = name.to_str()
104104
&& let Some(character) = unicode_core::lookup_character(name_str)
105+
&& self.inner.membership(character)
105106
{
106107
return Ok(character.to_string());
107108
}
@@ -119,11 +120,12 @@ mod unicodedata {
119120
default: OptionalArg<PyObjectRef>,
120121
vm: &VirtualMachine,
121122
) -> PyResult {
122-
if let Some(name) = self
123-
.extract_char(character, vm)?
124-
.to_char()
125-
.and_then(unicode_core::character_name)
126-
{
123+
if let Some(name) = self.extract_char(character, vm)?.to_char().and_then(|ch| {
124+
self.inner
125+
.membership(ch)
126+
.then(|| unicode_core::character_name(ch))
127+
.flatten()
128+
}) {
127129
return Ok(vm.ctx.new_str(name).into());
128130
}
129131
default.ok_or_else(|| vm.new_value_error("no such name"))

‎crates/unicode/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ writeable = { workspace = true }
2020

2121
[build-dependencies]
2222
icu_properties = { workspace = true }
23+
unicode_names2 = { workspace = true }
2324

2425
[lints]
2526
workspace = true

‎crates/unicode/build.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ fn full_data_parsers_latest() {
194194
let reader = UnicodeLineReader::from_file_name("UnicodeData.txt", true);
195195

196196
let mut decomp_lines = Vec::new();
197+
let mut algo_names = Vec::new();
197198
for line in reader {
198199
let decomp_field = line.field(
199200
NonZeroUsize::new(5).unwrap(),
@@ -205,9 +206,21 @@ fn full_data_parsers_latest() {
205206
if !decomp_field.is_empty() {
206207
decomp_lines.push((line.start, decomp_field.to_owned()));
207208
}
209+
210+
let name_field = line.field(
211+
NonZeroUsize::new(1).unwrap(),
212+
Some(&format!(
213+
"field 1 (name) missing from UnicodeData.txt: {}",
214+
line.line
215+
)),
216+
);
217+
if name_field.ends_with("First>") || name_field.ends_with("Last>") {
218+
algo_names.push((line.start, name_field.into()));
219+
}
208220
}
209221

210222
generate_decomp(decomp_lines);
223+
generate_algo_names(algo_names);
211224
}
212225

213226
fn generate_decomp(decomp_lines: Vec<(u32, String)>) {
@@ -287,6 +300,46 @@ fn generate_decomp(decomp_lines: Vec<(u32, String)>) {
287300
write_slice_pairs(&mut writer, "DECOMP_UPDATES", "(u32, u32)", &mut values);
288301
}
289302

303+
fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
304+
const TANGUT: &str = "<Tangut Ideograph";
305+
const TANGUT_SUP: &str = "<Tangut Ideograph Supplement";
306+
307+
let (names, []) = names.as_chunks::<2>() else {
308+
panic!("Uneven algorithmic names: {names:#?}");
309+
};
310+
311+
let mut values = Vec::new();
312+
for &[(start, ref raw_start), (end, ref raw_end)] in names {
313+
// Surrogate and private use ranges are algorithmic names.
314+
if (0xD800..=0xDFFF).contains(&start)
315+
|| (0xE000..=0xF8FF).contains(&start)
316+
|| (0xF0000..=0xFFFFD).contains(&start)
317+
|| (0x100000..=0x10FFFD).contains(&start)
318+
{
319+
continue;
320+
};
321+
// Skip existing as there are only a few unimplemented algorithmic names.
322+
if unicode_names2::name(char::from_u32(start).unwrap()).is_none() {
323+
match (raw_start.split_once(','), raw_end.split_once(',')) {
324+
(Some((TANGUT, _)), Some((TANGUT, _)))
325+
| (Some((TANGUT_SUP, _)), Some((TANGUT_SUP, _))) => {
326+
values.push((start, end, "AlgorithmicName::TangutIdeograph"));
327+
}
328+
(None, None) => panic!("Unexpected name parsed:\n{raw_start}\n{raw_end}"),
329+
_ => panic!("Unhandled algorithmic name:\n{raw_start}\n{raw_end}"),
330+
}
331+
}
332+
}
333+
334+
let mut writer = open_writer("algo_names.rs");
335+
write_slice_display(
336+
&mut writer,
337+
"ALGO_NAMES",
338+
"(u32, u32, AlgorithmicName)",
339+
&mut values,
340+
);
341+
}
342+
290343
/// Drive parsers that require the full 3.2.0 data.
291344
///
292345
/// As the full data set is HUGE, it's more efficient to parse it once then delegate to subparsers.

‎crates/unicode/src/data.rs‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use icu_properties::props::{
1919
};
2020
use rustpython_wtf8::CodePoint;
2121

22+
include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
2223
include!(concat!(env!("OUT_DIR"), "/generated/bidi_class_3_2.rs"));
2324
include!(concat!(env!("OUT_DIR"), "/generated/binary_props_3_2.rs"));
2425
include!(concat!(
@@ -75,6 +76,37 @@ impl DecompositionType {
7576
}
7677
}
7778

79+
#[derive(Clone, Copy, PartialEq)]
80+
enum AlgorithmicName {
81+
TangutIdeograph,
82+
}
83+
84+
impl AlgorithmicName {
85+
const fn name_base(self) -> &'static str {
86+
match self {
87+
Self::TangutIdeograph => "TANGUT IDEOGRAPH",
88+
}
89+
}
90+
91+
fn to_name(self, ch: char) -> String {
92+
match self {
93+
Self::TangutIdeograph => format!("{}-{:0X}", self.name_base(), ch as u32),
94+
}
95+
}
96+
97+
fn check_name(search_name: &str) -> Option<char> {
98+
if let Some(without_base) = search_name.strip_prefix(Self::TangutIdeograph.name_base()) {
99+
let cp = u32::from_str_radix(without_base.strip_prefix('-')?.trim(), 16).ok()?;
100+
let ch = char::from_u32(cp)?;
101+
if lookup_table(ALGO_NAMES, ch)? == Self::TangutIdeograph {
102+
return Some(ch);
103+
}
104+
}
105+
106+
None
107+
}
108+
}
109+
78110
fn lookup_table<T: Copy>(table: &[(u32, u32, T)], ch: char) -> Option<T> {
79111
let ch = ch as u32;
80112
table
@@ -91,7 +123,10 @@ fn lookup_table<T: Copy>(table: &[(u32, u32, T)], ch: char) -> Option<T> {
91123
.map(|i| table[i].2)
92124
}
93125

94-
fn membership_3_2(ch: u32) -> bool {
126+
#[cold]
127+
#[inline(never)]
128+
#[must_use]
129+
pub fn membership_3_2(ch: u32) -> bool {
95130
MEMBERSHIP_3_2
96131
.binary_search_by(|&(start, end)| {
97132
if ch > end {
@@ -136,12 +171,17 @@ pub fn unicode_version() -> String {
136171
}
137172

138173
/// Look up a character by its Unicode name (`unicodedata.lookup`).
139-
pub use unicode_names2::character as lookup_character;
174+
#[must_use]
175+
pub fn lookup_character(search_name: &str) -> Option<char> {
176+
unicode_names2::character(search_name).or_else(|| AlgorithmicName::check_name(search_name))
177+
}
140178

141179
/// The Unicode name of `ch` (`unicodedata.name`), if any.
142180
#[must_use]
143181
pub fn character_name(ch: char) -> Option<String> {
144-
unicode_names2::name(ch).map(|name| name.to_string())
182+
unicode_names2::name(ch)
183+
.map(|name| name.to_string())
184+
.or_else(|| lookup_table(ALGO_NAMES, ch).map(|v| v.to_name(ch)))
145185
}
146186

147187
/// A view over the Unicode character database at a fixed version.
@@ -159,6 +199,17 @@ impl Ucd {
159199
Self { modern }
160200
}
161201

202+
#[must_use]
203+
pub fn membership(self, ch: char) -> bool {
204+
if !self.modern {
205+
cold_path();
206+
membership_3_2(ch as u32)
207+
} else {
208+
// Rust chars always "exist" for modern Unicode or else they wouldn't be chars.
209+
true
210+
}
211+
}
212+
162213
#[must_use]
163214
pub fn category(&self, c: CodePoint) -> &'static str {
164215
let Some(c) = c.to_char() else {

0 commit comments

Comments
 (0)