Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/common/src/encodings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ pub mod errors {
let mut out = String::with_capacity(num_chars * 4);
for c in err_str.code_points() {
let c_u32 = c.to_u32();
if let Some(c_name) = c.to_char().and_then(rustpython_unicode::character_name) {
if let Some(c_name) = rustpython_unicode::data::name_for_namereplace(c_u32) {
write!(out, "\\N{{{c_name}}}").unwrap();
} else if c_u32 >= 0x10000 {
write!(out, "\\U{c_u32:08x}").unwrap();
Expand Down
18 changes: 18 additions & 0 deletions crates/unicode/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,7 @@ fn generate_blocks() {
/// `lookup_character`.
fn generate_name_lookups() {
let mut aliases = BTreeMap::new();
let mut alias_names = Vec::new();
for line in UnicodeLineReader::from_file_name("NameAliases.txt", true) {
let name = line
.field(
Expand All @@ -563,10 +564,12 @@ fn generate_name_lookups() {
)),
)
.to_ascii_uppercase();
alias_names.push(name.clone());
aliases.insert(name, line.start);
}

let mut sequences = BTreeMap::new();
let mut sequence_names = Vec::new();
let mut sequences_reader = UnicodeLineReader::from_file_name("NamedSequences.txt", true);
for line in sequences_reader.raw_iter() {
let (name, codes) = line
Expand All @@ -582,10 +585,25 @@ fn generate_name_lookups() {
escaped.push_str(&format!("{cp:X}"));
escaped.push('}');
}
sequence_names.push(name.clone());
sequences.insert(name, escaped);
}

let mut writer = open_writer("name_lookups.rs");
// CPython's makeunicodedata.py reserves these private code points in UCD
// file order, not in the alphabetic order used by the lookup maps.
assert!(alias_names.len() < 0x200);
for (constant, names) in [
("CAPI_ALIAS_NAMES", alias_names),
("CAPI_SEQUENCE_NAMES", sequence_names),
] {
write_slice_raw(&mut writer, constant, "&str", names, |writer, names| {
for name in names {
writeln!(writer, "{name:?},").unwrap();
}
});
}

write_slice_raw(
&mut writer,
"NAME_ALIASES",
Expand Down
43 changes: 42 additions & 1 deletion crates/unicode/src/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,24 @@ fn cmp_caseless(a: &str, b: &str) -> Ordering {
.cmp(b.bytes().map(|b| b.to_ascii_uppercase()))
}

/// Name used by `namereplace`, including CPython's private alias and sequence
/// records. Unlike `unicodedata.name`, this consults two private-use ranges
/// populated in UCD source-file order by CPython's database generator.
#[must_use]
pub fn name_for_namereplace(code: u32) -> Option<String> {
let name = code
.checked_sub(0xF0000)
.and_then(|i| CAPI_ALIAS_NAMES.get(i as usize))
.or_else(|| {
code.checked_sub(0xF0200)
.and_then(|i| CAPI_SEQUENCE_NAMES.get(i as usize))
});
if let Some(name) = name {
return Some(String::from(*name));
}
char::from_u32(code).and_then(character_name)
}

fn lookup_alias(name: &str) -> Option<char> {
NAME_ALIASES
.binary_search_by(|(a, _)| cmp_caseless(a, name))
Expand Down Expand Up @@ -551,12 +569,35 @@ impl Ucd {
mod tests {
use rustpython_wtf8::CodePoint;

use super::{LookupResult, Ucd, character_name, lookup_character, lookup_name};
use super::{
CAPI_ALIAS_NAMES, CAPI_SEQUENCE_NAMES, LookupResult, Ucd, character_name, lookup_character,
lookup_name, name_for_namereplace,
};

fn cp(ch: char) -> CodePoint {
CodePoint::from(ch)
}

#[test]
fn private_name_records_follow_ucd_file_order() {
for (start, names) in [(0xF0000, CAPI_ALIAS_NAMES), (0xF0200, CAPI_SEQUENCE_NAMES)] {
assert_eq!(name_for_namereplace(start - 1), None);
assert_eq!(name_for_namereplace(start + names.len() as u32), None);
for (index, &name) in names.iter().enumerate() {
let code = start + index as u32;
assert_eq!(name_for_namereplace(code).as_deref(), Some(name));
assert_eq!(char::from_u32(code).and_then(character_name), None);
}
}
assert_eq!(name_for_namereplace(0xF0000).as_deref(), Some("NULL"));
assert_eq!(name_for_namereplace(0xD800), None);
assert_eq!(name_for_namereplace(0x110000), None);
assert_eq!(
name_for_namereplace(0x41).as_deref(),
Some("LATIN CAPITAL LETTER A")
);
}

#[test]
fn version_matches_vendored_database() {
assert_eq!(super::unicode_version(), "17.0.0");
Expand Down
6 changes: 6 additions & 0 deletions extra_tests/snippets/stdlib_unicode_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,9 @@
else:
raise AssertionError("modern numeric property leaked into Unicode 3.2")
assert unicodedata.ucd_3_2_0.numeric("\u4e00") == 1.0

# Native name consumers use the UCD file-order private alias/sequence records.
assert chr(0xF0000).encode("ascii", "namereplace") == rb"\N{NULL}"
assert chr(0xF0200).encode("ascii", "namereplace") == rb"\N{KEYCAP NUMBER SIGN}"
assert unicodedata.name(chr(0xF0000), None) is None
assert unicodedata.name(chr(0xF0200), None) is None
Loading