Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions crates/stdlib/src/unicodedata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeFormArg {

#[pymodule]
mod unicodedata {
use alloc::ffi::CString;
use rustpython_unicode::{UNICODE_VERSION, UNICODE_VERSION_LEGACY};

use super::{NormalizeFormArg, unicode_core};
use crate::vm::{
Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
Expand Down Expand Up @@ -312,23 +315,25 @@ mod unicodedata {
inner: unicode_core::Ucd,
/// Owns the bytes `unidata_version` points at.
#[expect(dead_code, reason = "keeps the version string alive")]
version_owner: alloc::ffi::CString,
version_owner: CString,
#[pymember]
unidata_version: crate::vm::builtins::descriptor::CStrMember,
}

impl Ucd {
pub(super) fn new(modern: bool) -> Self {
let inner = unicode_core::Ucd::new(modern);
let text = if modern {
unicode_core::unicode_version()
UNICODE_VERSION
} else {
"3.2.0".to_owned()
UNICODE_VERSION_LEGACY
};
let version_owner = alloc::ffi::CString::new(text).unwrap_or_default();
let version_owner = CString::new(text)
.expect("Embedded Unicode version string should never have internal NULs");
let unidata_version =
crate::vm::builtins::descriptor::CStrMember::new(version_owner.as_ptr());
Self {
inner: unicode_core::Ucd::new(modern),
inner,
version_owner,
unidata_version,
}
Expand Down Expand Up @@ -503,6 +508,6 @@ mod unicodedata {

#[pyattr]
fn unidata_version(_vm: &VirtualMachine) -> String {
unicode_core::unicode_version()
UNICODE_VERSION.into()
}
}
49 changes: 37 additions & 12 deletions crates/unicode/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -457,17 +457,6 @@ fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
);
}

/// Block names and ranges from the same Unicode release as the other latest tables.
fn generate_blocks() {
let mut writer = open_writer("blocks.rs");
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
}
write_slice_post(&mut writer);
}

/// Formal name aliases (`NameAliases.txt`) and named sequences
/// (`NamedSequences.txt`). `unicodedata.lookup` consults both; `\N{}`
/// accepts only a single character, so named sequences stay out of
Expand Down Expand Up @@ -1024,6 +1013,42 @@ fn generate_numeric_type_3_2(rx: mpsc::Receiver<NumericTypeDiffs>) {
);
}

/// Block names and ranges from the same Unicode release as the other latest tables.
fn generate_blocks() {
let mut writer = open_writer("blocks.rs");
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
}
write_slice_post(&mut writer);
}

/// The version string of the latest Unicode database bundled with the standard
/// library (`unicodedata.unidata_version`).
pub fn unicode_version() {
// Report the vendored database version, not the compiler's Unicode version.
let version = open_reader("NameAliases.txt", true)
.lines()
.next()
.expect("should be able to read from vendored NameAliases.txt")
.expect("vendored NameAliases.txt must start with its Unicode version");
let version = version
.strip_prefix("# NameAliases-")
.expect("NameAliases.txt doesn't start with version info")
.strip_suffix(".txt")
.unwrap()
.trim();

println!("cargo:rustc-env=UNICODE_VERSION={version}");
}

/// Parsers that are standalone and read small files.
fn small_parsers() {
generate_blocks();
unicode_version();
}

/// Run each UCD parser in parallel.
fn drive_parsers() {
let parsers = [
Expand All @@ -1034,7 +1059,7 @@ fn drive_parsers() {
binary_props_3_2,
combining_class_3_2,
generate_name_lookups,
generate_blocks,
small_parsers,
];

let mut handles = Vec::with_capacity(parsers.len() + 2);
Expand Down
23 changes: 6 additions & 17 deletions crates/unicode/src/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ use icu_properties::props::{
};
use rustpython_wtf8::CodePoint;

pub const UNICODE_VERSION: &str = env!("UNICODE_VERSION");
pub const UNICODE_VERSION_LEGACY: &str = "3.2.0";

// Latest Unicode
include!(concat!(env!("OUT_DIR"), "/generated/blocks.rs"));
include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
Expand Down Expand Up @@ -237,20 +240,6 @@ pub fn membership_3_2(ch: u32) -> bool {
.is_ok()
}

/// The version string of the latest Unicode database bundled with the standard
/// library (`unicodedata.unidata_version`).
#[must_use]
pub fn unicode_version() -> String {
// Report the vendored database version, not the compiler's Unicode version.
include_str!("../unicode/latest/NameAliases.txt")
.lines()
.next()
.and_then(|header| header.strip_prefix("# NameAliases-"))
.and_then(|header| header.strip_suffix(".txt"))
.expect("vendored NameAliases.txt must start with its Unicode version")
.into()
}

/// Look up a name in the latest UCD (`unicodedata.lookup`).
///
/// Named sequences are included; `\N{...}` should use [`lookup_character`]
Expand Down Expand Up @@ -546,9 +535,9 @@ impl Ucd {
#[must_use]
pub fn unidata_version(&self) -> String {
if self.modern {
unicode_version()
UNICODE_VERSION.into()
} else {
"3.2.0".into()
UNICODE_VERSION_LEGACY.into()
}
}
}
Expand All @@ -565,7 +554,7 @@ mod tests {

#[test]
fn version_matches_vendored_database() {
assert_eq!(super::unicode_version(), "17.0.0");
assert_eq!(super::UNICODE_VERSION, "17.0.0");
assert_eq!(Ucd::new(true).unidata_version(), "17.0.0");
}

Expand Down
5 changes: 4 additions & 1 deletion crates/unicode/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,8 @@ pub mod grapheme;
pub mod identifier;
pub mod normalize;

pub use data::{LookupResult, Ucd, character_name, lookup_character, lookup_name, unicode_version};
pub use data::{
LookupResult, UNICODE_VERSION, UNICODE_VERSION_LEGACY, Ucd, character_name, lookup_character,
lookup_name,
};
pub use normalize::{NormalizeForm, is_normalized, normalize};
Loading