Skip to content

Commit a6271d2

Browse files
youknowoneyouknowone
committed
Address review feedback: test module, dead variant, WTF-8 normalization
- differential.rs: move the sweep tests into a `mod tests` block and drop the file-level allow of clippy::tests_outside_test_module and std_instead_of_alloc; the test uses alloc collections instead. - data.rs: remove the never-constructed DecompositionType::Canonical variant and its allow(unused); compatibility decomposition never produces it and canonical decomposition is handled through icu4x. - normalize.rs: is_normalized now takes &Wtf8 and checks each UTF-8 run, skipping lone surrogates, matching normalize's run-wise behavior. - unicodedata.rs: pass as_wtf8() to is_normalized. Assisted-by: Claude
1 parent 399c647 commit a6271d2

4 files changed

Lines changed: 222 additions & 204 deletions

File tree

‎crates/stdlib/src/unicodedata.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ mod unicodedata {
156156

157157
#[pymethod]
158158
fn is_normalized(&self, form: NormalizeFormArg, unistr: PyStrRef) -> bool {
159-
unicode_core::is_normalized(form.0, unistr.as_bytes())
159+
unicode_core::is_normalized(form.0, unistr.as_wtf8())
160160
}
161161

162162
#[pymethod]

‎crates/unicode/src/data.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ include!(concat!(
2929

3030
#[derive(Clone, Copy)]
3131
enum DecompositionType {
32-
#[allow(unused)]
33-
Canonical,
3432
Compat,
3533
Circle,
3634
Final,
@@ -52,7 +50,6 @@ enum DecompositionType {
5250
impl DecompositionType {
5351
const fn type_tag(self) -> &'static str {
5452
match self {
55-
Self::Canonical => "canonical",
5653
Self::Compat => "compat",
5754
Self::Circle => "circle",
5855
Self::Final => "final",

‎crates/unicode/src/normalize.rs‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
use core::str::FromStr;
66

77
use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};
8-
use rustpython_wtf8::{Wtf8, Wtf8Buf};
8+
use rustpython_wtf8::{Wtf8, Wtf8Buf, Wtf8Chunk};
99

1010
/// One of the four Unicode normalization forms.
1111
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -60,21 +60,27 @@ pub fn normalize(form: NormalizeForm, text: &Wtf8) -> Wtf8Buf {
6060
}
6161
}
6262

63-
/// Whether `bytes` (interpreted as UTF-8) is already in `form`
64-
/// (`unicodedata.is_normalized`).
63+
/// Whether `text` is already in `form` (`unicodedata.is_normalized`).
64+
///
65+
/// Lone surrogates split the text into valid UTF-8 runs; each run is checked
66+
/// independently, matching the run-wise normalization performed by [`normalize`].
6567
#[must_use]
66-
pub fn is_normalized(form: NormalizeForm, bytes: &[u8]) -> bool {
67-
match form {
68-
NormalizeForm::Nfc => ComposingNormalizerBorrowed::new_nfc().is_normalized_utf8(bytes),
69-
NormalizeForm::Nfkc => ComposingNormalizerBorrowed::new_nfkc().is_normalized_utf8(bytes),
70-
NormalizeForm::Nfd => DecomposingNormalizerBorrowed::new_nfd().is_normalized_utf8(bytes),
71-
NormalizeForm::Nfkd => DecomposingNormalizerBorrowed::new_nfkd().is_normalized_utf8(bytes),
72-
}
68+
pub fn is_normalized(form: NormalizeForm, text: &Wtf8) -> bool {
69+
let check: fn(&str) -> bool = match form {
70+
NormalizeForm::Nfc => |s| ComposingNormalizerBorrowed::new_nfc().is_normalized(s),
71+
NormalizeForm::Nfkc => |s| ComposingNormalizerBorrowed::new_nfkc().is_normalized(s),
72+
NormalizeForm::Nfd => |s| DecomposingNormalizerBorrowed::new_nfd().is_normalized(s),
73+
NormalizeForm::Nfkd => |s| DecomposingNormalizerBorrowed::new_nfkd().is_normalized(s),
74+
};
75+
text.chunks().all(|chunk| match chunk {
76+
Wtf8Chunk::Utf8(s) => check(s),
77+
Wtf8Chunk::Surrogate(_) => true,
78+
})
7379
}
7480

7581
#[cfg(test)]
7682
mod tests {
77-
use rustpython_wtf8::Wtf8Buf;
83+
use rustpython_wtf8::{CodePoint, Wtf8Buf};
7884

7985
use super::{NormalizeForm, is_normalized, normalize};
8086

@@ -83,7 +89,23 @@ mod tests {
8389
let composed = Wtf8Buf::from("é");
8490
let decomposed = normalize(NormalizeForm::Nfd, &composed);
8591
assert_eq!(normalize(NormalizeForm::Nfc, &decomposed), composed);
86-
assert!(is_normalized(NormalizeForm::Nfc, "é".as_bytes()));
87-
assert!(!is_normalized(NormalizeForm::Nfd, "é".as_bytes()));
92+
assert!(is_normalized(
93+
NormalizeForm::Nfc,
94+
Wtf8Buf::from("é").as_ref()
95+
));
96+
assert!(!is_normalized(
97+
NormalizeForm::Nfd,
98+
Wtf8Buf::from("é").as_ref()
99+
));
100+
}
101+
102+
#[test]
103+
fn is_normalized_skips_lone_surrogates() {
104+
// A lone surrogate splits the text into UTF-8 runs; each run is checked
105+
// independently, so a surrogate next to normalized text stays normalized.
106+
let mut buf = Wtf8Buf::from("é");
107+
buf.push(CodePoint::from_u32(0xD800).unwrap());
108+
assert!(is_normalized(NormalizeForm::Nfc, &buf));
109+
assert!(!is_normalized(NormalizeForm::Nfd, &buf));
88110
}
89111
}

0 commit comments

Comments
 (0)