Repository navigation
Expand file tree
/
Copy path_render.py
More file actions
1192 lines (1100 loc) · 60 KB
/
Copy path_render.py
File metadata and controls
1192 lines (1100 loc) · 60 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Rendering for the 2.0 API: ParsedName -> display strings.
Layering: imports nameparser._types, nameparser._lexicon for
Lexicon.default() (capitalized() with lexicon=None), _normalize and
FULL_STOPS, and nameparser._policy for the scripts that write no
initials (enforced by tests/v2/test_layering.py). Parsing code
never imports this module; ParsedName's rendering methods delegate
here via call-time imports.
Malformed str.format specs beyond unknown keys (positional fields,
bad conversions) surface the raw str.format error; only unknown KEYS
get the enriched KeyError.
"""
from __future__ import annotations
import re
import unicodedata
from collections.abc import Callable, Sequence
from nameparser._lexicon import (FULL_STOPS, Lexicon, _normalize,
_spells_an_initial)
from nameparser._policy import _NO_INITIALS, _script_matcher
from nameparser._types import (_NAME_ORDERS, FOLDED_TAG,
SHAPE_ACRONYM_TAG,
SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG,
UNCLASSIFIED_TAG, UNJOINED_CONJUNCTION_TAG,
UNJOINED_TAG, Ambiguity, ParsedName, Role,
Token)
#: Whitespace runs the collapse makes one space, save a U+202F inside a
#: word. rules/T6: "a narrow no-break space between two Mongolian
#: letters joins them into one word" -- the render side, so a spec's
#: collapse writes the word as str() does (decisions/T6, #1236). The
#: class is _tokenize._mongolian_letter, held to it by
#: tests/v2/test_regex_sync.py.
_MONGOLIAN_LETTER = "[\u180B-\u180F\u181A-\u18AF]"
_SPACES = re.compile(
rf"(?:(?!(?<={_MONGOLIAN_LETTER})\u202F(?={_MONGOLIAN_LETTER}))\s)+")
#: ASCII, Arabic, fullwidth, Ethiopic, Burmese, then N'Ko, Mongolian,
#: Manchu, Lisu, Vai and Bamum: _pipeline._state.COMMA_CHARS, which
#: render may not import (tests/v2/test_regex_sync.py holds the copy)
_COMMAS = ",\u060c\uff0c\u1363\u104a\u07f8\u1802\u1808\ua4fe\ua60d\ua6f5"
_COMMA_CHAR = re.compile(f"[{_COMMAS}]")
#: a space written before a comma of any script, which keeps the comma
_SPACE_BEFORE_COMMA = re.compile(rf"\s+([{_COMMAS}])")
_MC = re.compile(r"^(mc)(\w{2,})", re.IGNORECASE)
_WORD = re.compile(r"(\w|\.)+")
_NAME_RUN = re.compile(r"\w+")
#: str.format keys render() accepts: the seven role fields in canonical
#: order (derived from Role -- never restated) plus the derived views.
_DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names",
"generation", "credentials", "honorifics", "patronymic",
"surname2", "surname1", "surname1_particles",
"surname1_base")
_RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS
#: str.format keys initials() accepts: the three name-bearing roles.
_INITIALS_KEYS = (Role.GIVEN.value, Role.MIDDLE.value, Role.FAMILY.value)
#: Tags whose tokens contribute no initial outside the given group --
#: unless the token also carries UNJOINED_TAG, i.e. the whole part is
#: particles, in which case they are the part's only words and do
#: contribute (rules/R3, #404). The mark readmits a token carrying
#: EITHER tag: a conjunction with nothing to join is not acting as a
#: conjunction any more than a particle with nothing to join is acting
#: as a particle, so it is a name word of the part like the rest.
#: Not STABLE_TAGS -- that also contains "initial", which must contribute.
_SKIP_TAGS = frozenset({"particle", "conjunction"})
#: The given group's own skip set (rules/R3, #461): a connective
#: contributes no initial in ANY group where it is joining words, so
#: the given group no longer exempts it -- "one rule for every group".
#: The particle exemption stays: a given-group particle is a name word
#: there, which is what the whole-group exemption was for.
_SKIP_TAGS_GIVEN = frozenset({"conjunction"})
#: Either unjoined mark readmits the word it sits on.
_UNJOINED_MARKS = frozenset({UNJOINED_TAG, UNJOINED_CONJUNCTION_TAG})
# Ported verbatim from v1's "initial" pattern (minus the empty
# alternative), copied from _pipeline/_vocab.py's _INITIAL -- layering
# forbids importing the pipeline here; keep in sync by hand.
# Its one reader is _reads_as_conjunction below, and that reader only
# ever sees a bare string with no decision attached to it -- a field
# spliced in by replace(), carrying UNCLASSIFIED_TAG -- or a direct
# string call. For anything the parser classified, the tag is the
# answer and this pattern is not asked. So the two copies do not decide
# the same question about the same token -- _vocab's says what the
# parse decided, this one says what it WOULD have decided about text
# handed over unread -- which is why they must keep answering alike,
# and why test_regex_sync pins the patterns against each other.
# Deliberately NOT composed with _vocab's repertoire test (#320):
# layering forbids the import. The divergence is reachable only for a
# caller-added CJK conjunction spliced into a field, since no shipped
# vocabulary carries one, and it costs nothing there: CJK is caseless,
# so the carve-out's lower() and the fall-through's title case return
# the same string.
_INITIAL = re.compile(r"^(\w\.|[A-Z])$")
#: The hyphen clause's initial test (_cap_text, #478): _INITIAL's
#: period alternative alone. The bare-capital half would exempt a
#: capital connective on its CASE -- 'Y' in 'JOSE ORTEGA-Y-GASSET'
#: must lower -- the reading #458 removed. Pinned as that alternative
#: by tests/v2/test_regex_sync.py.
_DOTTED_INITIAL = re.compile(r"^\w\.$")
#: rules/T7's run of single letters joined by full stops and one
#: letter of it, _pipeline._vocab's _LETTER_RUN and _RUN_LETTER copied
#: by hand (layering, as for _INITIAL above) and pinned by
#: tests/v2/test_regex_sync.py. Read by initials(), which gives such a
#: word one initial per letter wherever the parse left it whole.
_LETTER = (r'[^\W\d_]'
'[̀-ͯ᪰-᷀-᷿⃐-︠-︯]*')
_LETTER_RUN = re.compile(r"(?:" + _LETTER + r"\.)+" + _LETTER + r"\.?")
_RUN_LETTER = re.compile(_LETTER + r"\.?")
# v1's "roman_numeral" pattern, the pipeline's _vocab._ROMAN copied
# by hand (layering, as for _INITIAL above) and pinned by
# tests/v2/test_regex_sync.py. Read only by _cap_word's numeral
# clause, which asks how a word the parse already put in the suffix
# role is WRITTEN, never whether it is a suffix
# (mechanisms.md#RENDER-HONORS-THE-PARSE).
_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I)
def _reads_as_conjunction(word: str, lex: Lexicon) -> bool:
"""v1's is_conjunction, asked only where the CALLER supplies no
token.
A token the parse classified carries its reading in its tags, and
case repair consults that tag first and never reaches here for a
token it holds. This function is reached only where there is no
decision to consult: a field spliced in as raw text (replace()),
carrying UNCLASSIFIED_TAG, or a direct call with no parse behind
it. It can only give the answer the parser would have given
from the word alone, the initial carve-out included ('E.' assigned
to middle is an initial, not the Italian conjunction). What it
cannot give is an answer the parse reached by looking at the whole
NAME -- rules/P8's one-case fork is the live example -- which
is why it is the fallback and the tags are the rule.
"""
# the shape on the composed spelling, as _normalize composes the
# lookup: a decomposed 'й.' is the initial its composed twin is (#542)
return bool(_normalize(word) in lex._sets.conjunctions
and not _INITIAL.fullmatch(unicodedata.normalize("NFC", word)))
def _collapse(rendered: str) -> str:
"""The #254 collapse: empty fields substitute '' and every artifact
of that is removed -- dangling empty-nickname wrappers, space runs,
space-before-comma and leading or trailing commas and spaces, the
comma of any script whose comma the parse reads."""
rendered = (rendered.replace(" ()", "")
.replace(" ''", "")
.replace(' ""', ""))
rendered = _SPACE_BEFORE_COMMA.sub(r"\1", rendered)
rendered = _SPACES.sub(" ", rendered.strip())
if rendered and _COMMA_CHAR.fullmatch(rendered[-1]):
rendered = rendered[:-1]
return rendered.strip(_COMMAS + " ")
def _format_spec(spec: str, values: dict[str, str], noun: str,
keys: tuple[str, ...]) -> str:
"""Shared tail of render()/initials(): fill the spec, enrich
unknown-KEY errors with the valid key list, collapse."""
if not isinstance(spec, str):
raise TypeError(f"spec must be a str, got {spec!r}")
try:
rendered = spec.format(**values)
except KeyError as exc:
raise KeyError(
f"unknown {noun} field {exc.args[0]!r}; valid fields: "
f"{', '.join(keys)}"
) from None
return _collapse(rendered)
def render(name: ParsedName, spec: str | None) -> str:
"""Fill the str.format spec from the seven role fields and the
derived views (empty fields substitute ''), then apply the #254
collapse. Unknown keys raise KeyError naming the valid fields.
None renders the order-aware default (_default_render)."""
if spec is None:
if name.order is not None:
return _default_render(name, name.order)
# no order recorded (a 2.x pickle, a name built by hand):
# 2.x's default spec, so such a name renders as it did
# (decisions/R6, #754)
spec = _V2_DEFAULT_SPEC
values = {key: getattr(name, key) for key in _RENDER_KEYS}
return _format_spec(spec, values, "render", _RENDER_KEYS)
# rules/R6: "the name is laid out in the order it was read in" -- this
# is the layout half: each slot a role and the decoration standing
# around its words, the given-first row being 2.x's default spec, a
# family-first row keeping the title first and the suffix last.
# decisions/R6 (#754), decisions/vocabulary-model's #754 entry, item 6.
_Slot = tuple[Role, str, str]
_DECORATION: dict[Role, tuple[str, str]] = {
Role.NICKNAME: ('"', '"'), Role.MAIDEN: ("(", ")")}
#: The role whose words follow each name role in the layout, with the
#: decoration of its own (the nickname after the given name, the maiden
#: name after the family).
_BESIDE = {Role.GIVEN: Role.NICKNAME, Role.FAMILY: Role.MAIDEN}
def _layout(order: tuple[Role, Role, Role]) -> tuple[_Slot, ...]:
roles = [Role.TITLE]
for role in order:
roles.append(role)
if role in _BESIDE:
roles.append(_BESIDE[role])
roles.append(Role.SUFFIX)
return tuple((r, *_DECORATION.get(r, ("", ""))) for r in roles)
_LAYOUTS = {order: _layout(order) for order in _NAME_ORDERS}
#: 2.x's default spec, what a name with no recorded order renders by.
_V2_DEFAULT_SPEC = ('{title} {given} "{nickname}" {middle} {family} '
"({maiden}) {suffix}")
#: rules/T2 and T3's dividers: a nakaguro or the Ethiopic wordspace,
#: or an interpunct or fullwidth full stop the parse divided at (one
#: written with a space beside it is a name word, rules/T3, so it is a
#: token and never stands in a gap).
_DOT_SEPARATORS = frozenset({"\u00B7", "\uFF0E", "\u30FB", "\uFF65",
"\u1361"})
_NAME_WORD_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY})
#: A comma the parse reads, and how it is written back: no space
#: before it and one after, a full-width comma carrying its own width.
_COMMA_WRITTEN = {",": ", ", "\u060C": "\u060C ", "\uFF0C": "\uFF0C",
"\u1363": "\u1363 ", "\u104A": "\u104A ",
"\u07F8": "\u07F8 ", "\u1802": "\u1802 ",
"\u1808": "\u1808 ", "\uA4FE": "\uA4FE ",
"\uA60D": "\uA60D ", "\uA6F5": "\uA6F5 "}
def _written_separator(gap: str) -> str | None:
"""The text written between two adjacent words, normalized, or
None where it is no separator R6 keeps (a delimiter, a marker word,
two commas, a hyphen, a zero-width or directional mark), so the
layout's own joiner stands."""
# rules/R6: "The separator is normalized by its kind."
if not gap:
return ""
core = gap.strip()
if not core:
return " "
if core in _DOT_SEPARATORS:
return core
return _COMMA_WRITTEN.get(core)
def _ordered(tokens: Sequence[Token]) -> list[Token]:
# a field's words in the order its view joins them: folded words
# first (mechanisms.md#FOLDED_TAG: "a rule that needs different
# rendering order tags the token, and the rendering views consult
# the tag")
return ([t for t in tokens if FOLDED_TAG in t.tags]
+ [t for t in tokens if FOLDED_TAG not in t.tags])
def _default_render(name: ParsedName,
order: tuple[Role, Role, Role]) -> str:
"""str()'s rendering: the layout `name.order` picks, two words side
by side both as written and as laid out keeping their written
separator (_written_separator), every other pair the layout's
joiner -- a space, the decoration between two fields, or the
suffix view's ', ' between credentials written apart."""
by_role: dict[Role, list[Token]] = {}
for tok in name.tokens:
if tok.text:
by_role.setdefault(tok.role, []).append(tok)
# where each word stands as written: consecutive positions in
# `tokens` with spans in ascending order are side by side there
position = {id(t): i for i, t in enumerate(name.tokens)}
out: list[str] = []
prev: Token | None = None
prev_close = ""
for role, open_, close in _LAYOUTS[order]:
words = by_role.get(role)
if not words:
continue
for k, tok in enumerate(_ordered(words)):
if prev is not None:
if k == 0 and (prev_close or open_):
out.append(f"{prev_close} {open_}")
else:
joined = _side_by_side(name, prev, tok, position)
if joined is None:
joined = (", " if role is Role.SUFFIX and k
and "joined" not in tok.tags else " ")
out.append(joined)
elif open_:
out.append(open_)
out.append(tok.text)
prev = tok
prev_close = close
if prev is not None:
out.append(prev_close)
# a comma that is a word of its own (a delimited nickname's) keeps
# no space before it, as 2.x's collapse wrote it
return _SPACE_BEFORE_COMMA.sub(r"\1", "".join(out))
def _side_by_side(name: ParsedName, a: Token, b: Token,
position: dict[int, int]) -> str | None:
"""The written separator between `a` and `b` where `b` is the word
written directly after `a`, else None."""
if (a.span is None or b.span is None
or position[id(b)] != position[id(a)] + 1):
return None
written = _written_separator(name.original[a.span.end:b.span.start])
# rules/R6: "A family comma between name words is never kept" --
# side by side in the layout where its given part went to the
# family (P1's fold); a comma before a suffix stays (decisions/R6)
if (written and written[0] in _COMMA_WRITTEN
and a.role in _NAME_WORD_ROLES and b.role in _NAME_WORD_ROLES):
return None
return written
# rules/R3: "initials take the first letter of each given, middle,
# and base family word; titles, suffixes, particles and nicknames
# contribute nothing"
def _first_letter(text: str) -> str:
"""The initial of `text`: its first character as its composed (NFC)
spelling reads it, written in the form `text` was written (#585).
That is the shortest start of `text` that composes to the composed
text's first character -- 'e' + U+0301 for a decomposed 'é', and
the two or three jamo of a decomposed hangul syllable, which are
letters rather than marks -- or, where a mark composing with
nothing stands inside that start, the character decomposed (even
where the writer composed part of it). A text whose first code
point holds a whole character gives text[0], as it always did --
a composed text, and one opening with a character Unicode excludes
from composition -- so a mark that composes with nothing (a
Bengali vowel sign) stays out of the initial in either form. Raises IndexError on an empty `text`, as
text[0] does."""
if text.isascii():
return text[0]
first = unicodedata.normalize("NFC", text)[0]
end = 1
while not unicodedata.normalize("NFC", text[:end]).startswith(first):
end += 1
# end == 1: one code point already holds the first letter, kept
# as written even where NFC would split it (U+0958, excluded from
# composition, is 'क' + a nukta composed; it keeps its nukta here)
if end == 1 or unicodedata.normalize("NFC", text[:end]) == first:
return text[:end]
# A mark that composes with nothing stands, in canonical order,
# between the letter and a mark that does ('o' + U+0331 + U+0301,
# where the composed spelling is 'ó' + U+0331), so no start of the
# text composes to `first` alone: write `first` decomposed, the
# form the writer used for it.
return unicodedata.normalize("NFD", first)
#: rules/R7: "a word whose first letter is in a script that writes no
#: initials contributes nothing to the initials" (decisions/R7, #754),
#: asked of the composed letter (#665), the spelling the veto in
#: _vocab reads; the conjoining jamo a decomposed syllable is written
#: in are HANGUL too since #1087.
_initialless = _script_matcher(*_NO_INITIALS, whole=True)
def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str:
"""First letter of each contributing token per group, v1 semantics:
delimiter follows each initial, separator sits between initials
within a group. Each group is ordered the way its FIELD is
ordered -- written order, except folded words, which initial
before the rest of the group (#408).
A token tagged conjunction contributes no initial in ANY group,
and one tagged particle contributes none in middle/family
(given-group particles always contribute); either unjoined mark
readmits the word it sits on, so the words of an all-particle part
(but a connective joining them, which carries neither mark) and a
connective with nothing in its part to join both count;
tags come from the pipeline --
hand-built untagged tokens all contribute, and so do the words of
a field spliced in by replace(), which the parse never read.
This view takes NO lexicon, so it has none to fall back to for
that text: `replace(family='de la vega')` initials every word of
that field where the same name parsed gives 'j. v.'
(rules/R3's Accepted bullet "This rule reads a part the parser read",
the limit this view keeps; decisions/R4 for why the fallback was tried
and dropped -- #464 is the crossing that would make it
answerable). Valid spec keys: given, middle, family."""
if not isinstance(delimiter, str):
raise TypeError(f"delimiter must be a str, got {delimiter!r}")
if not isinstance(separator, str):
raise TypeError(f"separator must be a str, got {separator!r}")
values: dict[str, str] = {}
# rules/R3's run of initials reads a lower-case run where the given,
# middle and family words are written wholly in lower case, as the
# parse does (T7), a title not asked;
# a loop, not a generator, which would resume a frame per word
all_lower = True
for t in name.tokens:
if t.role in _NAME_WORD_ROLES and not t.text.islower():
all_lower = False
break
for key in _INITIALS_KEYS:
role = Role(key)
tokens = name.tokens_for(role)
skip = _SKIP_TAGS_GIVEN if role is Role.GIVEN else _SKIP_TAGS
tokens = tuple(t for t in tokens
if not (skip & t.tags)
or _UNJOINED_MARKS & t.tags)
# mechanisms.md#FOLDED_TAG: "a rule that needs different
# rendering order tags the token, and the rendering views
# consult the tag" -- this is a rendering view, so it reads
# the tag the same way _types._text_for does, and for the same
# reason: the fold is an ORDER the parse recorded, not one the
# view is free to take again
# (mechanisms.md#RENDER-HONORS-THE-PARSE: "the parse decides
# it; the render views honor those decisions and never
# re-evaluate them"). Applied to every role this view renders,
# exactly as _text_for applies it -- the pipeline puts the tag
# on FAMILY tokens alone today, so GIVEN and MIDDLE are
# uniformity with the mechanism rather than reachable
# behavior; a producer that ever folds into another part would
# otherwise reopen #408 there.
# (a loop, not a generator, which would resume a frame per word)
letters: list[str] = []
for t in _ordered(tokens):
# rules/R3: "A word written as a run of initials gives one
# initial per letter" -- 'J. E. T. S.' for 'John E.T.
# Smith', in capitals or in a name written wholly in lower
# case -- decisions/T7 (#1296)
firsts = ([_first_letter(c) for c in _RUN_LETTER.findall(t.text)]
if "." in t.text and (all_lower or t.text.isupper())
and _LETTER_RUN.fullmatch(t.text)
else (_first_letter(t.text),))
for letter in firsts:
if letter.isascii() or not _initialless(
unicodedata.normalize("NFC", letter)):
letters.append(letter + delimiter)
values[key] = separator.join(letters)
return _format_spec(spec, values, "initials", _INITIALS_KEYS)
def _beside(text: str, at: int, step: int) -> str | None:
"""The character beside text[at], one `step` (-1 or 1) away past
any combining marks, or None at the edge of `text`. A mark belongs
to the letter before it, so a decomposed letter's neighbours are
its composed twin's (#542)."""
at += step
while 0 <= at < len(text) and unicodedata.category(text[at])[0] == "M":
at += step
return text[at] if 0 <= at < len(text) else None
def _letter_run_ge2(text: str) -> list[bool]:
"""One flag per alphanumeric character of `text`, in order: True
where it is a LETTER with a letter immediately before or after it
(_beside). Any other character -- a full stop, a space, a digit --
ends a run, and a digit's own flag is always False."""
def letter(c: str | None) -> bool:
return c is not None and c.isalpha()
return [c.isalpha() and (letter(_beside(text, i, -1))
or letter(_beside(text, i, 1)))
for i, c in enumerate(text) if c.isalnum()]
# rules/R4: "the writer split a chunk the mask keeps together, so
# the split-off letter is an initial" -- the one override below.
def _apply_mask(word: str, mask: str) -> str | None:
"""rules/R4's mask: `word` with each letter or digit recased to
the case of the mask's alphanumeric in the same position, every
other character kept where the writer put it ('ph.d.' under 'PhD'
is 'Ph.D.'). None where the two alphanumeric counts differ --
rare, since the key is the word folded, but a word in decomposed
hangul spells one syllable in two letters and still finds its
one-letter key -- and the caller falls through rather than guess.
The override cited above: a letter standing alone beside a full
stop (any of FULL_STOPS, though through capitalized() only the
ASCII period reaches here, _WORD splitting a token at any other;
beside as _beside reads it, past combining marks) is written upper
wherever the mask writes it inside a run of two or more letters
(_letter_run_ge2).
Casing goes through the whole word (word.lower()/word.upper())
when both keep its length, since per-character casing is
context-free (a Greek final sigma needs its neighbours). A mask
letter that is not lowercase reads as upper, so a titlecase
letter (Lt) is written upper -- a known limit."""
mask_chars = [c for c in mask if c.isalnum()]
if len(mask_chars) != sum(1 for c in word if c.isalnum()):
return None
mask_run = _letter_run_ge2(mask)
word_run = _letter_run_ge2(word)
lowered, uppered = word.lower(), word.upper()
same_length = len(lowered) == len(uppered) == len(word)
out: list[str] = []
at = 0
for i, c in enumerate(word):
if not c.isalnum():
out.append(c)
continue
upper = not mask_chars[at].islower() or (
mask_run[at] and c.isalpha() and not word_run[at]
and any(c is not None and c in FULL_STOPS
for c in (_beside(word, i, -1), _beside(word, i, 1))))
at += 1
if same_length:
out.append(uppered[i] if upper else lowered[i])
else:
out.append(c.upper() if upper else c.lower())
return "".join(out)
def _lowers_as_particle(normalized: str, word: str, role: Role,
tags: frozenset[str], lex: Lexicon) -> bool:
"""_cap_word's particle arm: `word` (folded to `normalized`) is a
particle doing a particle's work, so it is written in lowercase --
unless its kind is one written with a capital (a saint or a rank,
`lex._sets.capitalized_particles`) or the vocabulary records a
casing for it.
Shared with _cap_text, which asks it of each whole token before
splitting the token into words, so a marked one ("d'") is asked as
written rather than by its stem (#700)."""
return (normalized in lex._sets.particles
# rules/R4: "a particle of a kind written with a capital"
# -- a saint or a rank -- is written as a name word is
# (decisions/R4, #1069)
and normalized not in lex._sets.capitalized_particles
and not _spells_an_initial(normalized, word)
and role in (Role.MIDDLE, Role.FAMILY)
and UNJOINED_TAG not in tags
# rules/R4: "a particle the vocabulary records a
# casing for takes that casing" -- the mask below
# outranks the particle's lowercase (#604)
and normalized not in lex.capitalization_exceptions_map)
def _cap_word(word: str, role: Role, tags: frozenset[str],
lex: Lexicon, *, particle: bool = True) -> str:
# `particle=False` is the first word of the hyphen clause's opening
# part (_cap_text), or a particle opening the family name that its
# reading writes with a capital (_capital_openings): the particle arm
# is not asked, every other clause is.
# Clause order: the particle/connective arm, then the exceptions
# map as a mask, then the acronym clause (listed or by shape),
# then the numeral clause, then, per full-stop chunk, the chunk's
# own mask, the Mc prefix and title case (_title_chunks; v1's was
# str.capitalize() on the whole word). v1's cap_word had the first,
# second, a wider fifth (Mac as well as Mc) and a plainer sixth in
# this order;
# the mask ahead of the acronym clause is what gives bsc 'BSc'
# though bsc is a listed acronym too.
normalized = _normalize(word)
# rules/R4: "a part whose every word is particle vocabulary is
# repaired as ordinary name words, since none of them is doing a
# particle's work there" -- UNJOINED_TAG is that mark (#407).
# Only the PARTICLE conjunct is gated on it, and that is the rule
# rather than an omission -- rules/R4: "A CONNECTIVE the parse
# placed among the name words keeps its lowercase wherever it
# stands there, including inside a part whose other words the
# unjoined mark has turned into ordinary name words" -- so a
# conjunction keeps conjunction treatment even inside a part the
# mark has turned into ordinary name words.
# The `generation` guard below is the other half of that sentence:
# a word this vocabulary holds can ALSO be the generation it
# spells ('i' is the Catalan link and the roman numeral), and
# where the parse read the generation the token still carries the
# `conjunction` tag classify gave it -- so without the guard,
# `parse("John Quincy Smith i").capitalized(force=True)` gave
# 'John Quincy Smith i' where every release through 2.3 gave
# 'John Quincy Smith I' (#397 review). Such a token is repaired
# as the suffix it was read as, which is the rest of R4's
# sentence: "one the parse read as the generation it also spells
# is not a connective of this name at all".
# BOTH HALVES, and the role alone is not enough -- the role says
# where the word landed and the vocabulary says whether landing
# there made it a generation. A plain connective can land in the
# suffix field without being generational vocabulary at all (a
# third comma part: `Smith, John, and`), and on the role test
# alone every one of them was repaired as a name word --
# 'John Smith And' where 1.4.0, 2.0 through 2.3 and the parent
# commit all gave 'John Smith and', and 'John Smith De, Y' for a
# field spliced to suffix='de y' where R4's own Accepted
# paragraph says the vocabulary answers and the 'y' keeps its
# lowercase (#397 second review). `vocab:suffix` is classify's
# record of the vocabulary half, so the pair reads two decisions
# the parse already made and re-derives neither
# (mechanisms.md#RENDER-HONORS-THE-PARSE).
# It guards the whole test rather than the two conjunction arms
# alone, which reads as the wider claim and is not one: the
# particle arm asks for role MIDDLE or FAMILY, so a SUFFIX-roled
# token can never reach it either way.
# No SHIPPED name witnesses the difference: `particles` and
# `conjunctions` are disjoint in the default vocabulary and in
# every locale pack, so no shipped conjunction can sit in an
# all-particle part and carry the mark. That is a property of the
# shipped DATA, not an invariant -- a caller's Lexicon may give one
# word both readings, the way _pipeline/_post_rules.py's arms allow
# for. Measured: under `Lexicon.default().add('y', kind='particle')`,
# `anh y van` has
# an all-particle family whose `y` carries both tags and the mark,
# and gives 'Anh y Van'; gating this conjunct too would give
# 'Anh Y Van'. That is pinned by test_repair_keeps_a_conjunction_
# lowercase_in_a_particle_part -- until which gating it passed the
# whole suite.
# initials() does NOT match this carve-out, and since #461 that
# is a DECIDED disagreement rather than a recorded one: a
# connective that initials because it joins nothing is still not
# written the way a name is written, which is the sentence quoted
# above and this rule's own reason rather than a borrowing from
# R3. Under that same lexicon `Anh y Van` repairs to 'Anh y Van'
# and initials 'A. y. V.' -- the two views agreeing on this row
# because R2's mark readmits the word for both -- while
# `parse("Juan de y")` repairs to 'Juan de y' and initials
# 'J. y.', where they part. Pinned by
# test_initials_readmits_a_conjunction_in_a_particle_part and
# test_repair_keeps_a_lone_connective_lowercase_where_it_initials.
# That conjunct reads the TAG, not the word (#458). classify takes
# the conjunction-versus-initial decision once, over the whole
# token -- v1's is_conjunction excludes initials, so 'E.' in
# 'Scott E. Werner' is an initial and is never tagged (pinned live
# 2026-07-17) -- and a view honors that decision rather than
# taking it again from the spelling
# (mechanisms.md#RENDER-HONORS-THE-PARSE: "the render views honor
# those decisions and never re-evaluate them"), the tags being
# classify's record of it (mechanisms.md#VOCAB-TAGS: "later stages
# test tags"). Asking again was not even the same question:
# the copy of the initial pattern that stood here was the SHAPE
# half alone, and it re-decided per WORD of a token's text, so
# 'juan e-f smith' capitalized to 'Juan e-F Smith'.
# mechanisms.md#RENDER-HONORS-THE-PARSE: "a token the parse never
# saw carries no decision to honor, so a view falls back to the
# vocabulary" -- _reads_as_conjunction above, which is v1's
# predicate applied over TODAY's vocabulary rather than 1.4.0's.
# That is the honest claim and it is narrower than parity: the two
# vocabularies differ, so an assigned field can repair differently
# from 1.4.0 without this predicate differing at all. Measured in
# 2.x on the released wheel and the facade: `h.last = "хосе и
# мария сантос"` gave 'Хосе И Мария Сантос' on 1.4.0 and 'Хосе и
# Мария Сантос' in 2.x, the Cyrillic `и` being a 2.x conjunction
# and not a 1.4.0 one; `h.last = "de la vega"` gave 'de la Vega' on
# both.
# The mark, not the SPAN, is what says the text was never read:
# Parser.revise() also builds span-less tokens, from a sub-parse
# whose tags it keeps on purpose, and keying this on `span is None`
# overrode them -- `revise(middle='e-f')` repaired to 'e-F' where
# the same words parsed gave 'E-F' (#463 review).
# rules/S6: "its lower case in a name of mixed case is the suffix
# wherever the name keeps a family word" -- a suffix word that is also a
# name, in the suffix role, repairs to the one writing that keeps
# that reading; Title case would lean it back to the name. Only a
# word written `title_case_is_name` takes the cue (#1039): any other,
# an agnomen or `junior`, title-cases as any word does, and a
# credential written as a word (`kc`, #965) carries a mask, which
# this leaves to the exceptions below.
if (role is Role.SUFFIX and normalized in lex._sets.suffix_words_ambiguous
and normalized in lex._sets.title_case_name_words
and normalized not in lex.capitalization_exceptions_map):
return word.lower()
generation = role is Role.SUFFIX and "vocab:suffix" in tags
if not generation and (
(particle
and _lowers_as_particle(normalized, word, role, tags, lex))
or "conjunction" in tags
or (UNCLASSIFIED_TAG in tags
and _reads_as_conjunction(word, lex))):
return word.lower()
# v1 cap_word tries the edge-stripped form, then the period-free
# form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map). The
# value found is a MASK, not a replacement (#459): it recases the
# word as the writer punctuated it, so 'Ph.D.' under 'PhD' stays
# 'Ph.D.' and 'phd' becomes 'PhD'. v1 substituted the value, which
# is how 'md' became 'M.D.' and 'iii.' lost its period;
# rules/R4: "Repair changes case and nothing else". Role-free,
# as the map always was: an entry is the caller saying how a word
# is written wherever it stands.
undotted = normalized.replace(".", "")
for key in (normalized, undotted):
mask = lex.capitalization_exceptions_map.get(key)
if mask is not None:
masked = _apply_mask(word, mask)
if masked is not None:
return masked
# A credential acronym the exceptions map doesn't carry (mba, jd,
# qc, and md since #459 took it out of the map) is an initialism,
# not a word to title-case: a one-case name repairs to the
# acronym's caps instead of 'Mba' (#459). So is a word classify
# admitted to the credential class by its dotted SHAPE alone
# (SHAPE_ACRONYM_TAG; rules/S3's unlisted 'x.y.z.'), which has
# no vocabulary entry to be listed in. The mask above is asked
# first, which is what keeps a conventionally mixed-case acronym as
# it is written (bsc -> BSc) though it is listed here too. Gated
# on the SUFFIX role so a word that is a family name only happens
# to be in the vocabulary (anh van do) still repairs as an ordinary
# name word -- #459's given-role half, decided: repair follows the
# role the parse chose ('qc mp' -> 'Qc MP').
if role is Role.SUFFIX and (
undotted in lex._sets.suffix_acronyms
or SHAPE_ACRONYM_TAG in tags):
return word.upper()
# rules/R4: "A roman numeral the parse put in the suffix role is
# written in capitals" whether or not the vocabulary lists it: 'vi'
# through 'x' carry no vocabulary tag and title-cased to 'Vi'/'Ix'
# until #459, while 'ii'/'iii'/'iv' rode the exceptions map, which is
# why they left it. Suffix-gated for the acronym clause's reason
# -- 'Vi' is a given name -- and it is also what writes a
# generational 'i' the connective arm's `generation` guard let
# through ('Carod i' forced -> 'Carod I').
if role is Role.SUFFIX and _ROMAN.match(normalized):
return word.upper()
return _title_chunks(word, lex)
# rules/R4: "Each run of letters a full stop divides is written as a
# word of its own" and "A word that opens with a modifier letter that
# has no case" "takes its capital on the letter after that mark"; "A
# digit is not passed over that way". This is the fall-through: the
# mask, the Mc convention and title case, for a word no clause above
# claimed, asked of each chunk. decisions/R4 (#838, #1014).
def _title_chunks(word: str, lex: Lexicon) -> str:
"""`word` repaired chunk by chunk between its full stops, so a
chunk after one gets what a whole word gets ('j.mcdonald' is
'J.McDonald', 'j.macdonald' 'J.MacDonald' by its mask). The whole
word's mask was asked before the split; a chunk's is asked here
only where the word has a full stop to split at."""
chunks = word.split(".")
if len(chunks) == 1:
return _title_chunk(word)
return ".".join(_masked_chunk(chunk, lex) for chunk in chunks)
def _masked_chunk(chunk: str, lex: Lexicon) -> str:
"""`chunk` under its own mask, as a word standing alone takes it,
else _title_chunk's repair."""
mask = lex.capitalization_exceptions_map.get(_normalize(chunk))
if mask is not None:
masked = _apply_mask(chunk, mask)
if masked is not None:
return masked
return _title_chunk(chunk)
def _title_chunk(chunk: str) -> str:
"""`chunk` title-cased from its first character that is not a
modifier letter (category Lm), which also holds the apostrophe
'ʼn' upper-cases into, so 'ʼNa' is a fixpoint of repair. The Mc
prefix is decided on the composed spelling, so a chunk typed
decomposed (NFD) is read as its composed twin is: 'mc' + 'e' +
U+0301 + 'e' fails the regex's \\w{2,} at the mark (#542). The
name run after the prefix is then capitalized as written, running
on through its marks the way _sub_words does. A Mac prefix is
title-cased like any other word: a Mac name its bearers write
MacX is a mask (#1014)."""
lead = _past_modifiers(chunk, 0)
mc = _MC.match(unicodedata.normalize("NFC", chunk[lead:]))
if not mc:
return chunk[:lead] + _capitalize(chunk[lead:])
cut = end = lead + len(mc.group(1))
while run := _NAME_RUN.match(chunk, end):
end = _past_marks(chunk, run.end())
at = _past_modifiers(chunk, cut)
return (chunk[:lead] + _capitalize(chunk[lead:cut]) + chunk[cut:at]
+ _capitalize(chunk[at:end]) + chunk[end:])
# rules/R4: "A script that writes no initial capital is written in its
# lower case throughout" -- the title case of a word's first letter,
# where Georgian's is its Mkhedruli lower case. decisions/R4 (#1223).
def _capitalize(text: str) -> str:
"""str.capitalize(), except that a first letter whose lower case is
one letter that title-cases to itself, and is not the letter's own
title case, is written in that lower case: str.capitalize()
title-cases the letter as it stands, and a Mtavruli letter's title
case is itself, so 'ᲜᲘᲜᲝ' gave 'Ნინო'. The 46 Mtavruli letters are
the only ones the test reaches (measured over every codepoint,
2026-11-08); 'İ' is not among them, its lower case 'i' plus U+0307
title-casing to 'I' plus U+0307, not to itself. The one-letter guard keeps the word's
length should a later Unicode give a longer lower case that
title-cases to itself; no codepoint does today."""
out = text.capitalize()
if text:
low = text[0].lower()
if len(low) == 1 and low.title() == low != text[0].title():
return low + out[1:]
return out
def _past_modifiers(text: str, at: int) -> int:
"""`at` moved past the modifier letters (category Lm) at it."""
while at < len(text) and unicodedata.category(text[at]) == "Lm":
at += 1
return at
def _past_marks(text: str, end: int) -> int:
"""`end` moved past the combining marks (category M) at it."""
while end < len(text) and unicodedata.category(text[end])[0] == "M":
end += 1
return end
def _sub_words(cap: Callable[[str], str], text: str) -> str:
"""`text` with each word replaced by cap(word): what
_WORD.sub(cap, text) does, except that a word runs on through the
combining marks after it (#542). A combining mark is category M,
which \\w does not match, so a name typed decomposed (NFD) --
'garci' + U+0301 + 'a' -- would otherwise be two words, and the
'a' capitalized alone is 'A'. A mark heads no word of its own: one
with no word before it stays outside every word, since a word
starting with it would keep its letters lowercase under the
fall-through, which passes over a modifier letter (Lm) and not a
mark (M)."""
spans: list[list[int]] = []
for match in _WORD.finditer(text):
start, end = match.start(), _past_marks(text, match.end())
if spans and (spans[-1][1] == start
or _inner_apostrophe(text, spans[-1][1], start)):
spans[-1][1] = end
else:
spans.append([start, end])
out: list[str] = []
at = 0
for start, end in spans:
out += (text[at:start], cap(text[start:end]))
at = end
return "".join(out) + text[at:]
#: The apostrophes case repair joins a word across: U+0027, U+2018 and
#: U+2019, none of them a word character to _WORD. U+02BC, the one
#: Ukrainian orthography prescribes, is a modifier letter, which \w
#: matches, so it never divides a word; the fullwidth U+FF07 and the
#: Armenian U+055A still do.
_APOSTROPHES = frozenset("'\u2018\u2019")
#: The Cyrillic vowels a Ukrainian (я ю є ї) or Belarusian (е ё і ю я)
#: apostrophe stands before, marking the vowel as pronounced apart.
#: Lower case, and composed: a decomposed ї or ё opens with і or е.
_APOSTROPHE_VOWELS = frozenset("яюєїеёі")
# rules/R4: "An apostrophe between a Cyrillic letter and one of the
# vowels я, ю, є, ї, е, ё or і is part of the word's spelling" -- the
# test _sub_words asks before dividing a word at an apostrophe.
# decisions/R4 (#1214).
def _inner_apostrophe(text: str, end: int, start: int) -> bool:
"""The text between a word ending at `end` and the next starting at
`start` is one apostrophe standing between a Cyrillic letter (past
the combining marks after it) and one of _APOSTROPHE_VOWELS, so the
two are one word: 'МАР'ЯНА' is one word, 'О'КОННОР' two."""
if start != end + 1 or text[end] not in _APOSTROPHES:
return False
before = end - 1
while before >= 0 and unicodedata.category(text[before])[0] == "M":
before -= 1
return (before >= 0
and unicodedata.name(text[before], "").startswith("CYRILLIC")
and text[start].lower() in _APOSTROPHE_VOWELS)
def _joined_whole(text: str, role: Role, lex: Lexicon) -> str | None:
"""`text` repaired as ONE word where it holds a hyphen or slash and
the vocabulary records it whole: by its recorded casing (a mask, in
any role, as _cap_word applies it), else in capitals as a listed
acronym credential in the suffix role ('pa-c', 'otr/l'). None where
neither holds and the per-word walk takes the token (#808)."""
if "-" not in text and "/" not in text:
return None
normalized = _normalize(text)
undotted = normalized.replace(".", "")
for key in (normalized, undotted):
mask = lex.capitalization_exceptions_map.get(key)
if mask is not None:
masked = _apply_mask(text, mask)
if masked is not None:
return masked
if role is Role.SUFFIX and undotted in lex._sets.suffix_acronyms:
return text.upper()
return None
def _cap_text(text: str, role: Role, tags: frozenset[str],
lex: Lexicon, *, particle: bool = True) -> str:
# word-by-word within the token text: hyphenated names capitalize
# both sides ("mcdole-eisenhower" -> "McDole-Eisenhower"). The
# per-word walk is also why an UNCLASSIFIED token gets the
# vocabulary asked per word: the parse would have made one token
# per word of that text, so this is the granularity its answer
# would have had.
# `particle=False` leaves _cap_word's particle arm out of every
# word of the token, and the whole-token test below with it
# (capitalized, _capital_openings).
def cap(word: str) -> str:
return _cap_word(word, role, tags, lex, particle=particle)
# A particle written with a mark ("d'", "az-", "'t", #700) is one
# vocabulary word that the per-word walk below would split at the
# mark, asking the particle test of its stem ('d', 'az', 't')
# instead -- so rules/R4's "a particle is written in lowercase"
# is asked of the whole token first. Only the lowercase is taken
# here; wherever the arm declines (an all-particle part, a role
# other than middle or family, an initial's spelling, a recorded
# casing) the token goes to the walk exactly as before. A wholly
# alphabetic token is one word to the walk, which asks _cap_word this
# same arm of it, so only a token the walk could split pays here.
if (particle and not text.isalpha()
and _lowers_as_particle(_normalize(text), text, role, tags,
lex)):
return text.lower()
# rules/R4: "A credential acronym the exceptions map does not carry
# is an initialism" -- written whole when a hyphen or slash joins
# it, since the per-word walk below would capitalize 'otr/l' as the
# two name words 'Otr/L'; and a recorded casing is laid over the
# whole token for the same reason. decisions/R4 (#808).
if not text.isalpha() and (
whole := _joined_whole(text, role, lex)) is not None:
return whole
if "-" not in text:
return _sub_words(cap, text)
parts = text.split("-")
# A "named" part needs an alphanumeric, not just a _WORD match:
# _WORD also matches a run of bare periods (or underscores), so a
# part holding only punctuation -- the family TOKEN of 'jose
# .-y-garcia' is '.-y-garcia', which splits to ['.', 'y', 'garcia']
# -- is not a worded neighbour and must not count as one.
named = [at for at, part in enumerate(parts)
if any(c.isalnum() for c in part)]
if len(named) < 2:
return _sub_words(cap, text)
# rules/R4: "So is a particle opening a hyphenated word" -- the
# first word of the opening named part, asked without the particle
# arm, with a named part after it; a later word of that part
# ('la' in "d'la-vega") is asked as any word is. decisions/R4 (#1224).
def opening(part: str) -> str:
words: list[str] = []
def first_word(word: str) -> str:
# a run of bare periods or underscores is no word, as it is
# no worded part above
if any(c.isalnum() for c in word):
words.append(word)
return _cap_word(word, role, tags, lex,
particle=particle and len(words) > 1)
return _sub_words(first_word, part)
# rules/R4: "Inside a hyphenated word, a part that is
# connective vocabulary with a worded part on each side of it
# keeps its lowercase" (#478). Decided here, not in _cap_word,
# whose word has lost its neighbours. Only a part holding an
# alphanumeric is a neighbour: an empty one is skipped
# ('garcia--y-lopez' still lowers its 'y') and supplies none
# ('md-phd-'). An EDGE part stays ordinary name text ('juan e-f
# smith' keeps 'E-F'), so no word is re-read as connective or
# initial by its case (#458). A single letter with a period is an
# initial, as classify reads it, never the connective
# ('j.-e.-p. dupont' keeps 'E.'); the multi-letter 'und.' in
# 'hans smith-und.-jones' still lowers.
first, last = named[0], named[-1]
return "-".join(
part.lower()
if (first < at < last
and not _DOTTED_INITIAL.fullmatch(
unicodedata.normalize("NFC", part))
and _normalize(part) in lex._sets.conjunctions)
# rules/R4: "An elided particle opening a hyphenated word" keeps
# its lowercase, and inside one "A particle is written in
# lowercase" -- asked whole in every part, since the per-word
# walk would split it at its apostrophe and ask only its stem
# ('s'). decisions/R4 (#1280).
or (particle and part[:1] in _APOSTROPHES
and _lowers_as_particle(_normalize(part), part, role, tags,
lex))
else opening(part) if at == first
else _sub_words(cap, part)
for at, part in enumerate(parts))
def _capital_openings(tokens: Sequence[Token],
lex: Lexicon) -> frozenset[int]:
"""The indexes in `tokens` of the family words repaired without the
particle arm: a particle its reading writes with a capital
(`lex._sets.capital_opening_particles`, `ter`, `der`) whose word
before it in the family name, as the view joins the family's words
(_ordered), is no particle -- by its tag or, in a hyphenated word,
by the vocabulary of the word's last part. Opening the family, it
asks the same of the word written right before it, whatever its role
(`Van der Walt`, the given name the parse reads `Van`), and the
hyphen question of the last middle word, a surname's first half the
parse read as a middle name (`DER ZEE, ANKE MAITLAND-VAN`).
A token spliced in unread (UNCLASSIFIED_TAG) carries no tag, so the
vocabulary answers for it, as revise() would."""
capital = lex._sets.capital_opening_particles
particles = lex._sets.particles
def is_particle(t: Token) -> bool:
if UNCLASSIFIED_TAG in t.tags:
return _normalize(t.text) in particles
return "particle" in t.tags