Skip to content

Commit f72bdf2

Browse files
committed
ICU-9023 reduce norm2/nfkc.txt to a delta over nfc.txt
X-SVN-Rev: 31200
1 parent 1481912 commit f72bdf2

1 file changed

Lines changed: 73 additions & 45 deletions

File tree

‎tools/unicode/py/preparseucd.py‎

Lines changed: 73 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1116,7 +1116,7 @@ def WritePreparsedUCD(out_file):
11161116
WriteFieldsRangeProps(["cp"], start, end, props, out_file)
11171117

11181118
# Write Normalizer2 input files -------------------------------------------- ***
1119-
# Ported from genprops/store.c.
1119+
# Ported from gennorm/store.c.
11201120

11211121
def WriteAllCC(out_file):
11221122
out_file.write("# Canonical_Combining_Class (ccc) values\n");
@@ -1145,7 +1145,7 @@ def HasMapping(c):
11451145
return dt and dt != "None"
11461146

11471147

1148-
def HasOneWayMapping(c, with_compat):
1148+
def HasOneWayMapping(c):
11491149
while True:
11501150
props = GetProps(c)
11511151
dt = props.get("dt")
@@ -1167,43 +1167,76 @@ def HasOneWayMapping(c, with_compat):
11671167
c = int(nfd[0], 16) # continue
11681168
else:
11691169
# c has a compatibility mapping.
1170-
return with_compat
1170+
return True
11711171

11721172

1173-
def WriteAllMappings(out_file, with_compat):
1174-
if with_compat:
1175-
out_file.write("\n# Canonical and compatibility decomposition mappings\n")
1176-
else:
1173+
def WriteNorm2NFCTextFile(path):
1174+
year = datetime.date.today().strftime("%Y")
1175+
with open(os.path.join(path, "nfc.txt"), "w") as out_file:
1176+
out_file.write(
1177+
"""# Copyright (C) 1999-""" + year +
1178+
""", International Business Machines
1179+
# Corporation and others. All Rights Reserved.
1180+
#
1181+
# file name: nfc.txt
1182+
#
1183+
# machine-generated by ICU preparseucd.py
1184+
#
1185+
# Complete data for Unicode NFC normalization.
1186+
1187+
* Unicode """ + _ucd_version + """
1188+
1189+
""")
1190+
WriteAllCC(out_file)
11771191
out_file.write("\n# Canonical decomposition mappings\n")
1178-
for i in xrange(len(_starts) - 1):
1179-
start = _starts[i]
1180-
end = _starts[i + 1] - 1
1181-
props = _props[i]
1182-
dm = props.get("dm")
1183-
if dm and dm[0] != '<' and (with_compat or props["dt"] == "Can"):
1184-
assert start == end
1185-
separator = '>' if HasOneWayMapping(start, with_compat) else '='
1186-
out_file.write("%04X%s%s\n" % (start, separator, dm))
1192+
for i in xrange(len(_starts) - 1):
1193+
start = _starts[i]
1194+
end = _starts[i + 1] - 1
1195+
props = _props[i]
1196+
dm = props.get("dm")
1197+
if dm and dm[0] != '<' and props["dt"] == "Can":
1198+
assert start == end
1199+
# The Comp_Ex=Full_Composition_Exclusion property tells us
1200+
# whether the canonical decomposition round-trips.
1201+
separator = '>' if props.get("Comp_Ex") else '='
1202+
out_file.write("%04X%s%s\n" % (start, separator, dm))
11871203

11881204

1189-
def WriteNorm2TextFile(path, filename, with_compat):
1205+
def WriteNorm2NFKCTextFile(path):
11901206
year = datetime.date.today().strftime("%Y")
1191-
with open(os.path.join(path, filename), "w") as out_file:
1207+
with open(os.path.join(path, "nfkc.txt"), "w") as out_file:
11921208
out_file.write(
11931209
"""# Copyright (C) 1999-""" + year +
11941210
""", International Business Machines
11951211
# Corporation and others. All Rights Reserved.
11961212
#
1197-
# file name: """ + filename + """
1213+
# file name: nfkc.txt
11981214
#
11991215
# machine-generated by ICU preparseucd.py
12001216
#
1217+
# Data for Unicode NFKC normalization.
1218+
# This file contains only compatibility decomposition mappings,
1219+
# plus those canonical decompositions that change from NFC round-trip mappings
1220+
# to NFKC one-way mappings.
1221+
# Use this file as the second gennorm2 input file after nfc.txt.
1222+
1223+
* Unicode """ + _ucd_version + """
1224+
12011225
""")
1202-
type = "NFKC" if with_compat else "NFC"
1203-
out_file.write("# Complete data for Unicode " + type + " normalization.\n\n")
1204-
out_file.write("* Unicode " + _ucd_version + "\n\n")
1205-
WriteAllCC(out_file)
1206-
WriteAllMappings(out_file, with_compat)
1226+
for i in xrange(len(_starts) - 1):
1227+
start = _starts[i]
1228+
end = _starts[i + 1] - 1
1229+
props = _props[i]
1230+
dm = props.get("dm")
1231+
if dm and dm[0] != '<':
1232+
assert start == end
1233+
if props["dt"] != "Can":
1234+
# Compatibility decomposition.
1235+
out_file.write("%04X>%s\n" % (start, dm))
1236+
elif not props.get("Comp_Ex") and HasOneWayMapping(start):
1237+
# NFC round-trip mapping turns into NFKC one-way mapping.
1238+
out_file.write("%04X>%s # NFC round-trip, NFKC one-way\n" %
1239+
(start, dm))
12071240

12081241

12091242
def WriteNorm2NFKC_CFTextFile(path):
@@ -1222,7 +1255,7 @@ def WriteNorm2NFKC_CFTextFile(path):
12221255
# This file contains the Unicode NFKC_CF mappings,
12231256
# extracted from the UCD file DerivedNormalizationProps.txt,
12241257
# and reformatted into syntax for the gennorm2 Normalizer2 data generator tool.
1225-
# Use this file as the second gennorm2 input file after nfkc.txt.
1258+
# Use this file as the third gennorm2 input file after nfc.txt and nfkc.txt.
12261259
12271260
""")
12281261
out_file.write("* Unicode " + _ucd_version + "\n\n")
@@ -1234,30 +1267,25 @@ def WriteNorm2NFKC_CFTextFile(path):
12341267
end = _starts[i + 1] - 1
12351268
props = _props[i]
12361269
nfkc_cf = props.get("NFKC_CF")
1237-
if nfkc_cf != None and (not nfkc_cf or nfkc_cf[0] != '<'):
1238-
# Merge with the previous range if possible,
1239-
# or remember this range for merging.
1240-
if nfkc_cf == prev_nfkc_cf and (prev_end + 1) == start:
1241-
prev_end = end
1242-
else:
1243-
if prev_nfkc_cf != None:
1244-
if prev_start == prev_end:
1245-
out_file.write("%04X>%s\n" % (prev_start, prev_nfkc_cf))
1246-
else:
1247-
out_file.write("%04X..%04X>%s\n" % (prev_start, prev_end, prev_nfkc_cf))
1248-
prev_start = start
1249-
prev_end = end
1250-
prev_nfkc_cf = nfkc_cf
1251-
if prev_nfkc_cf != None:
1252-
if prev_start == prev_end:
1253-
out_file.write("%04X>%s\n" % (prev_start, prev_nfkc_cf))
1270+
# Merge with the previous range if possible,
1271+
# or remember this range for merging.
1272+
if nfkc_cf == prev_nfkc_cf and (prev_end + 1) == start:
1273+
prev_end = end
12541274
else:
1255-
out_file.write("%04X..%04X>%s\n" % (prev_start, prev_end, prev_nfkc_cf))
1275+
if prev_nfkc_cf != None and (not prev_nfkc_cf or prev_nfkc_cf[0] != '<'):
1276+
if prev_start == prev_end:
1277+
out_file.write("%04X>%s\n" % (prev_start, prev_nfkc_cf))
1278+
else:
1279+
out_file.write("%04X..%04X>%s\n" %
1280+
(prev_start, prev_end, prev_nfkc_cf))
1281+
prev_start = start
1282+
prev_end = end
1283+
prev_nfkc_cf = nfkc_cf
12561284

12571285

12581286
def WriteNorm2(path):
1259-
WriteNorm2TextFile(path, "nfc.txt", False)
1260-
WriteNorm2TextFile(path, "nfkc.txt", True)
1287+
WriteNorm2NFCTextFile(path)
1288+
WriteNorm2NFKCTextFile(path)
12611289
WriteNorm2NFKC_CFTextFile(path)
12621290

12631291
# Preprocessing ------------------------------------------------------------ ***

0 commit comments

Comments
 (0)