@@ -1116,7 +1116,7 @@ def WritePreparsedUCD(out_file):
11161116 WriteFieldsRangeProps (["cp" ], start , end , props , out_file )
11171117
11181118# Write Normalizer2 input files -------------------------------------------- ***
1119- # Ported from genprops /store.c.
1119+ # Ported from gennorm /store.c.
11201120
11211121def WriteAllCC (out_file ):
11221122 out_file .write ("# Canonical_Combining_Class (ccc) values\n " );
@@ -1145,7 +1145,7 @@ def HasMapping(c):
11451145 return dt and dt != "None"
11461146
11471147
1148- def HasOneWayMapping (c , with_compat ):
1148+ def HasOneWayMapping (c ):
11491149 while True :
11501150 props = GetProps (c )
11511151 dt = props .get ("dt" )
@@ -1167,43 +1167,76 @@ def HasOneWayMapping(c, with_compat):
11671167 c = int (nfd [0 ], 16 ) # continue
11681168 else :
11691169 # c has a compatibility mapping.
1170- return with_compat
1170+ return True
11711171
11721172
1173- def WriteAllMappings (out_file , with_compat ):
1174- if with_compat :
1175- out_file .write ("\n # Canonical and compatibility decomposition mappings\n " )
1176- else :
1173+ def WriteNorm2NFCTextFile (path ):
1174+ year = datetime .date .today ().strftime ("%Y" )
1175+ with open (os .path .join (path , "nfc.txt" ), "w" ) as out_file :
1176+ out_file .write (
1177+ """# Copyright (C) 1999-""" + year +
1178+ """, International Business Machines
1179+ # Corporation and others. All Rights Reserved.
1180+ #
1181+ # file name: nfc.txt
1182+ #
1183+ # machine-generated by ICU preparseucd.py
1184+ #
1185+ # Complete data for Unicode NFC normalization.
1186+
1187+ * Unicode """ + _ucd_version + """
1188+
1189+ """ )
1190+ WriteAllCC (out_file )
11771191 out_file .write ("\n # Canonical decomposition mappings\n " )
1178- for i in xrange (len (_starts ) - 1 ):
1179- start = _starts [i ]
1180- end = _starts [i + 1 ] - 1
1181- props = _props [i ]
1182- dm = props .get ("dm" )
1183- if dm and dm [0 ] != '<' and (with_compat or props ["dt" ] == "Can" ):
1184- assert start == end
1185- separator = '>' if HasOneWayMapping (start , with_compat ) else '='
1186- out_file .write ("%04X%s%s\n " % (start , separator , dm ))
1192+ for i in xrange (len (_starts ) - 1 ):
1193+ start = _starts [i ]
1194+ end = _starts [i + 1 ] - 1
1195+ props = _props [i ]
1196+ dm = props .get ("dm" )
1197+ if dm and dm [0 ] != '<' and props ["dt" ] == "Can" :
1198+ assert start == end
1199+ # The Comp_Ex=Full_Composition_Exclusion property tells us
1200+ # whether the canonical decomposition round-trips.
1201+ separator = '>' if props .get ("Comp_Ex" ) else '='
1202+ out_file .write ("%04X%s%s\n " % (start , separator , dm ))
11871203
11881204
1189- def WriteNorm2TextFile (path , filename , with_compat ):
1205+ def WriteNorm2NFKCTextFile (path ):
11901206 year = datetime .date .today ().strftime ("%Y" )
1191- with open (os .path .join (path , filename ), "w" ) as out_file :
1207+ with open (os .path .join (path , "nfkc.txt" ), "w" ) as out_file :
11921208 out_file .write (
11931209 """# Copyright (C) 1999-""" + year +
11941210 """, International Business Machines
11951211# Corporation and others. All Rights Reserved.
11961212#
1197- # file name: """ + filename + """
1213+ # file name: nfkc.txt
11981214#
11991215# machine-generated by ICU preparseucd.py
12001216#
1217+ # Data for Unicode NFKC normalization.
1218+ # This file contains only compatibility decomposition mappings,
1219+ # plus those canonical decompositions that change from NFC round-trip mappings
1220+ # to NFKC one-way mappings.
1221+ # Use this file as the second gennorm2 input file after nfc.txt.
1222+
1223+ * Unicode """ + _ucd_version + """
1224+
12011225""" )
1202- type = "NFKC" if with_compat else "NFC"
1203- out_file .write ("# Complete data for Unicode " + type + " normalization.\n \n " )
1204- out_file .write ("* Unicode " + _ucd_version + "\n \n " )
1205- WriteAllCC (out_file )
1206- WriteAllMappings (out_file , with_compat )
1226+ for i in xrange (len (_starts ) - 1 ):
1227+ start = _starts [i ]
1228+ end = _starts [i + 1 ] - 1
1229+ props = _props [i ]
1230+ dm = props .get ("dm" )
1231+ if dm and dm [0 ] != '<' :
1232+ assert start == end
1233+ if props ["dt" ] != "Can" :
1234+ # Compatibility decomposition.
1235+ out_file .write ("%04X>%s\n " % (start , dm ))
1236+ elif not props .get ("Comp_Ex" ) and HasOneWayMapping (start ):
1237+ # NFC round-trip mapping turns into NFKC one-way mapping.
1238+ out_file .write ("%04X>%s # NFC round-trip, NFKC one-way\n " %
1239+ (start , dm ))
12071240
12081241
12091242def WriteNorm2NFKC_CFTextFile (path ):
@@ -1222,7 +1255,7 @@ def WriteNorm2NFKC_CFTextFile(path):
12221255# This file contains the Unicode NFKC_CF mappings,
12231256# extracted from the UCD file DerivedNormalizationProps.txt,
12241257# and reformatted into syntax for the gennorm2 Normalizer2 data generator tool.
1225- # Use this file as the second gennorm2 input file after nfkc.txt.
1258+ # Use this file as the third gennorm2 input file after nfc.txt and nfkc.txt.
12261259
12271260""" )
12281261 out_file .write ("* Unicode " + _ucd_version + "\n \n " )
@@ -1234,30 +1267,25 @@ def WriteNorm2NFKC_CFTextFile(path):
12341267 end = _starts [i + 1 ] - 1
12351268 props = _props [i ]
12361269 nfkc_cf = props .get ("NFKC_CF" )
1237- if nfkc_cf != None and (not nfkc_cf or nfkc_cf [0 ] != '<' ):
1238- # Merge with the previous range if possible,
1239- # or remember this range for merging.
1240- if nfkc_cf == prev_nfkc_cf and (prev_end + 1 ) == start :
1241- prev_end = end
1242- else :
1243- if prev_nfkc_cf != None :
1244- if prev_start == prev_end :
1245- out_file .write ("%04X>%s\n " % (prev_start , prev_nfkc_cf ))
1246- else :
1247- out_file .write ("%04X..%04X>%s\n " % (prev_start , prev_end , prev_nfkc_cf ))
1248- prev_start = start
1249- prev_end = end
1250- prev_nfkc_cf = nfkc_cf
1251- if prev_nfkc_cf != None :
1252- if prev_start == prev_end :
1253- out_file .write ("%04X>%s\n " % (prev_start , prev_nfkc_cf ))
1270+ # Merge with the previous range if possible,
1271+ # or remember this range for merging.
1272+ if nfkc_cf == prev_nfkc_cf and (prev_end + 1 ) == start :
1273+ prev_end = end
12541274 else :
1255- out_file .write ("%04X..%04X>%s\n " % (prev_start , prev_end , prev_nfkc_cf ))
1275+ if prev_nfkc_cf != None and (not prev_nfkc_cf or prev_nfkc_cf [0 ] != '<' ):
1276+ if prev_start == prev_end :
1277+ out_file .write ("%04X>%s\n " % (prev_start , prev_nfkc_cf ))
1278+ else :
1279+ out_file .write ("%04X..%04X>%s\n " %
1280+ (prev_start , prev_end , prev_nfkc_cf ))
1281+ prev_start = start
1282+ prev_end = end
1283+ prev_nfkc_cf = nfkc_cf
12561284
12571285
12581286def WriteNorm2 (path ):
1259- WriteNorm2TextFile (path , "nfc.txt" , False )
1260- WriteNorm2TextFile (path , "nfkc.txt" , True )
1287+ WriteNorm2NFCTextFile (path )
1288+ WriteNorm2NFKCTextFile (path )
12611289 WriteNorm2NFKC_CFTextFile (path )
12621290
12631291# Preprocessing ------------------------------------------------------------ ***
0 commit comments