Skip to content

Commit 0cfdda3

Browse files
gitstergitster
committed
Merge branch 'bb/unicode-9.0'
The character width table has been updated to match Unicode 9.0 * bb/unicode-9.0: unicode_width.h: update the width tables to Unicode 9.0 update_unicode.sh: remove the plane filter update_unicode.sh: automatically download newer definition files update_unicode.sh: pin the uniset repo to a known good commit update_unicode.sh: remove an unnecessary subshell level update_unicode.sh: move it into contrib/update-unicode
2 parents 3da9366 + 9e6e9ae commit 0cfdda3

File tree

6 files changed

+163
-65
lines changed

6 files changed

+163
-65
lines changed

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,6 @@
203203
/config.mak.autogen
204204
/config.mak.append
205205
/configure
206-
/unicode
207206
/tags
208207
/TAGS
209208
/cscope*

‎contrib/update-unicode/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
uniset/
2+
UnicodeData.txt
3+
EastAsianWidth.txt

‎contrib/update-unicode/README‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
TL;DR: Run update_unicode.sh after the publication of a new Unicode
2+
standard and commit the resulting unicode_widths.h file.
3+
4+
The long version
5+
================
6+
7+
The Git source code ships the file unicode_widths.h which contains
8+
tables of zero and double width Unicode code points, respectively.
9+
These tables are generated using update_unicode.sh in this directory.
10+
update_unicode.sh itself uses a third-party tool, uniset, to query two
11+
Unicode data files for the interesting code points.
12+
13+
On first run, update_unicode.sh clones uniset from Github and builds it.
14+
This requires a current-ish version of autoconf (2.69 works per December
15+
2016).
16+
17+
On each run, update_unicode.sh checks whether more recent Unicode data
18+
files are available from the Unicode consortium, and rebuilds the header
19+
unicode_widths.h with the new data. The new header can then be
20+
committed.

‎contrib/update-unicode/update_unicode.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/sh
2+
#See http://www.unicode.org/reports/tr44/
3+
#
4+
#Me Enclosing_Mark an enclosing combining mark
5+
#Mn Nonspacing_Mark a nonspacing combining mark (zero advance width)
6+
#Cf Format a format control character
7+
#
8+
cd "$(dirname "$0")"
9+
UNICODEWIDTH_H=$(git rev-parse --show-toplevel)/unicode_width.h
10+
11+
wget -N http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt \
12+
http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt &&
13+
if ! test -d uniset; then
14+
git clone https://github.com/depp/uniset.git &&
15+
( cd uniset && git checkout 4b186196dd )
16+
fi &&
17+
(
18+
cd uniset &&
19+
if ! test -x uniset; then
20+
autoreconf -i &&
21+
./configure --enable-warnings=-Werror CFLAGS='-O0 -ggdb'
22+
fi &&
23+
make
24+
) &&
25+
UNICODE_DIR=. && export UNICODE_DIR &&
26+
cat >$UNICODEWIDTH_H <<-EOF
27+
static const struct interval zero_width[] = {
28+
$(uniset/uniset --32 cat:Me,Mn,Cf + U+1160..U+11FF - U+00AD)
29+
};
30+
static const struct interval double_width[] = {
31+
$(uniset/uniset --32 eaw:F,W)
32+
};
33+
EOF

‎unicode_width.h‎

Lines changed: 107 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ static const struct interval zero_width[] = {
2525
{ 0x0825, 0x0827 },
2626
{ 0x0829, 0x082D },
2727
{ 0x0859, 0x085B },
28-
{ 0x08E4, 0x0902 },
28+
{ 0x08D4, 0x0902 },
2929
{ 0x093A, 0x093A },
3030
{ 0x093C, 0x093C },
3131
{ 0x0941, 0x0948 },
@@ -120,6 +120,7 @@ static const struct interval zero_width[] = {
120120
{ 0x17C9, 0x17D3 },
121121
{ 0x17DD, 0x17DD },
122122
{ 0x180B, 0x180E },
123+
{ 0x1885, 0x1886 },
123124
{ 0x18A9, 0x18A9 },
124125
{ 0x1920, 0x1922 },
125126
{ 0x1927, 0x1928 },
@@ -158,7 +159,7 @@ static const struct interval zero_width[] = {
158159
{ 0x1CF4, 0x1CF4 },
159160
{ 0x1CF8, 0x1CF9 },
160161
{ 0x1DC0, 0x1DF5 },
161-
{ 0x1DFC, 0x1DFF },
162+
{ 0x1DFB, 0x1DFF },
162163
{ 0x200B, 0x200F },
163164
{ 0x202A, 0x202E },
164165
{ 0x2060, 0x2064 },
@@ -171,13 +172,13 @@ static const struct interval zero_width[] = {
171172
{ 0x3099, 0x309A },
172173
{ 0xA66F, 0xA672 },
173174
{ 0xA674, 0xA67D },
174-
{ 0xA69F, 0xA69F },
175+
{ 0xA69E, 0xA69F },
175176
{ 0xA6F0, 0xA6F1 },
176177
{ 0xA802, 0xA802 },
177178
{ 0xA806, 0xA806 },
178179
{ 0xA80B, 0xA80B },
179180
{ 0xA825, 0xA826 },
180-
{ 0xA8C4, 0xA8C4 },
181+
{ 0xA8C4, 0xA8C5 },
181182
{ 0xA8E0, 0xA8F1 },
182183
{ 0xA926, 0xA92D },
183184
{ 0xA947, 0xA951 },
@@ -204,7 +205,7 @@ static const struct interval zero_width[] = {
204205
{ 0xABED, 0xABED },
205206
{ 0xFB1E, 0xFB1E },
206207
{ 0xFE00, 0xFE0F },
207-
{ 0xFE20, 0xFE2D },
208+
{ 0xFE20, 0xFE2F },
208209
{ 0xFEFF, 0xFEFF },
209210
{ 0xFFF9, 0xFFFB },
210211
{ 0x101FD, 0x101FD },
@@ -228,30 +229,46 @@ static const struct interval zero_width[] = {
228229
{ 0x11173, 0x11173 },
229230
{ 0x11180, 0x11181 },
230231
{ 0x111B6, 0x111BE },
232+
{ 0x111CA, 0x111CC },
231233
{ 0x1122F, 0x11231 },
232234
{ 0x11234, 0x11234 },
233235
{ 0x11236, 0x11237 },
236+
{ 0x1123E, 0x1123E },
234237
{ 0x112DF, 0x112DF },
235238
{ 0x112E3, 0x112EA },
236-
{ 0x11301, 0x11301 },
239+
{ 0x11300, 0x11301 },
237240
{ 0x1133C, 0x1133C },
238241
{ 0x11340, 0x11340 },
239242
{ 0x11366, 0x1136C },
240243
{ 0x11370, 0x11374 },
244+
{ 0x11438, 0x1143F },
245+
{ 0x11442, 0x11444 },
246+
{ 0x11446, 0x11446 },
241247
{ 0x114B3, 0x114B8 },
242248
{ 0x114BA, 0x114BA },
243249
{ 0x114BF, 0x114C0 },
244250
{ 0x114C2, 0x114C3 },
245251
{ 0x115B2, 0x115B5 },
246252
{ 0x115BC, 0x115BD },
247253
{ 0x115BF, 0x115C0 },
254+
{ 0x115DC, 0x115DD },
248255
{ 0x11633, 0x1163A },
249256
{ 0x1163D, 0x1163D },
250257
{ 0x1163F, 0x11640 },
251258
{ 0x116AB, 0x116AB },
252259
{ 0x116AD, 0x116AD },
253260
{ 0x116B0, 0x116B5 },
254261
{ 0x116B7, 0x116B7 },
262+
{ 0x1171D, 0x1171F },
263+
{ 0x11722, 0x11725 },
264+
{ 0x11727, 0x1172B },
265+
{ 0x11C30, 0x11C36 },
266+
{ 0x11C38, 0x11C3D },
267+
{ 0x11C3F, 0x11C3F },
268+
{ 0x11C92, 0x11CA7 },
269+
{ 0x11CAA, 0x11CB0 },
270+
{ 0x11CB2, 0x11CB3 },
271+
{ 0x11CB5, 0x11CB6 },
255272
{ 0x16AF0, 0x16AF4 },
256273
{ 0x16B30, 0x16B36 },
257274
{ 0x16F8F, 0x16F92 },
@@ -262,31 +279,59 @@ static const struct interval zero_width[] = {
262279
{ 0x1D185, 0x1D18B },
263280
{ 0x1D1AA, 0x1D1AD },
264281
{ 0x1D242, 0x1D244 },
282+
{ 0x1DA00, 0x1DA36 },
283+
{ 0x1DA3B, 0x1DA6C },
284+
{ 0x1DA75, 0x1DA75 },
285+
{ 0x1DA84, 0x1DA84 },
286+
{ 0x1DA9B, 0x1DA9F },
287+
{ 0x1DAA1, 0x1DAAF },
288+
{ 0x1E000, 0x1E006 },
289+
{ 0x1E008, 0x1E018 },
290+
{ 0x1E01B, 0x1E021 },
291+
{ 0x1E023, 0x1E024 },
292+
{ 0x1E026, 0x1E02A },
265293
{ 0x1E8D0, 0x1E8D6 },
294+
{ 0x1E944, 0x1E94A },
266295
{ 0xE0001, 0xE0001 },
267296
{ 0xE0020, 0xE007F },
268297
{ 0xE0100, 0xE01EF }
269298
};
270299
static const struct interval double_width[] = {
271-
{ /* plane */ 0x0, 0x1C },
272-
{ /* plane */ 0x1C, 0x21 },
273-
{ /* plane */ 0x21, 0x22 },
274-
{ /* plane */ 0x22, 0x23 },
275-
{ /* plane */ 0x0, 0x0 },
276-
{ /* plane */ 0x0, 0x0 },
277-
{ /* plane */ 0x0, 0x0 },
278-
{ /* plane */ 0x0, 0x0 },
279-
{ /* plane */ 0x0, 0x0 },
280-
{ /* plane */ 0x0, 0x0 },
281-
{ /* plane */ 0x0, 0x0 },
282-
{ /* plane */ 0x0, 0x0 },
283-
{ /* plane */ 0x0, 0x0 },
284-
{ /* plane */ 0x0, 0x0 },
285-
{ /* plane */ 0x0, 0x0 },
286-
{ /* plane */ 0x0, 0x0 },
287-
{ /* plane */ 0x0, 0x0 },
288300
{ 0x1100, 0x115F },
301+
{ 0x231A, 0x231B },
289302
{ 0x2329, 0x232A },
303+
{ 0x23E9, 0x23EC },
304+
{ 0x23F0, 0x23F0 },
305+
{ 0x23F3, 0x23F3 },
306+
{ 0x25FD, 0x25FE },
307+
{ 0x2614, 0x2615 },
308+
{ 0x2648, 0x2653 },
309+
{ 0x267F, 0x267F },
310+
{ 0x2693, 0x2693 },
311+
{ 0x26A1, 0x26A1 },
312+
{ 0x26AA, 0x26AB },
313+
{ 0x26BD, 0x26BE },
314+
{ 0x26C4, 0x26C5 },
315+
{ 0x26CE, 0x26CE },
316+
{ 0x26D4, 0x26D4 },
317+
{ 0x26EA, 0x26EA },
318+
{ 0x26F2, 0x26F3 },
319+
{ 0x26F5, 0x26F5 },
320+
{ 0x26FA, 0x26FA },
321+
{ 0x26FD, 0x26FD },
322+
{ 0x2705, 0x2705 },
323+
{ 0x270A, 0x270B },
324+
{ 0x2728, 0x2728 },
325+
{ 0x274C, 0x274C },
326+
{ 0x274E, 0x274E },
327+
{ 0x2753, 0x2755 },
328+
{ 0x2757, 0x2757 },
329+
{ 0x2795, 0x2797 },
330+
{ 0x27B0, 0x27B0 },
331+
{ 0x27BF, 0x27BF },
332+
{ 0x2B1B, 0x2B1C },
333+
{ 0x2B50, 0x2B50 },
334+
{ 0x2B55, 0x2B55 },
290335
{ 0x2E80, 0x2E99 },
291336
{ 0x2E9B, 0x2EF3 },
292337
{ 0x2F00, 0x2FD5 },
@@ -313,11 +358,49 @@ static const struct interval double_width[] = {
313358
{ 0xFE68, 0xFE6B },
314359
{ 0xFF01, 0xFF60 },
315360
{ 0xFFE0, 0xFFE6 },
361+
{ 0x16FE0, 0x16FE0 },
362+
{ 0x17000, 0x187EC },
363+
{ 0x18800, 0x18AF2 },
316364
{ 0x1B000, 0x1B001 },
365+
{ 0x1F004, 0x1F004 },
366+
{ 0x1F0CF, 0x1F0CF },
367+
{ 0x1F18E, 0x1F18E },
368+
{ 0x1F191, 0x1F19A },
317369
{ 0x1F200, 0x1F202 },
318-
{ 0x1F210, 0x1F23A },
370+
{ 0x1F210, 0x1F23B },
319371
{ 0x1F240, 0x1F248 },
320372
{ 0x1F250, 0x1F251 },
373+
{ 0x1F300, 0x1F320 },
374+
{ 0x1F32D, 0x1F335 },
375+
{ 0x1F337, 0x1F37C },
376+
{ 0x1F37E, 0x1F393 },
377+
{ 0x1F3A0, 0x1F3CA },
378+
{ 0x1F3CF, 0x1F3D3 },
379+
{ 0x1F3E0, 0x1F3F0 },
380+
{ 0x1F3F4, 0x1F3F4 },
381+
{ 0x1F3F8, 0x1F43E },
382+
{ 0x1F440, 0x1F440 },
383+
{ 0x1F442, 0x1F4FC },
384+
{ 0x1F4FF, 0x1F53D },
385+
{ 0x1F54B, 0x1F54E },
386+
{ 0x1F550, 0x1F567 },
387+
{ 0x1F57A, 0x1F57A },
388+
{ 0x1F595, 0x1F596 },
389+
{ 0x1F5A4, 0x1F5A4 },
390+
{ 0x1F5FB, 0x1F64F },
391+
{ 0x1F680, 0x1F6C5 },
392+
{ 0x1F6CC, 0x1F6CC },
393+
{ 0x1F6D0, 0x1F6D2 },
394+
{ 0x1F6EB, 0x1F6EC },
395+
{ 0x1F6F4, 0x1F6F6 },
396+
{ 0x1F910, 0x1F91E },
397+
{ 0x1F920, 0x1F927 },
398+
{ 0x1F930, 0x1F930 },
399+
{ 0x1F933, 0x1F93E },
400+
{ 0x1F940, 0x1F94B },
401+
{ 0x1F950, 0x1F95E },
402+
{ 0x1F980, 0x1F991 },
403+
{ 0x1F9C0, 0x1F9C0 },
321404
{ 0x20000, 0x2FFFD },
322405
{ 0x30000, 0x3FFFD }
323406
};

‎update_unicode.sh‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

0 commit comments

Comments
 (0)