Repository navigation
Expand file tree
/
Copy pathunicodectype.c
More file actions
412 lines (353 loc) · 8.49 KB
/
Copy pathunicodectype.c
File metadata and controls
412 lines (353 loc) · 8.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
/*
Unicode character type helpers.
Written by Marc-Andre Lemburg (mal@lemburg.com).
Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
Copyright (c) Corporation for National Research Initiatives.
*/
#include "Python.h"
#include "unicodeobject.h"
#define ALPHA_MASK 0x01
#define DECIMAL_MASK 0x02
#define DIGIT_MASK 0x04
#define LOWER_MASK 0x08
#define LINEBREAK_MASK 0x10
#define SPACE_MASK 0x20
#define TITLE_MASK 0x40
#define UPPER_MASK 0x80
typedef struct {
const unsigned short flags;
const Py_UNICODE upper;
const Py_UNICODE lower;
const Py_UNICODE title;
const unsigned char decimal;
const unsigned char digit;
} _PyUnicode_TypeRecord;
#include "unicodetype_db.h"
static const _PyUnicode_TypeRecord *
gettyperecord(int code)
{
int index;
if (code < 0 || code >= 65536)
index = 0;
else {
index = index1[(code>>SHIFT)];
index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
}
return &_PyUnicode_TypeRecords[index];
}
/* Returns 1 for Unicode characters having the category 'Zl' or type
'B', 0 otherwise. */
int _PyUnicode_IsLinebreak(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & LINEBREAK_MASK) != 0;
}
/* Returns the titlecase Unicode characters corresponding to ch or just
ch if no titlecase mapping is known. */
Py_UNICODE _PyUnicode_ToTitlecase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
if (ctype->title)
return ch + ctype->title;
return ch + ctype->upper;
}
/* Returns 1 for Unicode characters having the category 'Lt', 0
otherwise. */
int _PyUnicode_IsTitlecase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & TITLE_MASK) != 0;
}
/* Returns the integer decimal (0-9) for Unicode characters having
this property, -1 otherwise. */
int _PyUnicode_ToDecimalDigit(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & DECIMAL_MASK) ? ctype->decimal : -1;
}
int _PyUnicode_IsDecimalDigit(register const Py_UNICODE ch)
{
if (_PyUnicode_ToDecimalDigit(ch) < 0)
return 0;
return 1;
}
/* Returns the integer digit (0-9) for Unicode characters having
this property, -1 otherwise. */
int _PyUnicode_ToDigit(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & DIGIT_MASK) ? ctype->digit : -1;
}
int _PyUnicode_IsDigit(register const Py_UNICODE ch)
{
if (_PyUnicode_ToDigit(ch) < 0)
return 0;
return 1;
}
/* Returns the numeric value as double for Unicode characters having
this property, -1.0 otherwise. */
/* TODO: replace with unicodetype_db.h table */
double _PyUnicode_ToNumeric(register const Py_UNICODE ch)
{
switch (ch) {
case 0x3007:
return (double) 0;
case 0x09F4:
case 0x215F:
case 0x2160:
case 0x2170:
case 0x3021:
case 0x3280:
return (double) 1;
case 0x00BD:
return (double) 1 / 2;
case 0x2153:
return (double) 1 / 3;
case 0x00BC:
return (double) 1 / 4;
case 0x2155:
return (double) 1 / 5;
case 0x2159:
return (double) 1 / 6;
case 0x215B:
return (double) 1 / 8;
case 0x0BF0:
case 0x1372:
case 0x2169:
case 0x2179:
case 0x2469:
case 0x247D:
case 0x2491:
case 0x277F:
case 0x2789:
case 0x2793:
case 0x3038:
case 0x3289:
return (double) 10;
case 0x0BF1:
case 0x137B:
case 0x216D:
case 0x217D:
return (double) 100;
case 0x0BF2:
case 0x216F:
case 0x217F:
case 0x2180:
return (double) 1000;
case 0x137C:
case 0x2182:
return (double) 10000;
case 0x216A:
case 0x217A:
case 0x246A:
case 0x247E:
case 0x2492:
return (double) 11;
case 0x216B:
case 0x217B:
case 0x246B:
case 0x247F:
case 0x2493:
return (double) 12;
case 0x246C:
case 0x2480:
case 0x2494:
return (double) 13;
case 0x246D:
case 0x2481:
case 0x2495:
return (double) 14;
case 0x246E:
case 0x2482:
case 0x2496:
return (double) 15;
case 0x09F9:
case 0x246F:
case 0x2483:
case 0x2497:
return (double) 16;
case 0x16EE:
case 0x2470:
case 0x2484:
case 0x2498:
return (double) 17;
case 0x16EF:
case 0x2471:
case 0x2485:
case 0x2499:
return (double) 18;
case 0x16F0:
case 0x2472:
case 0x2486:
case 0x249A:
return (double) 19;
case 0x09F5:
case 0x2161:
case 0x2171:
case 0x3022:
case 0x3281:
return (double) 2;
case 0x2154:
return (double) 2 / 3;
case 0x2156:
return (double) 2 / 5;
case 0x1373:
case 0x2473:
case 0x2487:
case 0x249B:
case 0x3039:
return (double) 20;
case 0x09F6:
case 0x2162:
case 0x2172:
case 0x3023:
case 0x3282:
return (double) 3;
case 0x00BE:
return (double) 3 / 4;
case 0x2157:
return (double) 3 / 5;
case 0x215C:
return (double) 3 / 8;
case 0x1374:
case 0x303A:
return (double) 30;
case 0x09F7:
case 0x2163:
case 0x2173:
case 0x3024:
case 0x3283:
return (double) 4;
case 0x2158:
return (double) 4 / 5;
case 0x1375:
return (double) 40;
case 0x2164:
case 0x2174:
case 0x3025:
case 0x3284:
return (double) 5;
case 0x215A:
return (double) 5 / 6;
case 0x215D:
return (double) 5 / 8;
case 0x1376:
case 0x216C:
case 0x217C:
return (double) 50;
case 0x216E:
case 0x217E:
return (double) 500;
case 0x2181:
return (double) 5000;
case 0x2165:
case 0x2175:
case 0x3026:
case 0x3285:
return (double) 6;
case 0x1377:
return (double) 60;
case 0x2166:
case 0x2176:
case 0x3027:
case 0x3286:
return (double) 7;
case 0x215E:
return (double) 7 / 8;
case 0x1378:
return (double) 70;
case 0x2167:
case 0x2177:
case 0x3028:
case 0x3287:
return (double) 8;
case 0x1379:
return (double) 80;
case 0x2168:
case 0x2178:
case 0x3029:
case 0x3288:
return (double) 9;
case 0x137A:
return (double) 90;
default:
return (double) _PyUnicode_ToDigit(ch);
}
}
int _PyUnicode_IsNumeric(register const Py_UNICODE ch)
{
if (_PyUnicode_ToNumeric(ch) < 0.0)
return 0;
return 1;
}
#ifndef WANT_WCTYPE_FUNCTIONS
/* Returns 1 for Unicode characters having the bidirectional type
'WS', 'B' or 'S' or the category 'Zs', 0 otherwise. */
int _PyUnicode_IsWhitespace(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & SPACE_MASK) != 0;
}
/* Returns 1 for Unicode characters having the category 'Ll', 0
otherwise. */
int _PyUnicode_IsLowercase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & LOWER_MASK) != 0;
}
/* Returns 1 for Unicode characters having the category 'Lu', 0
otherwise. */
int _PyUnicode_IsUppercase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & UPPER_MASK) != 0;
}
/* Returns the uppercase Unicode characters corresponding to ch or just
ch if no uppercase mapping is known. */
Py_UNICODE _PyUnicode_ToUppercase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return ch + ctype->upper;
}
/* Returns the lowercase Unicode characters corresponding to ch or just
ch if no lowercase mapping is known. */
Py_UNICODE _PyUnicode_ToLowercase(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return ch + ctype->lower;
}
/* Returns 1 for Unicode characters having the category 'Ll', 'Lu', 'Lt',
'Lo' or 'Lm', 0 otherwise. */
int _PyUnicode_IsAlpha(register const Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & ALPHA_MASK) != 0;
}
#else
/* Export the interfaces using the wchar_t type for portability
reasons: */
int _PyUnicode_IsWhitespace(register const Py_UNICODE ch)
{
return iswspace(ch);
}
int _PyUnicode_IsLowercase(register const Py_UNICODE ch)
{
return iswlower(ch);
}
int _PyUnicode_IsUppercase(register const Py_UNICODE ch)
{
return iswupper(ch);
}
Py_UNICODE _PyUnicode_ToLowercase(register const Py_UNICODE ch)
{
return towlower(ch);
}
Py_UNICODE _PyUnicode_ToUppercase(register const Py_UNICODE ch)
{
return towupper(ch);
}
int _PyUnicode_IsAlpha(register const Py_UNICODE ch)
{
return iswalpha(ch);
}
#endif