Skip to content

Commit 19aee94

Browse files
dpgeorgedpgeorge
committed
py/unicode: Clean up utf8 funcs and provide non-utf8 inline versions.
This patch provides inline versions of the utf8 helper functions for the case when unicode is disabled (MICROPY_PY_BUILTINS_STR_UNICODE set to 0). This saves code size. The unichar_charlen function is also renamed to utf8_charlen to match the other utf8 helper functions, and the signature of this function is adjusted for consistency (const char* -> const byte*, mp_uint_t -> size_t).
1 parent 49e0dd5 commit 19aee94

5 files changed

Lines changed: 21 additions & 22 deletions

File tree

‎py/misc.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,15 @@ typedef uint32_t unichar;
121121
typedef uint unichar;
122122
#endif
123123

124+
#if MICROPY_PY_BUILTINS_STR_UNICODE
124125
unichar utf8_get_char(const byte *s);
125126
const byte *utf8_next_char(const byte *s);
127+
size_t utf8_charlen(const byte *str, size_t len);
128+
#else
129+
static inline unichar utf8_get_char(const byte *s) { return *s; }
130+
static inline const byte *utf8_next_char(const byte *s) { return s + 1; }
131+
static inline size_t utf8_charlen(const byte *str, size_t len) { (void)str; return len; }
132+
#endif
126133

127134
bool unichar_isspace(unichar c);
128135
bool unichar_isalpha(unichar c);
@@ -135,7 +142,6 @@ bool unichar_islower(unichar c);
135142
unichar unichar_tolower(unichar c);
136143
unichar unichar_toupper(unichar c);
137144
mp_uint_t unichar_xdigit_value(unichar c);
138-
mp_uint_t unichar_charlen(const char *str, mp_uint_t len);
139145
#define UTF8_IS_NONASCII(ch) ((ch) & 0x80)
140146
#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80)
141147

‎py/modbuiltins.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) {
346346
const char *str = mp_obj_str_get_data(o_in, &len);
347347
#if MICROPY_PY_BUILTINS_STR_UNICODE
348348
if (MP_OBJ_IS_STR(o_in)) {
349-
len = unichar_charlen(str, len);
349+
len = utf8_charlen((const byte*)str, len);
350350
if (len == 1) {
351351
return mp_obj_new_int(utf8_get_char((const byte*)str));
352352
}

‎py/objstr.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1704,7 +1704,7 @@ STATIC mp_obj_t str_count(size_t n_args, const mp_obj_t *args) {
17041704

17051705
// if needle_len is zero then we count each gap between characters as an occurrence
17061706
if (needle_len == 0) {
1707-
return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char*)start, end - start) + 1);
1707+
return MP_OBJ_NEW_SMALL_INT(utf8_charlen(start, end - start) + 1);
17081708
}
17091709

17101710
// count the occurrences

‎py/objstrunicode.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ STATIC mp_obj_t uni_unary_op(mp_unary_op_t op, mp_obj_t self_in) {
104104
case MP_UNARY_OP_BOOL:
105105
return mp_obj_new_bool(str_len != 0);
106106
case MP_UNARY_OP_LEN:
107-
return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len));
107+
return MP_OBJ_NEW_SMALL_INT(utf8_charlen(str_data, str_len));
108108
default:
109109
return MP_OBJ_NULL; // op not supported
110110
}

‎py/unicode.c‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,9 @@ STATIC const uint8_t attr[] = {
6767
AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0
6868
};
6969

70-
// TODO: Rename to str_get_char
71-
unichar utf8_get_char(const byte *s) {
7270
#if MICROPY_PY_BUILTINS_STR_UNICODE
71+
72+
unichar utf8_get_char(const byte *s) {
7373
unichar ord = *s++;
7474
if (!UTF8_IS_NONASCII(ord)) return ord;
7575
ord &= 0x7F;
@@ -80,22 +80,14 @@ unichar utf8_get_char(const byte *s) {
8080
ord = (ord << 6) | (*s++ & 0x3F);
8181
}
8282
return ord;
83-
#else
84-
return *s;
85-
#endif
8683
}
8784

88-
// TODO: Rename to str_next_char
8985
const byte *utf8_next_char(const byte *s) {
90-
#if MICROPY_PY_BUILTINS_STR_UNICODE
9186
++s;
9287
while (UTF8_IS_CONT(*s)) {
9388
++s;
9489
}
9590
return s;
96-
#else
97-
return s + 1;
98-
#endif
9991
}
10092

10193
mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
@@ -109,21 +101,18 @@ mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
109101
return i;
110102
}
111103

112-
// TODO: Rename to str_charlen
113-
mp_uint_t unichar_charlen(const char *str, mp_uint_t len) {
114-
#if MICROPY_PY_BUILTINS_STR_UNICODE
115-
mp_uint_t charlen = 0;
116-
for (const char *top = str + len; str < top; ++str) {
104+
size_t utf8_charlen(const byte *str, size_t len) {
105+
size_t charlen = 0;
106+
for (const byte *top = str + len; str < top; ++str) {
117107
if (!UTF8_IS_CONT(*str)) {
118108
++charlen;
119109
}
120110
}
121111
return charlen;
122-
#else
123-
return len;
124-
#endif
125112
}
126113

114+
#endif
115+
127116
// Be aware: These unichar_is* functions are actually ASCII-only!
128117
bool unichar_isspace(unichar c) {
129118
return c < 128 && (attr[c] & FL_SPACE) != 0;
@@ -183,6 +172,8 @@ mp_uint_t unichar_xdigit_value(unichar c) {
183172
return n;
184173
}
185174

175+
#if MICROPY_PY_BUILTINS_STR_UNICODE
176+
186177
bool utf8_check(const byte *p, size_t len) {
187178
uint8_t need = 0;
188179
const byte *end = p + len;
@@ -210,3 +201,5 @@ bool utf8_check(const byte *p, size_t len) {
210201
}
211202
return need == 0; // no pending fragments allowed
212203
}
204+
205+
#endif

0 commit comments

Comments
 (0)