Skip to content

Commit b9dc23c

Browse files
dpgeorgenotro
dpgeorge
authored and
notro
committed
extmod/modure: Add ure.sub() function and method, and tests.
This feature is controlled at compile time by MICROPY_PY_URE_SUB, disabled by default. Thanks to @dmazzella for the original patch for this feature; see adafruit#3770.
1 parent cbeac09 commit b9dc23c

5 files changed

Lines changed: 213 additions & 0 deletions

File tree

‎extmod/modure.c‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,10 +249,127 @@ STATIC mp_obj_t re_split(size_t n_args, const mp_obj_t *args) {
249249
}
250250
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_split_obj, 2, 3, re_split);
251251

252+
#if MICROPY_PY_URE_SUB
253+
254+
STATIC mp_obj_t re_sub_helper(mp_obj_t self_in, size_t n_args, const mp_obj_t *args) {
255+
mp_obj_re_t *self = MP_OBJ_TO_PTR(self_in);
256+
mp_obj_t replace = args[1];
257+
mp_obj_t where = args[2];
258+
mp_int_t count = 0;
259+
if (n_args > 3) {
260+
count = mp_obj_get_int(args[3]);
261+
// Note: flags are currently ignored
262+
}
263+
264+
size_t where_len;
265+
const char *where_str = mp_obj_str_get_data(where, &where_len);
266+
Subject subj;
267+
subj.begin = where_str;
268+
subj.end = subj.begin + where_len;
269+
int caps_num = (self->re.sub + 1) * 2;
270+
271+
vstr_t vstr_return;
272+
vstr_return.buf = NULL; // We'll init the vstr after the first match
273+
mp_obj_match_t *match = mp_local_alloc(sizeof(mp_obj_match_t) + caps_num * sizeof(char*));
274+
match->base.type = &match_type;
275+
match->num_matches = caps_num / 2; // caps_num counts start and end pointers
276+
match->str = where;
277+
278+
for (;;) {
279+
// cast is a workaround for a bug in msvc: it treats const char** as a const pointer instead of a pointer to pointer to const char
280+
memset((char*)match->caps, 0, caps_num * sizeof(char*));
281+
int res = re1_5_recursiveloopprog(&self->re, &subj, match->caps, caps_num, false);
282+
283+
// If we didn't have a match, or had an empty match, it's time to stop
284+
if (!res || match->caps[0] == match->caps[1]) {
285+
break;
286+
}
287+
288+
// Initialise the vstr if it's not already
289+
if (vstr_return.buf == NULL) {
290+
vstr_init(&vstr_return, match->caps[0] - subj.begin);
291+
}
292+
293+
// Add pre-match string
294+
vstr_add_strn(&vstr_return, subj.begin, match->caps[0] - subj.begin);
295+
296+
// Get replacement string
297+
const char* repl = mp_obj_str_get_str((mp_obj_is_callable(replace) ? mp_call_function_1(replace, MP_OBJ_FROM_PTR(match)) : replace));
298+
299+
// Append replacement string to result, substituting any regex groups
300+
while (*repl != '\0') {
301+
if (*repl == '\\') {
302+
++repl;
303+
bool is_g_format = false;
304+
if (*repl == 'g' && repl[1] == '<') {
305+
// Group specified with syntax "\g<number>"
306+
repl += 2;
307+
is_g_format = true;
308+
}
309+
310+
if ('0' <= *repl && *repl <= '9') {
311+
// Group specified with syntax "\g<number>" or "\number"
312+
unsigned int match_no = 0;
313+
do {
314+
match_no = match_no * 10 + (*repl++ - '0');
315+
} while ('0' <= *repl && *repl <= '9');
316+
if (is_g_format && *repl == '>') {
317+
++repl;
318+
}
319+
320+
if (match_no >= (unsigned int)match->num_matches) {
321+
nlr_raise(mp_obj_new_exception_arg1(&mp_type_IndexError, MP_OBJ_NEW_SMALL_INT(match_no)));
322+
}
323+
324+
const char *start_match = match->caps[match_no * 2];
325+
if (start_match != NULL) {
326+
// Add the substring matched by group
327+
const char *end_match = match->caps[match_no * 2 + 1];
328+
vstr_add_strn(&vstr_return, start_match, end_match - start_match);
329+
}
330+
}
331+
} else {
332+
// Just add the current byte from the replacement string
333+
vstr_add_byte(&vstr_return, *repl++);
334+
}
335+
}
336+
337+
// Move start pointer to end of last match
338+
subj.begin = match->caps[1];
339+
340+
// Stop substitutions if count was given and gets to 0
341+
if (count > 0 && --count == 0) {
342+
break;
343+
}
344+
}
345+
346+
mp_local_free(match);
347+
348+
if (vstr_return.buf == NULL) {
349+
// Optimisation for case of no substitutions
350+
return where;
351+
}
352+
353+
// Add post-match string
354+
vstr_add_strn(&vstr_return, subj.begin, subj.end - subj.begin);
355+
356+
return mp_obj_new_str_from_vstr(mp_obj_get_type(where), &vstr_return);
357+
}
358+
359+
STATIC mp_obj_t re_sub(size_t n_args, const mp_obj_t *args) {
360+
return re_sub_helper(args[0], n_args, args);
361+
}
362+
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_sub_obj, 3, 5, re_sub);
363+
364+
#endif
365+
252366
STATIC const mp_rom_map_elem_t re_locals_dict_table[] = {
253367
{ MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&re_match_obj) },
254368
{ MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&re_search_obj) },
255369
{ MP_ROM_QSTR(MP_QSTR_split), MP_ROM_PTR(&re_split_obj) },
370+
#if MICROPY_PY_URE_SUB
371+
{ MP_ROM_QSTR(MP_QSTR_sub), MP_ROM_PTR(&re_sub_obj) },
372+
#endif
256373
};
257374

258375
STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table);
@@ -307,11 +424,22 @@ STATIC mp_obj_t mod_re_search(size_t n_args, const mp_obj_t *args) {
307424
}
308425
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_search_obj, 2, 4, mod_re_search);
309426

427+
#if MICROPY_PY_URE_SUB
428+
STATIC mp_obj_t mod_re_sub(size_t n_args, const mp_obj_t *args) {
429+
mp_obj_t self = mod_re_compile(1, args);
430+
return re_sub_helper(self, n_args, args);
431+
}
432+
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_sub_obj, 3, 5, mod_re_sub);
433+
#endif
434+
310435
STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = {
311436
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ure) },
312437
{ MP_ROM_QSTR(MP_QSTR_compile), MP_ROM_PTR(&mod_re_compile_obj) },
313438
{ MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&mod_re_match_obj) },
314439
{ MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&mod_re_search_obj) },
440+
#if MICROPY_PY_URE_SUB
441+
{ MP_ROM_QSTR(MP_QSTR_sub), MP_ROM_PTR(&mod_re_sub_obj) },
442+
#endif
315443
{ MP_ROM_QSTR(MP_QSTR_DEBUG), MP_ROM_INT(FLAG_DEBUG) },
316444
};
317445

‎py/mpconfig.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,6 +1172,10 @@ typedef double mp_float_t;
11721172
#define MICROPY_PY_URE_MATCH_SPAN_START_END (0)
11731173
#endif
11741174

1175+
#ifndef MICROPY_PY_URE_SUB
1176+
#define MICROPY_PY_URE_SUB (0)
1177+
#endif
1178+
11751179
#ifndef MICROPY_PY_UHEAPQ
11761180
#define MICROPY_PY_UHEAPQ (0)
11771181
#endif

‎tests/extmod/ure_sub.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
try:
2+
import ure as re
3+
except ImportError:
4+
try:
5+
import re
6+
except ImportError:
7+
print('SKIP')
8+
raise SystemExit
9+
10+
try:
11+
re.sub
12+
except AttributeError:
13+
print('SKIP')
14+
raise SystemExit
15+
16+
17+
def multiply(m):
18+
return str(int(m.group(0)) * 2)
19+
20+
print(re.sub("\d+", multiply, "10 20 30 40 50"))
21+
22+
print(re.sub("\d+", lambda m: str(int(m.group(0)) // 2), "10 20 30 40 50"))
23+
24+
def A():
25+
return "A"
26+
print(re.sub('a', A(), 'aBCBABCDabcda.'))
27+
28+
print(
29+
re.sub(
30+
r'def\s+([a-zA-Z_][a-zA-Z_0-9]*)\s*\(\s*\):',
31+
'static PyObject*\npy_\\1(void){\n return;\n}\n',
32+
'\n\ndef myfunc():\n\ndef myfunc1():\n\ndef myfunc2():'
33+
)
34+
)
35+
36+
print(
37+
re.compile(
38+
'(calzino) (blu|bianco|verde) e (scarpa) (blu|bianco|verde)'
39+
).sub(
40+
r'\g<1> colore \2 con \g<3> colore \4? ...',
41+
'calzino blu e scarpa verde'
42+
)
43+
)
44+
45+
# no matches at all
46+
print(re.sub('a', 'b', 'c'))
47+
48+
# with maximum substitution count specified
49+
print(re.sub('a', 'b', '1a2a3a', 2))
50+
51+
# invalid group
52+
try:
53+
re.sub('(a)', 'b\\2', 'a')
54+
except:
55+
print('invalid group')
56+
57+
# invalid group with very large number (to test overflow in uPy)
58+
try:
59+
re.sub('(a)', 'b\\199999999999999999999999999999999999999', 'a')
60+
except:
61+
print('invalid group')

‎tests/extmod/ure_sub_unmatched.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# test re.sub with unmatched groups, behaviour changed in CPython 3.5
2+
3+
try:
4+
import ure as re
5+
except ImportError:
6+
try:
7+
import re
8+
except ImportError:
9+
print('SKIP')
10+
raise SystemExit
11+
12+
try:
13+
re.sub
14+
except AttributeError:
15+
print('SKIP')
16+
raise SystemExit
17+
18+
# first group matches, second optional group doesn't so is replaced with a blank
19+
print(re.sub(r'(a)(b)?', r'\2-\1', '1a2'))

‎tests/extmod/ure_sub_unmatched.py.exp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1-a2

0 commit comments

Comments
 (0)