Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions mrbgems/mruby-regexp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ simulation) with backtracking fallback.
- `(?!...)` negative lookahead
- `(?<=...)` positive lookbehind (fixed-length only)
- `(?<!...)` negative lookbehind (fixed-length only)
- `(?imx-imx)` options for the rest of the enclosing group,
`(?imx-imx:...)` options for the group's own body

### Character Escapes

Expand Down Expand Up @@ -199,11 +201,6 @@ pattern analysis.
start and keep the last match that qualifies. The cost grows with the
number of positions a match starts at, where CRuby hands the search to
Onig.
- **No inline extended mode**: `(?x)` and `(?x:...)` raise a
`RegexpError`, because extended mode is applied to the whole pattern
before it is parsed. A `-x` is accepted and ignored, so inside a
pattern that is itself extended it does not bring back the
whitespace that pass removed.

## Named Captures

Expand Down
135 changes: 106 additions & 29 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -1044,10 +1044,11 @@ compute_fixed_len(re_compiler *c, uint32_t start, uint32_t end, int *chars_out)
and a further run of i/m/x to switch off, then stops at the terminator
(':' or ')'). `base` is the option set in effect on entry; the resulting
set is returned. Ruby's inline letters are i (IGNORECASE), m (DOTALL),
x (EXTENDED). Extended mode is applied by a whole-pattern preprocessing
pass that runs before the parser, so it cannot be scoped inline:
enabling it is rejected here, and see the 'x' branch below for why
disabling it is not. */
x (EXTENDED). The x bit is carried like the other two but nothing in the
parser reads it: free-spacing is applied by preprocess_pattern() before
the parser runs, and that pass tracks the same (?x) and (?-x) scopes over
the pattern as written, so by the time the letter is read here the
whitespace it governed is already gone or already kept. */
static uint32_t
parse_inline_flags(re_compiler *c, uint32_t base)
{
Expand All @@ -1058,22 +1059,7 @@ parse_inline_flags(re_compiler *c, uint32_t base)
uint32_t bit;
if (oc == 'i') bit = RE_FLAG_IGNORECASE;
else if (oc == 'm') bit = RE_FLAG_DOTALL;
else if (oc == 'x') {
if (!negate) {
compile_error(c, "inline extended mode (?x) is not supported");
return base; /* unreached: compile_error longjmps */
}
/* A '-x' is accepted and dropped. Regexp#to_s names every flag that
is off, so its result carries one whenever the pattern is not
extended, and rejecting it would make interpolation and
Regexp.new(re.to_s) raise for such a Regexp. Dropping it is exact
there, since the flag is already off. Inside a pattern that is
itself extended it is not: the preprocessing pass has removed the
whitespace by now and the scope cannot get it back. */
seen = TRUE;
next_char(c);
continue;
}
else if (oc == 'x') bit = RE_FLAG_EXTENDED;
else if (oc == '-' && !negate) { negate = TRUE; next_char(c); continue; }
else break;
if (negate) off |= bit;
Expand Down Expand Up @@ -1799,15 +1785,25 @@ compile_alt(re_compiler *c)
}

/*
* Does the pattern hold a (?# comment group opener? Cheap pre-check so an
* ordinary pattern without one skips preprocess_pattern() and its malloc.
* Does the pattern hold a group preprocess_pattern() rewrites: a (?#
* comment group, or an inline option group that turns x on, as in (?x),
* (?x:...) or (?ix-m:...)? Cheap pre-check so an ordinary pattern without
* one skips the pass and its malloc. An escaped or bracketed "(?" is a
* false positive here, which costs the pass and nothing else: the pass
* itself steps over escapes and classes.
*/
static mrb_bool
has_comment_group(const char *src, mrb_int len)
has_rewritten_group(const char *src, mrb_int len)
{
const char *p = src, *end = src + len;
while (p < end && (p = (const char*)memchr(p, '(', (size_t)(end - p))) != NULL) {
if (end - p >= 3 && p[1] == '?' && p[2] == '#') return TRUE;
if (end - p >= 3 && p[1] == '?') {
if (p[2] == '#') return TRUE;
/* The letters before a '-' are the ones switched on. */
for (const char *q = p + 2; q < end && (*q == 'i' || *q == 'm' || *q == 'x'); q++) {
if (*q == 'x') return TRUE;
}
}
p++;
}
return FALSE;
Expand Down Expand Up @@ -1888,24 +1884,76 @@ skip_uninterpreted(const char *src, const char *end, mrb_bool *in_class)
return NULL;
}

/* Read the letters of an inline option group whose "(?" starts at `src`,
as far as the free-spacing pass needs them: whether the group is one,
whether it is the toggle form (?imx) rather than the scoped (?imx:...),
and what it makes of x. `*x_on` is left as it was when the letters do
not name x. Returns the terminator's position, or NULL when the bytes
are not an option group at all, which includes every malformed one: the
parser reads those bytes too and reports them, and this pass has only to
agree with it about the well-formed ones. */
static const char*
scan_option_group(const char *src, const char *end, mrb_bool *toggle, mrb_bool *x_on)
{
if (end - src < 3 || src[1] != '?') return NULL;
const char *q = src + 2;
mrb_bool negate = FALSE, seen = FALSE;
for (; q < end; q++) {
if (*q == 'x') { *x_on = !negate; seen = TRUE; }
else if (*q == 'i' || *q == 'm') seen = TRUE;
else if (*q == '-' && !negate) negate = TRUE;
else break;
}
if (!seen || q >= end || (*q != ')' && *q != ':')) return NULL;
*toggle = (*q == ')');
return q;
}

/* The free-spacing pass's scope stack: one bit per open group, holding
what extended mode was outside it. */
static void
scope_set(uint8_t *scope, mrb_int depth, mrb_bool extended)
{
uint8_t bit = (uint8_t)(1u << (depth & 7));
if (extended) scope[depth >> 3] |= bit;
else scope[depth >> 3] &= (uint8_t)~bit;
}

static mrb_bool
scope_get(const uint8_t *scope, mrb_int depth)
{
return (scope[depth >> 3] >> (depth & 7)) & 1;
}

/*
* Rewrite the pattern before the parser sees it.
* Removes (?#...) comment groups always, and in extended mode (/x) also
* whitespace and #comments.
* Removes (?#...) comment groups always, and wherever extended mode is in
* effect also whitespace and #comments. Extended mode is in effect from the
* start when the Regexp carries /x, and it is switched inside the pattern
* by the inline option groups: (?x) and (?-x) for the rest of the enclosing
* group, (?x:...) and (?-x:...) for their own body. So this pass keeps a
* stack of one bit per open group, pushed at every '(' it interprets and
* popped at every ')', which is what lets a toggle end where its group does.
* That is the same scoping the parser gives i and m; x has to be resolved
* here because the bytes it governs are gone before the parser reads them.
* Whitespace inside [...] character classes is preserved, and so is a (?#
* written there, which is a literal member rather than a comment group.
* Escaped characters (\ followed by anything) are preserved.
* skip_uninterpreted() decides which bytes those are.
*
* The buffer comes from the GC arena, which holds it until the caller's frame
* is gone: the parser reads it from beginning to end, and every raise in
* between leaves this function nothing to be reached through.
* between leaves this function nothing to be reached through. The scope
* stack lives behind the rewritten pattern in the same allocation: a group
* opener is one byte of source, so len bits is room for every group.
*/
static char*
preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
mrb_bool extended, mrb_int *out_len)
{
char *buf = (char*)mrb_temp_alloc(mrb, len);
char *buf = (char*)mrb_temp_alloc(mrb, (size_t)len + ((size_t)len + 7) / 8);
uint8_t *scope = (uint8_t*)buf + len;
mrb_int depth = 0;
mrb_int o = 0;
mrb_bool in_class = FALSE;
const char *end = src + len;
Expand All @@ -1920,6 +1968,12 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
while (src < skip) buf[o++] = *src++;
continue;
}
if (ch == ')') {
/* An unmatched ')' has no scope to close; the parser reports it. */
if (depth > 0) extended = scope_get(scope, --depth);
buf[o++] = *src++;
continue;
}
if (ch == '(' && end - src >= 3 && src[1] == '?' && src[2] == '#') {
/* Comment group: ends at the first ')' not preceded by a backslash.
It does not nest, so (?#a(?#b)) closes at the first ')' and leaves
Expand All @@ -1943,6 +1997,29 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
buf[o++] = *src++;
continue;
}
if (ch == '(') {
mrb_bool toggle = FALSE, x_on = extended;
const char *term = scan_option_group(src, end, &toggle, &x_on);
if (term && toggle) {
/* (?imx) changes the rest of the enclosing group and opens none of
its own, so the stack is left alone. The group is copied through
for the parser, which applies i and m from the same letters. */
while (src <= term) buf[o++] = *src++;
extended = x_on;
continue;
}
/* Every other '(' opens a group whose ')' restores what x is now:
(?imx:...) after setting it for its body, and a plain, named,
non-capturing or lookaround group after leaving it as it is. */
scope_set(scope, depth++, extended);
if (term) {
while (src <= term) buf[o++] = *src++;
extended = x_on;
continue;
}
buf[o++] = *src++;
continue;
}
if (extended) {
if (ch == '#') {
/* skip to end of line */
Expand Down Expand Up @@ -2167,7 +2244,7 @@ mrb_re_compile(mrb_state *mrb, mrb_regexp_pattern *pat,
c.orig = pattern;
c.orig_end = pattern + len;

if ((flags & RE_FLAG_EXTENDED) || has_comment_group(pattern, len)) {
if ((flags & RE_FLAG_EXTENDED) || has_rewritten_group(pattern, len)) {
mrb_int slen;
pattern = preprocess_pattern(mrb, pattern, len,
(flags & RE_FLAG_EXTENDED) != 0, &slen);
Expand Down
7 changes: 7 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,13 @@ def initialize(first, second)
# the form recompiles, and the flags it names do not leak either way
assert_true Regexp.new(Regexp.new("abc", Regexp::IGNORECASE).to_s).match?("ABC")
assert_false Regexp.new(Regexp.new("abc").to_s + "d", Regexp::IGNORECASE).match?("ABCd")

# an extended pattern round-trips too: the "(?x-mi:" it prints is read
# back as free-spacing over the source it wraps
assert_equal "(?x-mi:a b)", Regexp.new("a b", Regexp::EXTENDED).to_s
assert_true Regexp.new(Regexp.new("a b", Regexp::EXTENDED).to_s).match?("ab")
assert_false Regexp.new(Regexp.new("a b", Regexp::EXTENDED).to_s).match?("a b")
assert_true(/#{Regexp.new("a b", Regexp::EXTENDED)}c d/.match?("abc d"))
end

assert("Regexp#to_s - interpolation") do
Expand Down
45 changes: 33 additions & 12 deletions mrbgems/mruby-regexp/test/regexp_syntax.rb
Original file line number Diff line number Diff line change
Expand Up @@ -341,21 +341,42 @@
assert_equal 0, (/(?m:a.b)/ =~ "a\nb")
assert_nil (/a.b/ =~ "a\nb")

# x (extended) cannot be scoped inline with the current architecture, so
# turning it on is rejected.
assert_raise(RegexpError) { Regexp.new("(?x)a b") }
assert_raise(RegexpError) { Regexp.new("(?x:a b)") }

# Turning it off is accepted, because Regexp#to_s writes a '-x' for every
# pattern that is not extended and that form has to recompile.
# x (extended) is scoped inline like the other two: the toggle form
# reaches the end of the enclosing group, the scoped form its own body.
assert_equal 0, (/(?x)a b/ =~ "ab")
assert_nil (/(?x)a b/ =~ "a b")
assert_equal 0, (/(?x:a b)c d/ =~ "abc d")
assert_equal 0, (/(a(?x)b c)d e/ =~ "abcd e")
assert_equal 0, (/(?<n>(?x)a b)c d/ =~ "abc d")
assert_equal 0, (/(?=(?x)a b)ab c/ =~ "ab c")
assert_equal 0, (/(?xi)a b/ =~ "AB")
assert_equal 0, (/(?x)a b(?-x)c d/ =~ "abc d")
assert_equal 0, (/(?x:a(?-x:b c)d)/ =~ "ab cd")

# Free-spacing follows the scope: a comment runs to the end of the line,
# a (?# group is dropped as always, and an escape or a class keeps its
# whitespace.
assert_equal 0, (/(?x)a#c
b/ =~ "ab")
assert_equal 0, (/(?x)(?#c d) e/ =~ "e")
assert_equal 0, (/(?x)a\ b/ =~ "a b")
assert_equal 0, (/(?x)[a b]/ =~ " ")
assert_equal 0, (/[(?x] a/ =~ "( a")
assert_equal 0, (/\(?x a/ =~ "(x a")

# A comment swallows the rest of its line, closing parenthesis included,
# as it does in CRuby.
assert_true Regexp.new("(?x)a #b)").match?("a")
assert_raise(RegexpError) { Regexp.new("(?x)a #b\n(c") }

# Turning it off inside a pattern that is itself extended brings the
# whitespace back for that scope.
assert_equal 0, (/(?-x:a b)/ =~ "a b")
assert_equal 0, (/(?i-mx:a)b/ =~ "Ab")
assert_true Regexp.new("(?-mix:a b)").match?("a b")

# The '-x' is dropped rather than honoured, so in a pattern that is
# itself extended the whitespace stays stripped. CRuby matches "a b"
# here.
assert_true Regexp.new("(?-x:a b)", Regexp::EXTENDED).match?("ab")
assert_true Regexp.new("(?-x:a b)", Regexp::EXTENDED).match?("a b")
assert_true Regexp.new("(?-x)a b", Regexp::EXTENDED).match?("a b")
assert_true Regexp.new("(?x)a b", Regexp::EXTENDED).match?("ab")
end

assert("Regexp - comment groups (?#...)") do
Expand Down
Loading