Skip to content

mruby-regexp: share the pattern-skipping rules between the two prescans #7165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 1 commit into from
Aug 14, 2026
Merged

mruby-regexp: share the pattern-skipping rules between the two prescans #7165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 69 additions & 57 deletions mrbgems/mruby-regexp/src/re_compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1630,7 +1630,7 @@ has_comment_group(const char *src, mrb_int len)
Returns the position just past its closing "]", or NULL if it is not one.
compile_charclass() consumes such a bracket as a unit, so its ']' does not
end the class; a malformed one falls through and the '[' is an ordinary
member. Both scans below have to agree with the parser on this. */
member. The scan below has to agree with the parser on this. */
static const char*
skip_posix_bracket(const char *src, const char *end)
{
Expand All @@ -1645,13 +1645,70 @@ skip_posix_bracket(const char *src, const char *end)
return NULL;
}

/*
* Step over the one construct at `src` that a pattern scan must not read
* into: an escape sequence, or a character class from its '[' through its
* ']'. Returns the position just past it, having updated *in_class, or NULL
* when the byte at `src` is neither and the caller has to handle it itself.
* A class spans several calls, with *in_class carrying the state between
* them, so the caller keeps one flag and starts it FALSE.
*
* preprocess_pattern() and has_named_group() both walk the pattern hunting
* for a "(?" opener, and both have to agree with the parser on when a '(' is
* an opener rather than an escaped or bracketed byte. The rules for that live
* here alone, so a correction to them cannot be made in one walk and missed
* in the other.
*/
static const char*
skip_uninterpreted(const char *src, const char *end, mrb_bool *in_class)
{
char ch = *src;

if (ch == '\\' && src + 1 < end) {
mrb_bool unicode = (src[1] == 'u');
src += 2;
/* A `\u{...}` list is a single escape rather than `\u` followed by a
brace group: it separates its codepoints with spaces, which the
free-spacing pass would otherwise remove, joining `\u{61 62}` into the
one codepoint `\u{6162}`. An unterminated list runs to the end. */
if (unicode && src < end && *src == '{') {
while (src < end) {
if (*src++ == '}') break;
}
}
return src;
}

if (*in_class) {
/* A POSIX bracket is consumed as a unit by compile_charclass(), so the
']' that closes it does not close the class. */
const char *q = skip_posix_bracket(src, end);
if (q) return q;
if (ch == ']') *in_class = FALSE;
return src + 1;
}

if (ch == '[') {
*in_class = TRUE;
src++;
/* A ']' written first is a literal member, optionally after '^',
mirroring the `first` flag in compile_charclass(). */
if (src < end && *src == '^') src++;
if (src < end && *src == ']') src++;
return src;
}

return NULL;
}

/*
* Rewrite the pattern before the parser sees it.
* Removes (?#...) comment groups always, and in extended mode (/x) also
* whitespace and #comments.
* Whitespace inside [...] character classes is preserved, and so is a (?#
* written there, which is a literal member rather than a comment group.
* Escaped characters (\ followed by anything) are preserved.
* skip_uninterpreted() decides which bytes those are.
*/
static char*
preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
Expand All @@ -1664,40 +1721,12 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,

while (src < end) {
char ch = *src;
if (ch == '\\' && src + 1 < end) {
mrb_bool unicode = (src[1] == 'u');
buf[o++] = *src++;
buf[o++] = *src++;
/* A `\u{...}` list separates its codepoints with spaces, so the brace
group has to be copied whole: the free-spacing pass below would
otherwise join `\u{61 62}` into the single codepoint `\u{6162}`. */
if (unicode && src < end && *src == '{') {
while (src < end) {
char u = *src;
buf[o++] = *src++;
if (u == '}') break;
}
}
continue;
}
if (in_class) {
/* Copy a POSIX bracket whole and keep the class open. */
const char *q = skip_posix_bracket(src, end);
if (q) {
while (src < q) buf[o++] = *src++;
continue;
}
if (ch == ']') in_class = FALSE;
buf[o++] = *src++;
continue;
}
if (ch == '[') {
in_class = TRUE;
buf[o++] = *src++;
/* A ']' written first is a literal member, optionally after '^',
mirroring the `first` flag in compile_charclass(). */
if (src < end && *src == '^') buf[o++] = *src++;
if (src < end && *src == ']') buf[o++] = *src++;
/* An escape or a character class is copied through untouched: neither
holds a comment group, and inside a class the free-spacing rules do
not apply. */
const char *skip = skip_uninterpreted(src, end, &in_class);
if (skip) {
while (src < skip) buf[o++] = *src++;
continue;
}
if (ch == '(' && end - src >= 3 && src[1] == '?' && src[2] == '#') {
Expand Down Expand Up @@ -1748,9 +1777,8 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
* (?<name>...) is the only spelling of a definition this gem accepts; the
* (?'name'...) form raises "undefined (?...) sequence", so the scan looks for
* "(?<" alone. It excludes (?<= and (?<!, which are lookbehind rather than a
* definition, and it skips escape pairs and character classes so that /\(?/
* and /[(?<]/ are not false positives, with a POSIX bracket and a leading
* literal ']' not ending a class, as in preprocess_pattern() above.
* definition, and it steps over escapes and character classes with
* skip_uninterpreted(), so that /\(?/ and /[(?<]/ are not false positives.
*
* A truncated "(?<" at the end of the pattern is counted as a named group,
* which is harmless: the parser reaches the same bytes and raises there.
Expand All @@ -1763,25 +1791,9 @@ has_named_group(const char *src, mrb_int len)

while (src < end) {
char ch = *src;
if (ch == '\\' && src + 1 < end) {
src += 2;
continue;
}
if (in_class) {
const char *q = skip_posix_bracket(src, end);
if (q) {
src = q;
continue;
}
if (ch == ']') in_class = FALSE;
src++;
continue;
}
if (ch == '[') {
in_class = TRUE;
src++;
if (src < end && *src == '^') src++;
if (src < end && *src == ']') src++;
const char *skip = skip_uninterpreted(src, end, &in_class);
if (skip) {
src = skip;
continue;
}
if (ch == '(' && end - src >= 3 && src[1] == '?' && src[2] == '<') {
Expand Down
72 changes: 72 additions & 0 deletions mrbgems/mruby-regexp/test/regexp_syntax.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,78 @@
end
end

assert("Regexp - the /x pass and the named-group scan skip the same constructs") do
# Two walks read the pattern before the parser does: the /x free-spacing
# pass and the named-group pre-scan. Both have to step over the same
# escapes, character classes and POSIX brackets, so each row below is read
# by both at once. A rule lost from the free-spacing pass strips a space it
# should have kept; the same rule lost from the pre-scan turns a bracketed
# "(?<" into a phantom named group, which demotes the plain (b) that
# follows and shortens the match. Either way the row fails.
x = Regexp::EXTENDED

# an escape pair hides the '(' from both
assert_equal ["(<a>b", "b"],
Regexp.new('\(?<a> (b)', x).match("(<a>b").to_a

# a character class hides "(?<" and keeps its own spaces
assert_equal ["(?< b", "b"],
Regexp.new('[(?< a>]+ (b)', x).match("(?< b").to_a

# a ']' written first is a member, so the class runs past it
assert_equal ["] (?<b", "b"],
Regexp.new('[] (?<a>]+(b)', x).match("] (?<b").to_a
assert_equal ["zzb", "b"],
Regexp.new('[^] (?<a>]+(b)', x).match("zzb").to_a

# a POSIX bracket's ']' does not close the class either
assert_equal ["a (?<b", "b"],
Regexp.new('[[:alpha:] (?<a>]+(b)', x).match("a (?<b").to_a

# a `\u{...}` list is one escape, so its separating space survives /x and
# its bytes are not read as pattern syntax
assert_equal ["ab"], Regexp.new('\u{61 62}', x).match("ab").to_a
assert_equal ["abcd", "c", "d"],
Regexp.new('\u{61 62}(c)(d)', x).match("abcd").to_a
assert_equal ["abcd", "c"],
Regexp.new('\u{61 62}(?<n>c)(d)', x).match("abcd").to_a
assert_equal ["abcd", "c", "d"],
Regexp.new('[\u{61 62}]+(c)(d)', x).match("abcd").to_a

# With no whitespace, no #comment and no (?#...) to remove, /x rewrites
# nothing, so both compiles must agree; they do not take the same road,
# though: without /x the pre-scan reads the pattern as written, while with
# /x it reads what the free-spacing pass emitted. The two walks disagreeing
# about where a class or an escape ends is exactly what shows up here.
[
['\(?<a>(b)', "(<a>b"],
['[(?<a>]+(b)', "(?<b"],
['[](?<a>]+(b)', "](?<b"],
['[^](?<a>]+(b)', "zzb"],
['[[:alpha:](?<a>]+(b)', "a(?<b"],
['[\]](?<a>x)(y)', "]xy"],
['\\\\(?<a>x)(y)', "\\xy"],
['\u{61}(?<n>b)(c)', "abc"],
['[\u{61}]+(?<n>b)(c)', "abc"],
['(a)(?<b>b)', "ab"],
].each do |pat, subject|
assert_equal Regexp.new(pat).match(subject).to_a,
Regexp.new(pat, x).match(subject).to_a
end

# A `\u{...}` list is not a place a named group can be declared, so the
# scan must not read "(?<" out of one. The list here is malformed either
# way and the pattern is rejected either way, but which error comes first
# depends on the scan: taking the "(?<" for a declaration turns on the
# demotion that rejects the leading \1 before the parser ever reaches the
# bad list. CRuby reports the list, and so does the scan that treats
# `\u{...}` as one escape.
assert_raise_with_message(RegexpError,
"invalid Unicode list: /\\1\\u{(?<a>/") do
Regexp.new("\\1\\u{(?<a>")
end
end

assert("Regexp - case in when") do
result = case "hello123"
when /\d+/ then "has digits"
Expand Down
Loading