Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions mrbgems/mruby-regexp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ simulation) with backtracking fallback.
- `(?<name>...)` named capture group
- `|` alternation
- `\1`-`\9` backreferences
- `\k<name>`, `\k'name'` named backreferences
- `(?=...)` positive lookahead
- `(?!...)` negative lookahead
- `(?<=...)` positive lookbehind (fixed-length only)
Expand Down Expand Up @@ -132,6 +133,29 @@ pattern analysis.
- **Step limit on backtracking**: Patterns that require the
backtracking engine are subject to a step limit.

## Named Captures

As in CRuby, declaring a named group anywhere in a pattern changes how the
whole pattern is numbered: a plain `(...)` groups without capturing, and a
numbered backreference is a `RegexpError` in every spelling (`\1`, `\k<1>`,
`\k<-1>`). Refer to a group by name instead.

```ruby
md = /(?<a>a)(b)/.match("ab")
md.size # => 2
md.captures # => ["a"]
md[:a] # => "a"
md[2] # => nil

"aa".match(/(?<n>\w)\k<n>/)[0] # => "aa"

Regexp.new("(a)(?<b>b)\\1")
# RegexpError: numbered backref/call is not allowed. (use name)
```

A pattern with no named group numbers its groups as usual, and `\1`-`\9` work
there.

## Configuration

```c
Expand Down
115 changes: 103 additions & 12 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ typedef struct {
uint16_t num_named;
mrb_bool has_backref;
mrb_bool needs_backtrack;
mrb_bool dont_capture; /* pattern declares a named group: plain (...) does not capture */
char *stripped; /* allocated buffer for pattern preprocessing */
} re_compiler;

Expand Down Expand Up @@ -736,6 +737,12 @@ compile_atom(re_compiler *c)
}
}

/* Onigmo's ONIG_OPTION_DONT_CAPTURE_GROUP, which CRuby turns on for a
pattern that declares a named group: a plain (...) then groups
without capturing, so the numbered side counts only the named
groups. The named group itself keeps its number. */
if (c->dont_capture && cap_name == NULL) capturing = FALSE;

uint16_t group = 0;
if (capturing) {
if (c->num_captures >= RE_MAX_CAPTURES) {
Expand Down Expand Up @@ -790,6 +797,9 @@ compile_atom(re_compiler *c)
next_char(c);
ch = peek(c);
if (ch >= '1' && ch <= '9') {
if (c->dont_capture) {
compile_error(c, "numbered backref/call is not allowed. (use name)");
}
next_char(c);
emit(c, RE_BACKREF, (uint8_t)(ch - '0'), (c->flags & RE_FLAG_IGNORECASE) ? 1 : 0);
c->has_backref = TRUE;
Expand Down Expand Up @@ -851,6 +861,14 @@ compile_atom(re_compiler *c)

int group = -1;
if (name_len > 0 && (name[0] == '-' || (name[0] >= '0' && name[0] <= '9'))) {
/* CRuby rejects a numbered backreference in a named pattern whatever
its spelling, and it has to be rejected here too: once plain groups
stop consuming numbers, both the absolute bound and the relative
form's `num_captures - n` below would silently resolve to a
different group instead of erroring. */
if (c->dont_capture) {
compile_error(c, "numbered backref/call is not allowed. (use name)");
}
mrb_bool relative = (name[0] == '-');
int n = 0;
for (uint32_t i = (relative ? 1 : 0); i < name_len; i++) {
Expand Down Expand Up @@ -1176,6 +1194,25 @@ has_comment_group(const char *src, mrb_int len)
return FALSE;
}

/* Inside a character class, is `src` the start of a POSIX bracket [:name:]?
Returns the position just past its closing "]", or NULL if it is not one.
compile_charclass() consumes such a bracket as a unit, so its ']' does not
end the class; a malformed one falls through and the '[' is an ordinary
member. Both scans below have to agree with the parser on this. */
static const char*
skip_posix_bracket(const char *src, const char *end)
{
if (!(*src == '[' && src + 1 < end && src[1] == ':')) return NULL;
const char *q = src + 2;
while (q < end && *q != ':' && *q != ']') q++;
/* Compare the distance rather than q + 1: the loop above stops with
q == end for a bracket the pattern truncates, as in /[[:alpha/, and
forming q + 1 from a one-past-the-end pointer is undefined even where
the && never reads through it. */
if (end - q >= 2 && q[0] == ':' && q[1] == ']') return q + 2;
return NULL;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

/*
* Rewrite the pattern before the parser sees it.
* Removes (?#...) comment groups always, and in extended mode (/x) also
Expand All @@ -1201,18 +1238,11 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
continue;
}
if (in_class) {
/* compile_charclass() consumes a POSIX bracket as a unit, so the ']'
of [:name:] does not end the class. Copy it whole and keep the
class open; a malformed bracket falls through and the '[' is
copied as an ordinary member, which is what the parser does too. */
if (ch == '[' && src + 1 < end && src[1] == ':') {
const char *q = src + 2;
while (q < end && *q != ':' && *q != ']') q++;
if (q + 1 < end && *q == ':' && q[1] == ']') {
q += 2;
while (src < q) buf[o++] = *src++;
continue;
}
/* Copy a POSIX bracket whole and keep the class open. */
const char *q = skip_posix_bracket(src, end);
if (q) {
while (src < q) buf[o++] = *src++;
continue;
}
if (ch == ']') in_class = FALSE;
buf[o++] = *src++;
Expand Down Expand Up @@ -1267,6 +1297,63 @@ preprocess_pattern(mrb_state *mrb, const char *src, mrb_int len,
return buf;
}

/*
* Does the pattern declare a named group anywhere? Answering this before the
* parser starts is what lets compile_atom() demote a plain (...) that comes
* before the named group that causes the demotion.
*
* (?<name>...) is the only spelling of a definition this gem accepts; the
* (?'name'...) form raises "undefined (?...) sequence", so the scan looks for
* "(?<" alone. It excludes (?<= and (?<!, which are lookbehind rather than a
* definition, and it skips escape pairs and character classes so that /\(?/
* and /[(?<]/ are not false positives, with a POSIX bracket and a leading
* literal ']' not ending a class, as in preprocess_pattern() above.
*
* A truncated "(?<" at the end of the pattern is counted as a named group,
* which is harmless: the parser reaches the same bytes and raises there.
*/
static mrb_bool
has_named_group(const char *src, mrb_int len)
{
const char *end = src + len;
mrb_bool in_class = FALSE;

while (src < end) {
char ch = *src;
if (ch == '\\' && src + 1 < end) {
src += 2;
continue;
}
if (in_class) {
const char *q = skip_posix_bracket(src, end);
if (q) {
src = q;
continue;
}
if (ch == ']') in_class = FALSE;
src++;
continue;
}
if (ch == '[') {
in_class = TRUE;
src++;
if (src < end && *src == '^') src++;
if (src < end && *src == ']') src++;
continue;
}
if (ch == '(' && end - src >= 3 && src[1] == '?' && src[2] == '<') {
/* src + 3 is at most end here, since the test above leaves three bytes
to read, so the one-past-the-end pointer it can form is a position C
allows. */
if (src + 3 >= end || (src[3] != '=' && src[3] != '!')) return TRUE;
src += 3;
continue;
}
src++;
}
return FALSE;
}

/*
* Compute the set of bytes that could be the first consumed byte of a match.
* Walks bytecode from pc=0, following epsilon transitions (SAVE, JMP, SPLIT).
Expand Down Expand Up @@ -1373,6 +1460,10 @@ mrb_re_compile(mrb_state *mrb, const char *pattern, mrb_int len, uint32_t flags)
c.p = pattern;
c.flags = flags;
c.num_captures = 1; /* group 0 = whole match */
/* Scan the same bytes the parser is about to read: preprocess_pattern() has
already taken out the /x free-spacing, the #comments and the (?#...)
groups. */
c.dont_capture = has_named_group(pattern, len);

/* group 0 start */
emit(&c, RE_SAVE, 0, 0);
Expand Down
76 changes: 76 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,15 @@

assert_equal " 1 ", Regexp.new('[[:digit:] ]+', Regexp::EXTENDED).match(" 1 ")[0]

# a bracket the pattern truncates leaves the scan with nothing after the
# name, and the class is still the parser's error to report
assert_raise_with_message(RegexpError, "unterminated character class: /[[:alpha/") do
Regexp.new("[[:alpha", Regexp::EXTENDED)
end
assert_raise_with_message(RegexpError, "unterminated character class: /[[:alpha:/") do
Regexp.new("[[:alpha:", Regexp::EXTENDED)
end

# a ']' written first in a class is a literal member, so the class is
# still open after it
re = Regexp.new('[] ]', Regexp::EXTENDED)
Expand Down Expand Up @@ -1282,6 +1291,59 @@ class StringMatchHelperOverride < String
assert_nil md[2]
end

assert("Regexp - a named group makes plain groups non-capturing") do
# Onigmo's ONIG_OPTION_DONT_CAPTURE_GROUP, which CRuby turns on once the
# pattern declares a named group: (...) then groups without capturing.
md = /(?<a>a)(b)/.match("ab")
assert_equal 2, md.size
assert_equal ["ab", "a"], md.to_a
assert_equal ["a"], md.captures
assert_nil md[2]
assert_raise_with_message(IndexError, "index 2 out of matches") { md.begin(2) }
assert_equal "a", md[:a]

# a plain group written before the named group is demoted just the same,
# which is what the pre-scan buys: the parser reaches it before it has seen
# the declaration that decides the question
md = /(a)(?<b>b)/.match("ab")
assert_equal 2, md.size
assert_equal ["ab", "b"], md.to_a
assert_equal ["b"], md.captures
assert_equal "b", md[1]
assert_equal "b", md[:b]

# the shrunken count is what $2, $+ and a \2 in a replacement read
"ab" =~ /(?<a>a)(b)/
assert_nil $2
assert_equal "a", $+
assert_equal "[]", "ab".sub(/(?<a>a)(b)/, '[\2]')

Comment thread
coderabbitai[bot] marked this conversation as resolved.
# (?<= and (?<! open a lookbehind, not a named group, so they demote nothing
assert_equal ["b", "b"], /(?<=a)(b)/.match("ab").to_a
assert_equal ["b", "b"], /(?<!x)(b)/.match("ab").to_a
# nor does a "(?<" that is escaped or sits inside a character class
assert_equal ["(<a>b", "b"], /\(?<a>(b)/.match("(<a>b").to_a
assert_equal ["(?<b", "b"], /[(?<a>]+(b)/.match("(?<b").to_a
assert_equal ["a(?<b", "b"], /[[:alpha:](?<]+(b)/.match("a(?<b").to_a
# nor one inside a (?#...) comment group, which is gone before the scan runs
assert_equal ["b", "b"], /(?# (?<a>x )(b)/.match("b").to_a

# in /x mode the scan reads the pattern after free-spacing and comments go
assert_equal ["xy", "x"], /(?<a>x) # (b)
(y)/x.match("xy").to_a
assert_equal ["y", "y"], /# (?<a>x)
(y)/x.match("y").to_a

# a truncated "(?<" is still the parser's error, not a silent named group
assert_raise(RegexpError) { Regexp.new("(?<") }

# the scan runs on every pattern, so a truncated POSIX bracket reaches
# skip_posix_bracket() without /x too, and is still the parser's error
assert_raise_with_message(RegexpError, "unterminated character class: /[[:alpha/") do
Regexp.new("[[:alpha")
end
end

assert("String#sub with block") do
assert_equal "HELLO world", "hello world".sub(/\w+/) { |m| m.upcase }
end
Expand Down Expand Up @@ -1608,6 +1670,20 @@ def -(other)
assert_nil "ab".match(/(?<n>a)\k<n>/i)
# an unknown name is an error
assert_raise(RegexpError) { Regexp.new("\\k<missing>") }

# once the pattern has a named group a numbered backreference is rejected,
# whatever its spelling, because there is no longer a number to reach
msg = "numbered backref/call is not allowed. (use name)"
assert_raise_with_message(RegexpError, "#{msg}: /(a)(?<b>b)\\1/") do
Regexp.new("(a)(?<b>b)\\1")
end
assert_raise_with_message(RegexpError, "#{msg}: /(a)(?<b>b)\\k<1>/") do
Regexp.new("(a)(?<b>b)\\k<1>")
end
assert_raise_with_message(RegexpError, "#{msg}: /(a)(?<b>b)\\k<-1>/") do
Regexp.new("(a)(?<b>b)\\k<-1>")
end
assert_raise(RegexpError) { Regexp.new("(?<b>b)\\k'1'") }
end

assert("Regexp - numeric \\k backreference out of int range") do
Expand Down
Loading