Skip to content

mruby-regexp: let a byte that starts no character begin a match #7059

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 3 commits into from
Aug 10, 2026
Merged

mruby-regexp: let a byte that starts no character begin a match #7059

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da71232
mruby-regexp: let a byte that starts no character begin a match
takumin Aug 9, 2026
608ec7d
mruby-regexp: avoid forming a pointer before the string start
takumin Aug 9, 2026
9bde072
mruby-regexp: guard the seeding, not the iteration, inside a character
takumin Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions mrbgems/mruby-regexp/include/re_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,20 @@ mrb_re_decode_char(const char *s, const char *end, int *len, mrb_bool binary)
return mrb_re_utf8_decode(s, end, len);
}

/* TRUE when s points into the middle of a character that starts earlier in
the string, so it is not a place a match may start at. A byte that looks
like a continuation byte but follows no lead byte that reaches it belongs
to no character and stands on its own. */
static inline mrb_bool
mrb_re_utf8_continuation_p(const char *s)
mrb_re_utf8_interior_p(const char *str, const char *s, const char *end)
{
return (((uint8_t)*s & 0xC0) == 0x80);
if (((uint8_t)*s & 0xC0) != 0x80) return FALSE;
for (int back = 1; back <= 3 && back <= s - str; back++) {
const char *lead = s - back;
if (((uint8_t)*lead & 0xC0) == 0x80) continue; /* another continuation byte */
return mrb_re_utf8_charlen(lead, end) > back;
}
return FALSE;
}

/* Execute a match.
Expand Down
37 changes: 22 additions & 15 deletions mrbgems/mruby-regexp/src/re_exec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,19 +326,22 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
if (sp > str_end) break;
}
}
/* Don't seed a new match attempt at a UTF-8 continuation byte --
a multi-byte char's interior is not a valid char boundary, and
starting a thread there mis-decodes the char (e.g. a class-
match on a stray 0x82 instead of the leader's full codepoint). */
if (!s.binary && curr.count == 0 && sp < str_end && mrb_re_utf8_continuation_p(sp)) {
continue;
/* Don't seed a new match attempt inside a character. Its interior is
not a char boundary, and starting a thread there mis-decodes the
char (e.g. a class match on a stray 0x82 instead of the leader's
full codepoint). A byte that no lead byte reaches belongs to no
character and is a boundary of its own.
Threads seeded earlier are still stepped at this position, so the
test guards the seeding alone and never skips the iteration. */
if (s.binary || sp >= str_end ||
!mrb_re_utf8_interior_p(str, sp, str_end)) {
int slot = match_only ? 0 : pool_alloc(&s);
if (!match_only) memset(CAP(&s, slot), -1, sizeof(int) * ncap);
s.gen++;
s.cut = FALSE;
add_thread(&s, &curr, 0, slot, sp);
if (s.matched && curr.count == 0) break;
}
int slot = match_only ? 0 : pool_alloc(&s);
if (!match_only) memset(CAP(&s, slot), -1, sizeof(int) * ncap);
s.gen++;
s.cut = FALSE;
add_thread(&s, &curr, 0, slot, sp);
if (s.matched && curr.count == 0) break;
}

if (sp >= str_end) break;
Expand Down Expand Up @@ -674,7 +677,7 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
while (sp < str_end && !FIRST_BYTE_OK(pat, (uint8_t)*sp)) sp++;
if (sp > str_end) break;
}
if (!binary && sp < str_end && mrb_re_utf8_continuation_p(sp)) {
if (!binary && sp < str_end && mrb_re_utf8_interior_p(str, sp, str_end)) {
continue;
}
memset(caps, -1, sizeof(int) * ncap);
Expand All @@ -697,7 +700,7 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
static int
literal_exec(const mrb_regexp_pattern *pat,
const char *str, mrb_int len, mrb_int start,
int *captures, int captures_size)
int *captures, int captures_size, mrb_bool binary)
{
const char *sp = str + start;
const char *str_end = str + len;
Expand All @@ -706,6 +709,10 @@ literal_exec(const mrb_regexp_pattern *pat,
while (sp + plen <= str_end) {
const char *found = (const char*)memchr(sp, pat->prefix[0], str_end - sp);
if (!found || found + plen > str_end) return 0;
if (!binary && mrb_re_utf8_interior_p(str, found, str_end)) {
sp = found + 1; /* not a char boundary, same rule as the other engines */
continue;
}
if (plen == 1 || memcmp(found + 1, pat->prefix + 1, plen - 1) == 0) {
/* match found */
if (captures && captures_size >= 2) {
Expand All @@ -726,7 +733,7 @@ mrb_re_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
int *captures, int captures_size, mrb_bool binary)
{
if (pat->is_literal) {
return literal_exec(pat, str, len, start, captures, captures_size);
return literal_exec(pat, str, len, start, captures, captures_size, binary);
}
if (pat->has_backref || pat->needs_backtrack) {
return backtrack_exec(mrb, pat, str, len, start, captures, captures_size, binary);
Expand Down
38 changes: 38 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,44 @@
assert_equal 2, "Ā".match(/./)[0].bytesize
end

assert("Regexp - a byte that belongs to no character is a match position") do
# A byte in 0x80-0xBF is the interior of a character only while a lead byte
# in front of it reaches that far. One that stands on its own is a boundary
# like any other, and the engines used to disagree about it: the literal
# fast path matched there, the NFA never started a match there.
b = "\x81"
assert_equal 0, (b + b).match(Regexp.new(b + b)).begin(0)
assert_equal 2, (b + b).match(Regexp.new(b + "+"))[0].bytesize
assert_equal 2, (b + b).match(Regexp.new(b + "*"))[0].bytesize
assert_equal 1, (b + b).match(Regexp.new(b + "?"))[0].bytesize
assert_equal 1, ("x" + b + b).match(Regexp.new(b + "+")).begin(0)
# Inside a character there is still no match position.
assert_nil "あ".match(Regexp.new("\x81"))
assert_nil "あ".match(Regexp.new("\x82"))
assert_nil "\u{1D54F}".match(Regexp.new("\x95"))
# Next to one there is.
assert_equal 0, (b + "あ").match(Regexp.new(b)).begin(0)
# Through pre_match, since #begin counts characters where the build has
# them and bytes where it does not.
assert_equal 3, ("あ" + b).match(Regexp.new(b)).pre_match.bytesize
end

assert("Regexp - an attempt in flight opens no match position inside a character") do
# "ĵ" is C4 B5 and "µ" is C2 B5, so the two share their trailing byte. That
# byte is the interior of "ĵ" and no match may start there, but the test
# for it only ran while nothing was in flight. The branch of `.?` that
# consumes the character parks a thread past it, and the attempt seeded at
# the shared byte then matched it on its own, cutting "ĵ" in half.
assert_nil "ĵ".match(/.?[µ]/)
assert_nil "ĵ".gsub(/.?[µ]/, "!").match(/!/)
assert_nil ("あ" + "ĵ").match(/.?[µ]/)
# A character the class does hold is still found through the same branch.
assert_equal 4, ("ĵ" + "µ").match(/.?[µ]/)[0].bytesize
assert_equal 5, ("あ" + "µ").match(/.?[µ]/)[0].bytesize
# And so is the byte itself where no lead byte reaches it.
assert_equal 2, ("x" + "\xb5").match(/.?[µ]/)[0].bytesize
end

assert("Regexp - multibyte (UTF-8) match extraction") do
# Capture offsets are recorded in bytes; substring extraction must honor
# them as byte ranges so multibyte matches are not corrupted.
Expand Down
Loading