Skip to content

mruby-regexp: undo what a lookaround captured when the match backtracks past it #7273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 2 commits into from
Aug 19, 2026
+216 −51
Merged

mruby-regexp: undo what a lookaround captured when the match backtracks past it #7273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1c0a5d2
mruby-regexp: undo what a lookaround captured when the match backtrac…
takumin Aug 19, 2026
90c7f08
Merge branch 'master' into pr7273
matz Aug 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions mrbgems/mruby-regexp/include/re_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ enum re_opcode {
backtracking into it. */
RE_ATOMIC_END, /* end of an atomic group's body: offset = the number of
the RE_ATOMIC it closes */
RE_LOOK_END, /* end of a lookaround's sub-pattern: offset = the
lookaround's number, from the same count as RE_ATOMIC;
a = 1 for a negative one. The instruction after it is
the text after the lookaround, which is where the
opener's `offset` points, so the opener finds its end
at offset - 1. */
};

/* Bytecode instruction (4 bytes each for alignment) */
Expand Down
28 changes: 20 additions & 8 deletions mrbgems/mruby-regexp/src/re_compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,10 @@ typedef struct {
uint16_t num_groups; /* groups opened so far, counting the plain ones a
named pattern demotes: what decides whether
`\NN` is a backreference or an octal escape */
uint32_t num_cuts; /* atomic groups numbered so far, the possessive
repeats among them: each takes the next number,
so no two that nest share one; see RE_ATOMIC */
uint32_t num_cuts; /* atomic groups and lookarounds numbered so far,
the possessive repeats among them: each takes
the next number, so no two that nest share one;
see RE_ATOMIC and RE_LOOK_END */
uint32_t atom_start; /* where the atom a quantifier binds to begins;
compile_quantified sets it to the position
before the atom, and a `\u{...}` list moves it
Expand Down Expand Up @@ -1154,7 +1155,7 @@ compute_fixed_len(re_compiler *c, uint32_t start, uint32_t end, int *chars_out)
/* For simplicity, reject alternation in lookbehind */
return -1;
}
case RE_MATCH:
case RE_LOOK_END:
*chars_out = chars;
return len;
default:
Expand Down Expand Up @@ -1335,6 +1336,19 @@ emit_codepoint(re_compiler *c, uint32_t cp)
number/name` for one within it that names no group. */
#define RE_MAX_BACKREF_NUM 2147483647

/* Compile the sub-pattern of a lookaround, whose opener has just been
emitted, up to and including the RE_LOOK_END that closes it. The end
carries the lookaround's number, from the count the atomic groups use, so
that a cut aimed at this lookaround is told from one aimed at a group
around or inside it; see bt_match(). */
static void
compile_look_body(re_compiler *c, mrb_bool negative)
{
uint16_t cut = (uint16_t)++c->num_cuts;
compile_alt(c);
emit(c, RE_LOOK_END, negative, cut);
}

/* Compile a single atom (character, class, group, etc.). Returns whether one
was read: FALSE when what stands at the parse point is not an atom, either a
quantifier metacharacter or the end of the sequence, neither of them
Expand Down Expand Up @@ -1369,8 +1383,7 @@ compile_atom(re_compiler *c)
mrb_bool negative = (c->p[1] == '!');
next_char(c); next_char(c); /* skip ?= or ?! */
uint32_t la_pos = emit(c, negative ? RE_NEG_LOOKAHEAD : RE_LOOKAHEAD, 0, 0);
compile_alt(c);
emit(c, RE_MATCH, 0, 0); /* end of lookahead sub-pattern */
compile_look_body(c, negative);
c->pat->code[la_pos].offset = (uint16_t)c->code_len; /* patch: skip past sub-pattern */
if (peek(c) != ')') compile_error(c, "unmatched '('");
next_char(c);
Expand All @@ -1385,8 +1398,7 @@ compile_atom(re_compiler *c)
uint32_t lb_pos = emit(c, negative ? RE_NEG_LOOKBEHIND : RE_LOOKBEHIND, 0, 0);
emit(c, RE_LB_WIDTH, 0, 0);
uint32_t sub_start = c->code_len;
compile_alt(c);
emit(c, RE_MATCH, 0, 0);
compile_look_body(c, negative);
c->pat->code[lb_pos].offset = (uint16_t)c->code_len;

/* measure the sub-pattern for both rewind units */
Expand Down
176 changes: 133 additions & 43 deletions mrbgems/mruby-regexp/src/re_exec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -632,8 +632,10 @@ lookbehind_start(const mrb_regexp_pattern *pat, const char *str,
group's body may be tried for it, so the frames between that end and the
RE_ATOMIC that opened the group hand BT_CUT of the group's number up
unchanged, undoing their captures as they go, and the frame that ran that
RE_ATOMIC turns it into BT_FAIL. A cut never reaches a lookaround from
inside its sub-pattern: the RE_ATOMIC that absorbs it is in there too.
RE_ATOMIC turns it into BT_FAIL. A lookaround runs its sub-pattern the
same way, the text after it going on inside the sub-pattern's frames from
the RE_LOOK_END (see there), so a cut can pass through one; the opener
absorbs its own number like an RE_ATOMIC and hands any other up.
The fourth answer, BT_LIMIT, is a frame giving up at the recursion or step
limit. A frame that gets it hands it up; a SPLIT takes it as that branch
failing and answers with its other branch, as it would with a failure.
Expand All @@ -658,20 +660,46 @@ typedef struct {
int *captures;
int ncap;
int steps;
/* Per pc, the offset the running iteration of the loop that pc keys began
at, or -1 while none is running. A repetition whose body can match empty
has to stop once an iteration ends where it began, or it would go round
at the same position until a limit refused it and answer with whatever
the alternatives left inside the limit produce. Onigmo stops it with a
null check around the body; this array is that check's memory. The pc
that keys a loop is its marked head for e* (the SPLIT/SPLITNG whose
offset is the exit; the JMP closing the body reads the record) and its
marked back edge for e+ (the SPLIT/SPLITNG at the end of the body, which
both writes and reads it); see mark_empty_loops(). */
int *iter_at;
/* Per pc, the offset the frame that pc keys was entered at, or -1 while
none is running; what a record means is what its pc is. For the edge of
a repetition whose body can match empty it is where the running
iteration began: such a repetition has to stop once an iteration ends
where it began, or it would go round at the same position until a limit
refused it and answer with whatever the alternatives left inside the
limit produce. Onigmo stops it with a null check around the body; this
array is that check's memory. The pc that keys a loop is its marked head
for e* (the SPLIT/SPLITNG whose offset is the exit; the JMP closing the
body reads the record) and its marked back edge for e+ (the
SPLIT/SPLITNG at the end of the body, which both writes and reads it);
see mark_empty_loops(). For the RE_LOOK_END of a lookaround it is where
the lookaround was entered, which is where the text after it goes on
from; see bt_look(). */
int *entered_at;
/* Per pc, the pass that wrote entered_at[pc]. A pass is one run of a
lookaround's sub-pattern, told by the depth of the bt_look() frame
running it, and 0 is the pattern outside every lookaround; `pass` is the
one the frames now running are in. A record is read as the running
iteration's only by the pass that wrote it: the text after a positive
lookaround runs inside its sub-pattern's frames (see RE_LOOK_END), so a
repetition around the lookaround re-enters the sub-pattern while the
records of the loops inside it from the pass before are still live, and
the first iteration of an e+, which reads its record without having
written it, would take one of those for its own where the positions
coincide: `(?=(b|)+)+` on "b" re-enters at 0 while the pass before left
1 for `(b|)+`, and its first iteration, ending at 1, would stop there
with "b" captured, where a fresh pass goes round once more and leaves
"". For the RE_LOOK_END the record is the pass the lookaround was entered
from, which the text after it runs in. */
int *entered_in;
int pass;
mrb_bool binary;
} bt_state;

/* Whether the loop `key` keys is at the end of an iteration that began at
sp: the record is this pass's and names sp. */
#define ITER_EMPTY(m, key, sp) \
((m)->entered_at[key] == (int)((sp) - (m)->str) && (m)->entered_in[key] == (m)->pass)

static int bt_match(bt_state *m, const char *sp, uint32_t pc, int depth);

/* Run the frame at pc as the start of an iteration of the loop `key` keys,
Expand All @@ -684,13 +712,54 @@ static int bt_match(bt_state *m, const char *sp, uint32_t pc, int depth);
static int
bt_iter(bt_state *m, const char *sp, uint32_t pc, uint32_t key, int depth)
{
int old = m->iter_at[key];
m->iter_at[key] = (int)(sp - m->str);
int old_at = m->entered_at[key], old_in = m->entered_in[key];
m->entered_at[key] = (int)(sp - m->str);
m->entered_in[key] = m->pass;
int r = bt_match(m, sp, pc, depth);
m->iter_at[key] = old;
m->entered_at[key] = old_at;
m->entered_in[key] = old_in;
return r;
}

/* Run the sub-pattern of the lookaround whose opener is at pc: it begins at
`body` and matches from `from`, which is sp for a lookahead and the
rewound start for a lookbehind. The record of sp, and of the pass the
lookaround is entered from, lasts as long as the frame, as an iteration's
does in bt_iter(); the RE_LOOK_END closing the sub-pattern reads it (see
there). The sub-pattern runs as a pass of its own, this frame's depth,
which no pass still live has, so the records of the loops inside it that
another run of the same sub-pattern may have left live are not taken for
this run's (see bt_state).

The answer is the sub-pattern's, and the cut of this lookaround's number
is what the sub-pattern matching comes back as: for a negative lookaround
from the RE_LOOK_END itself, and the answer is BT_MATCH, with every
capture the sub-pattern wrote undone on the way up; for a positive one
from the text after the lookaround failing, which the RE_LOOK_END ran
inside the sub-pattern's frames, so the sub-pattern matched once and that
was its only match: BT_FAIL, the captures undone the same way. BT_MATCH
from a positive one is the whole pattern having matched through that
text. Everything else, a failure, a limit or another group's cut, goes up
as it is. */
static int
bt_look(bt_state *m, const char *sp, const char *from, uint32_t pc,
uint32_t body, int depth)
{
uint32_t end = m->pat->code[pc].offset - 1;
re_inst end_inst = m->pat->code[end];
int old_at = m->entered_at[end], old_in = m->entered_in[end];
int old_pass = m->pass;
m->entered_at[end] = (int)(sp - m->str);
m->entered_in[end] = old_pass;
m->pass = depth;
int r = bt_match(m, from, body, depth);
m->pass = old_pass;
m->entered_at[end] = old_at;
m->entered_in[end] = old_in;
if (r != BT_CUT(end_inst.offset)) return r;
return end_inst.a ? BT_MATCH : BT_FAIL;
}

/*
* Backtracking engine for patterns with backreferences.
* Step-limited to prevent ReDoS.
Expand Down Expand Up @@ -755,12 +824,12 @@ bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)

case RE_JMP:
/* A backward jump closes e* and returns to its head. When the head is
marked, the body can match empty and iter_at[head] holds where the
marked, the body can match empty and entered_at[head] holds where the
iteration that just ended began (see bt_iter()): an iteration that
ended where it began matched empty, and the repetition stops here,
taking the head's exit and keeping what the iteration captured, as
Onigmo's null check does. */
if (inst.a && m->iter_at[inst.offset] == (int)(sp - str)) {
if (inst.a && ITER_EMPTY(m, inst.offset, sp)) {
pc = pat->code[inst.offset].offset;
break;
}
Expand All @@ -780,7 +849,7 @@ bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)
pc = inst.offset;
break;
}
if (m->iter_at[pc] == (int)(sp - str)) { pc++; break; }
if (ITER_EMPTY(m, pc, sp)) { pc++; break; }
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
return bt_iter(m, sp, inst.offset, pc, depth + 1);
Expand All @@ -803,7 +872,7 @@ bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)
if (r != BT_FAIL && r != BT_LIMIT) return r;
return bt_iter(m, sp, pc + 1, pc, depth + 1);
}
if (m->iter_at[pc] == (int)(sp - str)) { pc++; break; }
if (ITER_EMPTY(m, pc, sp)) { pc++; break; }
int r = bt_iter(m, sp, inst.offset, pc, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
pc++;
Expand Down Expand Up @@ -910,21 +979,19 @@ bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)
break;

case RE_LOOKAHEAD:
{
/* A sub-pattern answers BT_MATCH, BT_FAIL or BT_LIMIT, never a cut.
The two failures go up as they are; the four lookarounds only
differ in what a match means. */
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r != BT_MATCH) return r;
pc = inst.offset;
}
break;
/* A positive lookaround never goes on in this frame: its RE_LOOK_END
has run the text after it, so the sub-pattern's answer is the
frame's, whichever it is. */
return bt_look(m, sp, sp, pc, pc + 1, depth + 1);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

case RE_NEG_LOOKAHEAD:
{
int r = bt_match(m, sp, pc + 1, depth + 1);
/* The sub-pattern matching is the assertion failing; the sub-pattern
running out of alternatives is the assertion holding, and the text
after it goes on here. A limit goes up as it is. */
int r = bt_look(m, sp, sp, pc, pc + 1, depth + 1);
if (r == BT_MATCH) return BT_FAIL;
if (r == BT_LIMIT) return r;
if (r != BT_FAIL) return r;
pc = inst.offset;
}
break;
Expand All @@ -933,25 +1000,44 @@ bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)
{
const char *back = lookbehind_start(pat, str, str_end, sp, pc, binary);
if (!back) return BT_FAIL; /* not enough text before */
int r = bt_match(m, back, pc + 2, depth + 1);
if (r != BT_MATCH) return r;
pc = inst.offset;
return bt_look(m, sp, back, pc, pc + 2, depth + 1);
}
break;

case RE_NEG_LOOKBEHIND:
{
const char *back = lookbehind_start(pat, str, str_end, sp, pc, binary);
if (back) {
int r = bt_match(m, back, pc + 2, depth + 1);
int r = bt_look(m, sp, back, pc, pc + 2, depth + 1);
if (r == BT_MATCH) return BT_FAIL;
if (r == BT_LIMIT) return r;
if (r != BT_FAIL) return r;
}
/* if not enough text before, negative lookbehind succeeds */
pc = inst.offset;
}
break;

case RE_LOOK_END:
{
/* The sub-pattern has matched. For a negative lookaround that is
the whole of its answer, and it goes up as a cut so that the
frames of the sub-pattern undo their captures and try no other
branch on the way; bt_look() reads it back as the match it is.
For a positive one the text after the lookaround goes on from
where the lookaround was entered, inside the sub-pattern's frames
as the text after an atomic group does (RE_ATOMIC_END): a failure
there is a cut, undone for and not backtracked into, since the
sub-pattern matching once is its only match. A limit goes up as
it is. The text after runs in the pass the lookaround was entered
from, which is what its loops' records are keyed by; the pass of
this sub-pattern comes back for the frames above on the way up. */
if (inst.a) return BT_CUT(inst.offset);
int pass = m->pass;
m->pass = m->entered_in[pc];
int r = bt_match(m, str + m->entered_at[pc], pc + 1, depth + 1);
m->pass = pass;
return (r == BT_FAIL) ? BT_CUT(inst.offset) : r;
}

case RE_ATOMIC:
{
/* The body runs on through its RE_ATOMIC_END to the end of the
Expand Down Expand Up @@ -990,19 +1076,23 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
int ncap = pat->num_captures * 2;
if (ncap == 0) ncap = 2;

/* One block: the capture slots, then an iteration record per pc. Every
record a search writes it undoes before returning (see bt_iter()), so
the array is filled once for all start positions. */
int *caps = (int*)mrb_malloc(mrb, sizeof(int) * (ncap + pat->code_len));
/* One block: the capture slots, then an entry record per pc and the pass
that wrote it. Every record a search writes it undoes before returning
(see bt_iter() and bt_look()), so the arrays are filled once for all
start positions. */
int *caps = (int*)mrb_malloc(mrb, sizeof(int) * (ncap + 2 * pat->code_len));
bt_state m;
m.pat = pat;
m.str = str;
m.str_end = str_end;
m.captures = caps;
m.ncap = ncap;
m.iter_at = caps + ncap;
m.entered_at = caps + ncap;
m.entered_in = m.entered_at + pat->code_len;
m.pass = 0;
m.binary = binary;
memset(m.iter_at, -1, sizeof(int) * pat->code_len);
memset(m.entered_at, -1, sizeof(int) * pat->code_len);
memset(m.entered_in, 0, sizeof(int) * pat->code_len);

for (const char *sp = str + start; sp <= str_end && sp <= start_cap; sp++) {
/* Skip ahead using literal prefix or first-byte bitmap */
Expand Down
Loading
Loading