Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions mrbgems/mruby-regexp/include/re_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,8 @@ typedef struct mrb_regexp_pattern {
#define MRB_REGEXP_STEP_LIMIT 1000000
#endif

/* Recursion-depth limit for bt_match: bounds C stack growth on
patterns like `(?=)+` that recurse without consuming input. */
/* Recursion-depth limit for bt_match, which recurses at every fork and
every capture: bounds C stack growth on a long subject or a deep pattern. */
#ifndef MRB_REGEXP_RECURSION_LIMIT
#define MRB_REGEXP_RECURSION_LIMIT 1000
#endif
Expand Down
32 changes: 24 additions & 8 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -2165,9 +2165,12 @@ first_set_walk(const re_inst *code, uint32_t code_len,
return FALSE;
}

/* TRUE when an epsilon-only path runs from pc to goal, so the repetition that
goal closes can complete an iteration without consuming. seen[] is marked
with `mark` rather than cleared, so one buffer serves every edge. */
/* TRUE when a path that need not consume runs from pc to goal, so the
repetition that goal closes can complete an iteration without consuming.
A lookaround is zero-width whatever its sub-pattern does, so the walk
steps over the sub-pattern; a backreference to a group that captured
empty consumes nothing, so it can be on such a path. seen[] is marked with
`mark` rather than cleared, so one buffer serves every edge. */
static mrb_bool
epsilon_path(const re_inst *code, uint32_t pc, uint32_t goal,
uint32_t *seen, uint32_t mark)
Expand All @@ -2180,9 +2183,12 @@ epsilon_path(const re_inst *code, uint32_t pc, uint32_t goal,
case RE_BOL: case RE_EOL: case RE_BOT: case RE_EOT: case RE_EOTNL:
case RE_WBOUND: case RE_NWBOUND:
case RE_ATOMIC: case RE_ATOMIC_END:
case RE_BACKREF:
pc++;
break;
case RE_JMP:
case RE_LOOKAHEAD: case RE_NEG_LOOKAHEAD:
case RE_LOOKBEHIND: case RE_NEG_LOOKBEHIND:
pc = code[pc].offset;
break;
case RE_SPLIT:
Expand All @@ -2191,17 +2197,22 @@ epsilon_path(const re_inst *code, uint32_t pc, uint32_t goal,
pc++;
break;
default:
return FALSE; /* consumes input, or is an assertion this walk cannot judge */
return FALSE; /* consumes input */
}
}
return TRUE;
}

/* Find the repetitions whose body can match empty and mark the backward edge
that closes each one, so the Pike VM knows which loops need the empty-
iteration handling in add_thread() and which can stay on the cheap path.
Returns how deeply those loops nest, which bounds the VM's epsilon passes
and the thread lists sized from them; see RE_MAX_PASS and RE_LIST_CAPA. */
that closes each one, so that both engines know which loops need the
empty-iteration handling (add_thread() and bt_match()) and which can stay
on the cheap path. A repetition laid out as e* is closed by a jump back to
its SPLIT/SPLITNG head, and that head is marked too: it is where an
iteration begins, which the backtracker has to record; see bt_iter(). The
head is a forward edge, and a forward edge's mark means nothing else.
Returns how deeply the marked loops nest, which bounds the VM's epsilon
passes and the thread lists sized from them; see RE_MAX_PASS and
RE_LIST_CAPA. */
static uint8_t
mark_empty_loops(mrb_state *mrb, re_inst *code, uint32_t code_len)
{
Expand All @@ -2222,6 +2233,11 @@ mark_empty_loops(mrb_state *mrb, re_inst *code, uint32_t code_len)
if (in.offset > pc) continue; /* forward edge: alternation, not a loop */
if (!epsilon_path(code, in.offset, pc, seen, ++mark)) continue;
code[pc].a = 1;
if (in.op == RE_JMP) {
/* The head was passed earlier in this scan, so its mark stays. */
mrb_assert(code[in.offset].op == RE_SPLIT || code[in.offset].op == RE_SPLITNG);
code[in.offset].a = 1;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
delta[in.offset]++;
delta[pc + 1]--; /* the closing edge itself still sits inside the loop */
}
Expand Down
146 changes: 124 additions & 22 deletions mrbgems/mruby-regexp/src/re_exec.c
Original file line number Diff line number Diff line change
Expand Up @@ -626,29 +626,76 @@ lookbehind_start(const mrb_regexp_pattern *pat, const char *str,
limit. A frame that gets it hands it up; a SPLIT takes it as that branch
failing and answers with its other branch, as it would with a failure.
What no frame does is turn it into a cut or into a lookaround's answer,
since a limit says nothing about the text. That matters because the
recursion limit is what stops a repetition whose body can match empty
under this engine (see MRB_REGEXP_RECURSION_LIMIT), and the repetition
being stopped may be inside an atomic group's body, where a cut would keep
its exit from being taken, or inside a negative lookaround, where reading
the limit as "no match" would make the assertion hold. */
since a limit says nothing about the text: the frame giving up may be
inside an atomic group's body, where a cut would keep the group's exit
from being taken, or inside a negative lookaround, where reading the limit
as "no match" would make the assertion hold. */
#define BT_FAIL 0
#define BT_MATCH 1
#define BT_LIMIT 2
#define BT_CUT(atomic_depth) (-(int)(atomic_depth))

/* What one backtrack_exec() call shares between its bt_match() frames: the
pattern, the subject, the capture slots being written, the step count and
the iteration records. A frame's own state is its position, its pc and its
depth. */
typedef struct {
const mrb_regexp_pattern *pat;
const char *str;
const char *str_end;
int *captures;
int ncap;
int steps;
/* Per pc, the offset the running iteration of the loop that pc keys began
at, or -1 while none is running. A repetition whose body can match empty
has to stop once an iteration ends where it began, or it would go round
at the same position until a limit refused it and answer with whatever
the alternatives left inside the limit produce. Onigmo stops it with a
null check around the body; this array is that check's memory. The pc
that keys a loop is its marked head for e* (the SPLIT/SPLITNG whose
offset is the exit; the JMP closing the body reads the record) and its
marked back edge for e+ (the SPLIT/SPLITNG at the end of the body, which
both writes and reads it); see mark_empty_loops(). */
int *iter_at;
mrb_bool binary;
} bt_state;

static int bt_match(bt_state *m, const char *sp, uint32_t pc, int depth);

/* Run the frame at pc as the start of an iteration of the loop `key` keys,
recording where it begins so that the edge closing the body can tell an
empty iteration. The record lasts exactly as long as the frame: the frame
that ran the edge into the body is the one that undoes it, so backtracking
out of an iteration finds the record of the one it lands in, and the
branch that begins an iteration is run this way rather than in place even
when it is the frame's last, so that there is one place to undo it. */
static int
bt_iter(bt_state *m, const char *sp, uint32_t pc, uint32_t key, int depth)
{
int old = m->iter_at[key];
m->iter_at[key] = (int)(sp - m->str);
int r = bt_match(m, sp, pc, depth);
m->iter_at[key] = old;
return r;
}

/*
* Backtracking engine for patterns with backreferences.
* Step-limited to prevent ReDoS.
*/
static int
bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
const char *sp, uint32_t pc, int *captures, int ncap, int *steps,
int depth, mrb_bool binary)
bt_match(bt_state *m, const char *sp, uint32_t pc, int depth)
{
const mrb_regexp_pattern *pat = m->pat;
const char *str = m->str;
const char *str_end = m->str_end;
int *captures = m->captures;
int ncap = m->ncap;
mrb_bool binary = m->binary;

if (depth > MRB_REGEXP_RECURSION_LIMIT) return BT_LIMIT;
while (pc < pat->code_len) {
if (++(*steps) > MRB_REGEXP_STEP_LIMIT) return BT_LIMIT;
if (++m->steps > MRB_REGEXP_STEP_LIMIT) return BT_LIMIT;

re_inst inst = pat->code[pc];
switch (inst.op) {
Expand Down Expand Up @@ -695,20 +742,63 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
return BT_MATCH;

case RE_JMP:
/* A backward jump closes e* and returns to its head. When the head is
marked, the body can match empty and iter_at[head] holds where the
iteration that just ended began (see bt_iter()): an iteration that
ended where it began matched empty, and the repetition stops here,
taking the head's exit and keeping what the iteration captured, as
Onigmo's null check does. */
if (inst.a && m->iter_at[inst.offset] == (int)(sp - str)) {
pc = pat->code[inst.offset].offset;
break;
}
pc = inst.offset;
break;

case RE_SPLIT:
/* Greedy fork: pc+1 first, then the jump target. A marked one is an
edge of a repetition whose body can match empty. Forward, it heads
e*, and pc+1 begins an iteration. Backward, it closes e+?, and its
target begins the next iteration, unless the one that just ended was
empty: then there is only the exit, as at a marked RE_JMP. */
if (inst.a) {
if (inst.offset > pc) {
int r = bt_iter(m, sp, pc + 1, pc, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
pc = inst.offset;
break;
}
if (m->iter_at[pc] == (int)(sp - str)) { pc++; break; }
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
return bt_iter(m, sp, inst.offset, pc, depth + 1);
}
{
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
}
pc = inst.offset;
break;

case RE_SPLITNG:
/* Non-greedy fork: the jump target first, then pc+1. Marked, forward
it heads e*? and backward it closes e+; the iteration-starting branch
is the other one from RE_SPLIT's, and the empty-iteration stop is the
same. */
if (inst.a) {
if (inst.offset > pc) {
int r = bt_match(m, sp, inst.offset, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
return bt_iter(m, sp, pc + 1, pc, depth + 1);
}
if (m->iter_at[pc] == (int)(sp - str)) { pc++; break; }
int r = bt_iter(m, sp, inst.offset, pc, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
pc++;
break;
}
{
int r = bt_match(pat, str, str_end, sp, inst.offset, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, inst.offset, depth + 1);
if (r != BT_FAIL && r != BT_LIMIT) return r;
}
pc++;
Expand All @@ -727,7 +817,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
if (slot < ncap) {
int old = captures[slot];
captures[slot] = (int)(sp - str);
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r == BT_MATCH) return r;
/* undone for a cut as for a failure: the group the cut fails may
be the one this slot was written inside */
Expand Down Expand Up @@ -812,15 +902,15 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
/* A sub-pattern answers BT_MATCH, BT_FAIL or BT_LIMIT, never a cut.
The two failures go up as they are; the four lookarounds only
differ in what a match means. */
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r != BT_MATCH) return r;
pc = inst.offset;
}
break;

case RE_NEG_LOOKAHEAD:
{
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
if (r == BT_MATCH) return BT_FAIL;
if (r == BT_LIMIT) return r;
pc = inst.offset;
Expand All @@ -831,7 +921,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
{
const char *back = lookbehind_start(pat, str, str_end, sp, pc, binary);
if (!back) return BT_FAIL; /* not enough text before */
int r = bt_match(pat, str, str_end, back, pc + 2, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, back, pc + 2, depth + 1);
if (r != BT_MATCH) return r;
pc = inst.offset;
}
Expand All @@ -841,7 +931,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
{
const char *back = lookbehind_start(pat, str, str_end, sp, pc, binary);
if (back) {
int r = bt_match(pat, str, str_end, back, pc + 2, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, back, pc + 2, depth + 1);
if (r == BT_MATCH) return BT_FAIL;
if (r == BT_LIMIT) return r;
}
Expand All @@ -857,7 +947,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
here: the answer is passed up, except that a cut aimed at this
group is this group failing, which the caller backtracks over the
way it would any other failed atom. */
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
return (r == BT_CUT(inst.offset)) ? BT_FAIL : r;
}

Expand All @@ -867,7 +957,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
when what follows fails, the failure is a cut, so that the SPLITs
inside the body do not get to try their other branches. A limit
is not the text failing and goes up as it is. */
int r = bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps, depth + 1, binary);
int r = bt_match(m, sp, pc + 1, depth + 1);
return (r == BT_FAIL) ? BT_CUT(inst.offset) : r;
}

Expand All @@ -888,7 +978,19 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
int ncap = pat->num_captures * 2;
if (ncap == 0) ncap = 2;

int *caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap);
/* One block: the capture slots, then an iteration record per pc. Every
record a search writes it undoes before returning (see bt_iter()), so
the array is filled once for all start positions. */
int *caps = (int*)mrb_malloc(mrb, sizeof(int) * (ncap + pat->code_len));
bt_state m;
m.pat = pat;
m.str = str;
m.str_end = str_end;
m.captures = caps;
m.ncap = ncap;
m.iter_at = caps + ncap;
m.binary = binary;
memset(m.iter_at, -1, sizeof(int) * pat->code_len);

for (const char *sp = str + start; sp <= str_end && sp <= start_cap; sp++) {
/* Skip ahead using literal prefix or first-byte bitmap */
Expand All @@ -906,9 +1008,9 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
continue;
}
memset(caps, -1, sizeof(int) * ncap);
int steps = 0;
m.steps = 0;

if (bt_match(pat, str, str_end, sp, 0, caps, ncap, &steps, 0, binary) == BT_MATCH) {
if (bt_match(&m, sp, 0, 0) == BT_MATCH) {
if (captures) {
int copy = ncap < captures_size ? ncap : captures_size;
memcpy(captures, caps, sizeof(int) * copy);
Expand Down
Loading
Loading