Skip to content

mruby-regexp: hold a character class's ranges sorted and free of overlaps #7207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 2 commits into from
Aug 16, 2026
Merged

mruby-regexp: hold a character class's ranges sorted and free of overlaps #7207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b53fafa
mruby-regexp: hold a character class's ranges sorted and free of over…
takumin Aug 16, 2026
a6e2cac
mruby-regexp: ask for the sources of a run of ASCII members at once
takumin Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 120 additions & 33 deletions mrbgems/mruby-regexp/src/re_compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,31 +191,79 @@ class_get_bit(const re_charclass *cc, uint8_t ch)
return (cc->bitmap[ch >> 3] >> (ch & 7)) & 1;
}

/* Append a non-ASCII codepoint range [lo, hi]. Both bounds must be >= 128. */
/* Add a non-ASCII codepoint range [lo, hi]. Both bounds must be >= 128.

The list is held sorted by `lo`, with no two entries that overlap or touch.
A range the class already covers then costs nothing to add again: the
search below finds the entry holding it and widens that one where it has
to, rather than appending a second entry naming what the first already
accepts. Merging with the entry appended last left that to scan order, and
closing a class under folding has none: the folds of [U+0080-U+2FFF] land
inside the range that was written, and each of them appended an entry that
range already covered, 527 of them for that class alone.

Sorting them is the same answer at match time, where class_match() reads
the list through, and at emit time, where every entry is bytecode.

A byte range carries RE_CLASS_BYTE in both bounds, which is the top bit, so
the tagged entries sort above every codepoint and no gap of one can close
between the two kinds. */
static void
class_add_range(re_compiler *c, re_charclass *cc, uint32_t lo, uint32_t hi)
{
/* Merge with the previous range when the new one is contiguous with or
overlaps it. Codepoints are appended in scan order, so an ascending run
(the common case, e.g. a long [...] enumeration) collapses to a single
range instead of one entry per codepoint. */
if (cc->num_ranges > 0) {
uint32_t *last = &cc->ranges[2 * (cc->num_ranges - 1)];
uint32_t n = cc->num_ranges;

/* Within one walk the ranges arrive ascending, so the entry standing
highest is usually the one the next range joins. Nothing stands above it
to close a gap to, which is what lets it be widened where it is. */
if (n > 0) {
uint32_t *last = &cc->ranges[2 * (n - 1)];
if (lo >= last[0] && lo <= last[1] + 1) {
if (hi > last[1]) last[1] = hi;
return;
}
}
if (cc->num_ranges >= cc->range_capa) {

/* The first entry that reaches [lo, hi] or begins after it: everything
before it ends more than one below `lo` and cannot join. */
uint32_t i = 0, j = n;
while (i < j) {
uint32_t mid = i + (j - i) / 2;
if (cc->ranges[2 * mid + 1] + 1 < lo) i = mid + 1;
else j = mid;
}

if (i < n && cc->ranges[2 * i] <= hi + 1) {
/* It joins entry i. Widening it can close the gap to the entries above,
so they are swallowed and what is left of the list is closed up. */
uint32_t *r = &cc->ranges[2 * i];
if (lo < r[0]) r[0] = lo;
if (hi > r[1]) r[1] = hi;
uint32_t k = i + 1;
while (k < n && cc->ranges[2 * k] <= r[1] + 1) {
if (cc->ranges[2 * k + 1] > r[1]) r[1] = cc->ranges[2 * k + 1];
k++;
}
if (k > i + 1) {
memmove(&cc->ranges[2 * (i + 1)], &cc->ranges[2 * k],
sizeof(uint32_t) * 2 * (n - k));
cc->num_ranges = n - (k - i - 1);
}
return;
}

if (n >= cc->range_capa) {
/* range_capa/num_ranges are uint32_t: doubling from 32768 no longer
wraps to 0 (which fed a size-0 realloc and a write through NULL). */
uint32_t new_capa = cc->range_capa ? cc->range_capa * 2 : 4;
cc->ranges = (uint32_t*)mrb_realloc(c->mrb, cc->ranges, sizeof(uint32_t) * 2 * new_capa);
cc->range_capa = new_capa;
}
cc->ranges[2 * cc->num_ranges] = lo;
cc->ranges[2 * cc->num_ranges + 1] = hi;
cc->num_ranges++;
memmove(&cc->ranges[2 * (i + 1)], &cc->ranges[2 * i],
sizeof(uint32_t) * 2 * (n - i));
cc->ranges[2 * i] = lo;
cc->ranges[2 * i + 1] = hi;
cc->num_ranges = n + 1;
}

/* Add a single non-ASCII codepoint to the class. */
Expand Down Expand Up @@ -257,6 +305,25 @@ class_add_fold_counterparts(re_compiler *c, uint16_t id, uint32_t cp)
}

#ifdef RE_UNICODE_CASE
/* The part of the class at or above `cp`, as the first range holding a member
that high. FALSE where the class holds none. The list is sorted, so this is
a search rather than a scan, and it answers against the list as it stands
rather than against a position taken before it moved. */
static mrb_bool
class_next_range(const re_charclass *cc, uint32_t cp, uint32_t *lo, uint32_t *hi)
{
uint32_t n = cc->num_ranges, i = 0, j = n;
while (i < j) {
uint32_t mid = i + (j - i) / 2;
if (cc->ranges[2 * mid + 1] < cp) i = mid + 1;
else j = mid;
}
if (i >= n) return FALSE;
*lo = cc->ranges[2 * i] > cp ? cc->ranges[2 * i] : cp;
*hi = cc->ranges[2 * i + 1];
return TRUE;
}

/* Closure for the range walks, which report counterpart spans one at a time.
A span can straddle 128 (U+017F folds to 's'), so it is split the same way
a written range is. */
Expand Down Expand Up @@ -748,36 +815,56 @@ compile_charclass(re_compiler *c)
the first already added. */
class_fold_sink sink = { c, cc };

/* Round one: the fold of every member joins the class. The codepoint list
is read from a snapshot of its length, since the additions append to
the same list and an unbounded walk would keep folding what it just
added. */
uint32_t nranges = cc->num_ranges;
for (uint32_t i = 0; i < nranges; i++) {
if (cc->ranges[2 * i] & RE_CLASS_BYTE) continue;
mrb_uni_case_fold_range(cc->ranges[2 * i], cc->ranges[2 * i + 1],
class_fold_add, &sink);
/* Round one: the fold of every member joins the class. Both rounds walk
the list by codepoint rather than by index, since class_add_range()
inserts where the order puts an entry and an index would name a
different entry after one did. A walk that starts each step above the
range it just handed over reaches every entry the round began with, and
terminates whatever the additions do.

What it may skip is an entry inserted behind the cursor, and neither
round has anything to say about one. Folding is idempotent, so the fold
of a fold this round added is the fold itself; and nothing folds to a
character that folds elsewhere, so a source the next round adds has no
source of its own. */
for (uint32_t cp = 0;;) {
uint32_t lo, hi;
/* The tagged byte ranges sort above every codepoint, so the first one
reached ends the walk rather than being stepped over. */
if (!class_next_range(cc, cp, &lo, &hi) || (lo & RE_CLASS_BYTE)) break;
mrb_uni_case_fold_range(lo, hi, class_fold_add, &sink);
cp = hi + 1;
}
for (int ch = 'A'; ch <= 'Z'; ch++) {
if (class_get_bit(cc, (uint8_t)ch)) class_set_bit(cc, (uint8_t)(ch + 32));
}

/* Round two: every source of a member joins it too. The bitmap is walked
upwards, so the upper case letter set here is behind the cursor and is
never asked for sources of its own, which is correct: nothing folds to
an upper case letter. */
nranges = cc->num_ranges;
for (uint32_t i = 0; i < nranges; i++) {
if (cc->ranges[2 * i] & RE_CLASS_BYTE) continue;
mrb_uni_case_unfold_range(cc->ranges[2 * i], cc->ranges[2 * i + 1],
class_fold_add, &sink);
/* Round two: every source of a member joins it too. */
for (uint32_t cp = 0;;) {
uint32_t lo, hi;
if (!class_next_range(cc, cp, &lo, &hi) || (lo & RE_CLASS_BYTE)) break;
mrb_uni_case_unfold_range(lo, hi, class_fold_add, &sink);
cp = hi + 1;
}
/* An ASCII member can have a non-ASCII source (U+212A folds to 'k'),
which the range walk cannot reach: the bitmap holds no ranges. A run of
set bits is asked about in one question, since a question costs a walk
of the tables whatever it spans. The tables hold no ASCII source, ASCII
being what they are the rest of, so nothing this walk finds lands in
the bitmap and no run of it grows while it is being read. */
for (int ch = 0; ch < 128; ch++) {
if (!class_get_bit(cc, (uint8_t)ch)) continue;
if (ch >= 'a' && ch <= 'z') class_set_bit(cc, (uint8_t)(ch - 32));
/* An ASCII member can have a non-ASCII source (U+212A folds to 'k'),
which the range walk cannot reach: the bitmap holds no ranges. */
mrb_uni_case_unfold_range((uint32_t)ch, (uint32_t)ch, class_fold_add, &sink);
int end = ch;
while (end + 1 < 128 && class_get_bit(cc, (uint8_t)(end + 1))) end++;
mrb_uni_case_unfold_range((uint32_t)ch, (uint32_t)end, class_fold_add, &sink);
ch = end;
}
/* The upper case letter of a lower case member is the source the tables
do not hold, so it is set here. Nothing folds to an upper case letter,
which is what lets this come after the walk above rather than adding to
what that walk is asked about. */
for (int ch = 'a'; ch <= 'z'; ch++) {
if (class_get_bit(cc, (uint8_t)ch)) class_set_bit(cc, (uint8_t)(ch - 32));
}
#else
/* The same closure, restricted to the foldings this build has. Refusing
Expand Down
24 changes: 24 additions & 0 deletions mrbgems/mruby-regexp/test/regexp_utf8.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,30 @@
assert_equal 0, ("₅" =~ /[a-z₀-₉]/)
end

assert("Regexp - a class holds the union of its ranges however they are written") do
# The ranges are held sorted and free of overlaps, so writing one inside
# another, writing a pair the wrong way round, or naming the same member
# twice all come to the class the union spells once. What it takes for that
# to hold is a search: only the range written last used to be widened, so
# anything written out of order was kept as a second entry naming what the
# first already accepted.
skip unless __ENCODING__ == "UTF-8"
[/[Ā-Ȁ]/, /[ƀ-ȀĀ-Ɛ]/,
/[Ā-Őő-Ȁ]/, /[Ā-ȀĠ-İ]/,
/[ȀĀ-ȀĀ]/].each do |re|
assert_equal 0, ("Ā" =~ re)
assert_equal 0, ("Ő" =~ re)
assert_equal 0, ("Ȁ" =~ re)
assert_nil ("ÿ" =~ re)
assert_nil ("ȁ" =~ re)
assert_nil ("a" =~ re)
# The negation reads the same class, so it draws the same boundary.
neg = Regexp.new("[^" + re.source[1..-1])
assert_nil ("Ő" =~ neg)
assert_equal 0, ("ȁ" =~ neg)
end
end

assert("Regexp - quantifier over multi-byte char class") do
assert_equal "a#b#c", "a₀₁b₂c".gsub(/[₀-₉]+/, "#")
assert_equal ["₀₁₂"], "₀₁₂".scan(/[₀-₉]+/)
Expand Down
Loading