Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 59 additions & 16 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@
#include <mruby/internal.h>
#include <string.h>

/* Class IDs are stored in re_inst.a (uint8_t), so at most 256 distinct
character classes can be encoded. Without this cap, class_capa
(uint16_t) overflows on doubling past 32768 (8 -> 16 -> ... -> 32768
-> 0), mrb_realloc with size 0 returns NULL, and the next memset
crashes; even before that, the (uint8_t)id cast at emit sites would
silently alias different classes. */
#define RE_MAX_CLASSES 256

/* Compiler state.

Everything the compile allocates and the finished pattern goes on owning
Expand Down Expand Up @@ -46,6 +54,9 @@ typedef struct {
before the atom, and a `\u{...}` list moves it
forward so the quantifier repeats the last
codepoint alone */
uint32_t literal_cp[RE_MAX_CLASSES]; /* by class id: the codepoint whose
/i literal the class stands for, 0 for a class
made by anything else; see literal_class() */
} re_compiler;

static void compile_alt(re_compiler *c); /* forward */
Expand Down Expand Up @@ -177,14 +188,6 @@ next_char(re_compiler *c)
return (uint8_t)*c->p++;
}

/* Class IDs are stored in re_inst.a (uint8_t), so at most 256 distinct
character classes can be encoded. Without this cap, class_capa
(uint16_t) overflows on doubling past 32768 (8 -> 16 -> ... -> 32768
-> 0), mrb_realloc with size 0 returns NULL, and the next memset
crashes; even before that, the (uint8_t)id cast at emit sites would
silently alias different classes. */
#define RE_MAX_CLASSES 256

static uint16_t
add_class(re_compiler *c)
{
Expand All @@ -203,6 +206,40 @@ add_class(re_compiler *c)
return id;
}

/* The class a /i literal for `cp` compiles to, whether it exists yet or not.

The class holds `cp` and its case counterparts, and nothing else reaches it:
not the flags in force, not the pattern around it, and no writer once the
emitter that made it has returned. What it holds is a function of `cp`, so
the second occurrence of a codepoint can name the class the first one made
rather than make another. Each occurrence used to make its own, and a class
id is a uint8_t, so a phrase of a few hundred letters under /i ran out of
ids and was refused as having too many character classes, where the
classes it needed were as many as its distinct letters.

Every class is recorded by id, which keeps the record the size of the id
space and lets it be searched to `num_classes` alone. It sits in the
compiler's own frame rather than behind `pat`, since nothing outlives the
compile that would want it. Zero marks a class that stands for no literal:
it cannot be mistaken for one, since U+0000 has no case and neither caller
folds it. Only the id is handed back, and it is for the caller to fill a
class that is new, so that a class this function made is never taken for
one it found. */
static uint16_t
literal_class(re_compiler *c, uint32_t cp, mrb_bool *found)
{
for (uint16_t id = 0; id < c->pat->num_classes; id++) {
if (c->literal_cp[id] == cp) {
*found = TRUE;
return id;
}
}
uint16_t id = add_class(c);
c->literal_cp[id] = cp;
*found = FALSE;
return id;
}

static void
class_set_bit(re_charclass *cc, uint8_t ch)
{
Expand Down Expand Up @@ -1080,10 +1117,13 @@ emit_char(re_compiler *c, uint8_t ch)
{
if ((c->flags & RE_FLAG_IGNORECASE) &&
((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))) {
uint16_t id = add_class(c);
class_set_bit(&c->pat->classes[id], ch);
class_set_bit(&c->pat->classes[id], (uint8_t)(ch ^ 0x20)); /* the other case */
class_add_fold_counterparts(c, id, ch);
mrb_bool found;
uint16_t id = literal_class(c, ch, &found);
if (!found) {
class_set_bit(&c->pat->classes[id], ch);
class_set_bit(&c->pat->classes[id], (uint8_t)(ch ^ 0x20)); /* the other case */
class_add_fold_counterparts(c, id, ch);
}
emit(c, RE_CLASS, (uint8_t)id, 0);
return;
}
Expand Down Expand Up @@ -1136,10 +1176,13 @@ emit_cp_folded(re_compiler *c, uint32_t cp)
if (f != cp) { alt[n++] = f; alt[n++] = f - 32; }
#endif
if (n == 0) return FALSE;
uint16_t id = add_class(c);
class_add_codepoint(c, &c->pat->classes[id], cp);
for (int i = 0; i < n; i++) {
class_add_member(c, &c->pat->classes[id], alt[i], FALSE);
mrb_bool found;
uint16_t id = literal_class(c, cp, &found);
if (!found) {
class_add_codepoint(c, &c->pat->classes[id], cp);
for (int i = 0; i < n; i++) {
class_add_member(c, &c->pat->classes[id], alt[i], FALSE);
}
}
emit(c, RE_CLASS, (uint8_t)id, 0);
return TRUE;
Expand Down
20 changes: 20 additions & 0 deletions mrbgems/mruby-regexp/test/regexp_syntax.rb
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,26 @@
assert_true re.match?("Abc")
end

assert("Regexp - /i literals share one class per letter") do
# Under /i a letter compiles to a class of its cases, and a class id is a
# byte, so a pattern holds at most 256 of them. Every occurrence used to take
# one, and a phrase of a few hundred letters was refused as too many
# character classes; the second occurrence of a letter now names the class
# the first one made.
re = Regexp.new("a" * 300, Regexp::IGNORECASE)
assert_true re.match?("A" * 300)
assert_true re.match?("a" * 300)
assert_false re.match?("A" * 299)
re = Regexp.new("aA" * 150, Regexp::IGNORECASE)
assert_true re.match?("AA" * 150)
assert_true re.match?("aa" * 150)
# The class is consulted only where /i is on: outside it the same letter
# matches its own case alone.
re = Regexp.new("(?i:a)a")
assert_true re.match?("Aa")
assert_false re.match?("AA")
end

assert("Regexp - case insensitive character class") do
# /i used to be folded in only where a single literal was emitted, so a
# character class ignored it entirely.
Expand Down
42 changes: 42 additions & 0 deletions mrbgems/mruby-regexp/test/unicode_case.rb
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,45 @@
assert_nil "ss".match(/ß/i)
assert_nil "ff".match(/ff/i)
end

assert("Regexp - /i literals share one class per codepoint") do
skip unless __ENCODING__ == "UTF-8"
# A folded literal compiles to a class, and a class id is a byte, so a
# pattern holds at most 256 of them. Every occurrence used to take one, and a
# phrase of a few hundred letters was refused as too many character
# classes; the second occurrence of a codepoint now names the class the
# first one made.
re = Regexp.new("д" * 300, Regexp::IGNORECASE)
assert_true re.match?("Д" * 300)
assert_true re.match?("д" * 300)
assert_false re.match?("Д" * 299)
# Both cases of a letter, and a run of one letter next to a run of another,
# each cost as many classes as they have distinct codepoints.
re = Regexp.new("дД" * 150, Regexp::IGNORECASE)
assert_true re.match?("ДД" * 150)
assert_true re.match?("дд" * 150)
re = Regexp.new("д" * 256 + "a" * 256, Regexp::IGNORECASE)
assert_true re.match?("Д" * 256 + "A" * 256)
# A `\u` escape and a spelled out character name the same codepoint, so
# they share the class as well.
re = Regexp.new("\\u{434}" * 150 + "д" * 150, Regexp::IGNORECASE)
assert_true re.match?("Д" * 300)
# The class is consulted only where /i is on: outside it the same codepoint
# is its bytes and matches its own case alone.
re = Regexp.new("(?i:д)д")
assert_true re.match?("Дд")
assert_false re.match?("ДД")
# What the cap counts now is distinct codepoints. Cyrillic, Latin-1, Greek
# and Armenian letters below come to 281 of them, past what the id can hold,
# so the pattern is still refused; CRuby has no such cap.
cyr = (0x400..0x45f).map {|c| c.chr("UTF-8") }.join
lat = ((0xc0..0xd6).to_a + (0xd8..0xde).to_a + (0xe0..0xf6).to_a +
(0xf8..0xfe).to_a).map {|c| c.chr("UTF-8") }.join
grk = ((0x391..0x3a1).to_a + (0x3a3..0x3a9).to_a +
(0x3b1..0x3c9).to_a).map {|c| c.chr("UTF-8") }.join
arm = ((0x531..0x556).to_a + (0x561..0x586).to_a).map {|c| c.chr("UTF-8") }.join
assert_equal 281, (cyr + lat + grk + arm).length
re = Regexp.new(cyr + lat + grk, Regexp::IGNORECASE)
assert_true re.match?((cyr + lat + grk).upcase)
assert_raise(RegexpError) { Regexp.new(cyr + lat + grk + arm, Regexp::IGNORECASE) }
end
Loading