Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion mrbgems/mruby-regexp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,10 @@ means the pattern wants a build that converts case by Unicode.
Those two are the only foldings whose result is an ASCII letter, and both
builds carry them, so that folding "ASCII only" covers the whole of the
equivalence class an ASCII letter belongs to rather than the part of it that
is ASCII.
is ASCII. A class holding the letter only through `\w`, `[:word:]` or
`[:ascii:]` does not reach them: those are sets ASCII defines, so `[\w]`
under `/i` stays the ASCII word characters and `[^\w]` accepts `"K"` (U+212A),
as in CRuby. A letter written out beside the shorthand (`[\ws]`) folds as usual.

## License

Expand Down
50 changes: 39 additions & 11 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -446,13 +446,18 @@ class_is_ascii_only(const re_charclass *cc)

/* Set ASCII bits for a POSIX class name (e.g. "alpha") into a 128-bit map.
Returns FALSE for an unknown name. Semantics are ASCII, like this gem's
\w/\d shorthands; non-ASCII codepoints are not classified. */
\w/\d shorthands; non-ASCII codepoints are not classified.

*ascii_set is TRUE for a name whose set ASCII defines, [:word:] and
[:ascii:], as opposed to one ASCII merely bounds here; the distinction is
what compile_charclass() folds by. */
static mrb_bool
posix_class_bits(uint8_t *bits, const char *name, size_t len)
posix_class_bits(uint8_t *bits, const char *name, size_t len, mrb_bool *ascii_set)
{
#define NAME_IS(s) (len == sizeof(s) - 1 && memcmp(name, s, len) == 0)
#define BSET(ch) (bits[(ch) >> 3] |= (uint8_t)(1u << ((ch) & 7)))
#define BRANGE(lo, hi) do { for (int i = (lo); i <= (hi); i++) BSET(i); } while (0)
*ascii_set = NAME_IS("word") || NAME_IS("ascii");
if (NAME_IS("alpha")) { BRANGE('a','z'); BRANGE('A','Z'); }
else if (NAME_IS("digit")) { BRANGE('0','9'); }
else if (NAME_IS("alnum")) { BRANGE('a','z'); BRANGE('A','Z'); BRANGE('0','9'); }
Expand Down Expand Up @@ -741,6 +746,12 @@ compile_charclass(re_compiler *c)
negated = TRUE;
}

/* What \w, \W, [:word:] and [:ascii:] add is held apart until the class
has been closed under folding, and joins the bitmap after; see the fold
below for why. Only the bitmap and utf8_any are ever written here. */
re_charclass ascii_set;
memset(&ascii_set, 0, sizeof(ascii_set));

mrb_bool first = TRUE;
while (peek(c) != ']' || first) {
if (peek(c) < 0) compile_error(c, "unterminated character class");
Expand All @@ -757,16 +768,18 @@ compile_charclass(re_compiler *c)
while (peek(c) >= 0 && peek(c) != ':' && peek(c) != ']') next_char(c);
if (peek(c) == ':' && c->p + 1 < c->src_end && c->p[1] == ']') {
uint8_t bits[16] = {0};
if (!posix_class_bits(bits, name, (size_t)(c->p - name))) {
mrb_bool by_ascii;
if (!posix_class_bits(bits, name, (size_t)(c->p - name), &by_ascii)) {
compile_error(c, "invalid POSIX bracket class");
}
next_char(c); /* ':' */
next_char(c); /* ']' */
re_charclass *dst = by_ascii ? &ascii_set : cc;
for (int i = 0; i < 128; i++) {
mrb_bool in = (bits[i >> 3] >> (i & 7)) & 1;
if (in != neg) class_set_bit(cc, (uint8_t)i);
if (in != neg) class_set_bit(dst, (uint8_t)i);
}
if (neg) cc->utf8_any = TRUE; /* [:^...:] matches non-ASCII too */
if (neg) dst->utf8_any = TRUE; /* [:^...:] matches non-ASCII too */
continue;
}
c->p = save; /* not a POSIX class; treat '[' as a literal below */
Expand All @@ -781,7 +794,7 @@ compile_charclass(re_compiler *c)
esc == 's' || esc == 'S' || esc == 'h' || esc == 'H') {
next_char(c); /* '\\' */
next_char(c); /* spec */
class_add_shorthand(cc, esc);
class_add_shorthand((esc == 'w' || esc == 'W') ? &ascii_set : cc, esc);
continue;
}
}
Expand Down Expand Up @@ -821,16 +834,28 @@ compile_charclass(re_compiler *c)

/* Close the class under case folding for /i. This runs once the class is
complete, so it covers every form the loop above merges in: POSIX
brackets, shorthands, ranges and single literals. Negation is applied at
match time against the same class (RE_NCLASS), so closing the positive
set is also what keeps [^a-c] and [^Ā] from accepting what they were
written to reject.
brackets, ranges and single literals. Negation is applied at match time
against the same class (RE_NCLASS), so closing the positive set is also
what keeps [^a-c] and [^Ā] from accepting what they were written to
reject.

Closing means: x belongs to the class whenever some written member folds
the same way x does. A byte member has no case: it stands for no character,
so nothing folds to it and it folds to nothing. Every walk below steps over
the tagged ranges, which is also what keeps /i from refusing a class of
continuation bytes on a build without the folding tables. */
continuation bytes on a build without the folding tables.

The word class and [:ascii:] are still held apart here, so the closure
never sees them. Each is a set ASCII defines: \w is [a-zA-Z0-9_] and no
more, so a fold that leaves ASCII leaves the set, and [\w] under /i is
the ASCII word characters where [k] under /i reaches U+212A. CRuby reads
them the same way, keeping the two out of the class it folds across the
boundary from, and it is what makes [^\w] under /i accept U+017F. Both
hold both cases of every letter they hold, so the ASCII part of the
closure has nothing to add to them, and joining them after it costs
nothing. The other POSIX brackets are ASCII here only for want of a
table, and stay in: CRuby folds them too, and there their members above
ASCII hold what the fold adds anyway. */
if (c->flags & RE_FLAG_IGNORECASE) {
#ifdef RE_UNICODE_CASE
/* That takes two rounds rather than one walk in each direction, because a
Expand Down Expand Up @@ -917,6 +942,9 @@ compile_charclass(re_compiler *c)
#endif
}

for (int i = 0; i < RE_CLASS_BITMAP_SIZE; i++) cc->bitmap[i] |= ascii_set.bitmap[i];
if (ascii_set.utf8_any) cc->utf8_any = TRUE;

cc->negated = negated;
emit(c, negated ? RE_NCLASS : RE_CLASS, (uint8_t)id, 0);
}
Expand Down
54 changes: 54 additions & 0 deletions mrbgems/mruby-regexp/test/regexp_syntax.rb
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,60 @@
assert_true Regexp.new("[^k]").match?(kelvin)
end

assert("Regexp - /i keeps the word class inside ASCII") do
# `\w` is [a-zA-Z0-9_] and no more, and [:word:] and [:ascii:] are sets
# ASCII defines the same way, so /i folds none of them across the boundary:
# the fold of a member that leaves ASCII leaves the set. CRuby reads them
# the same way. The negated forms are where it shows: [^\w] under /i has to
# accept U+212A and U+017F, which are not word characters, and used to
# reject them because the closure of [k] and [s] had been applied to `\w`.
# Both sources lie above ASCII, so they are characters only where the
# pattern and the subject are read as characters.
skip unless __ENCODING__ == "UTF-8"
kelvin = "K"
long_s = "ſ"
[kelvin, long_s].each do |ch|
assert_false Regexp.new("[\\w]", Regexp::IGNORECASE).match?(ch)
assert_true Regexp.new("[^\\w]", Regexp::IGNORECASE).match?(ch)
assert_false Regexp.new("[[:ascii:]]", Regexp::IGNORECASE).match?(ch)
assert_true Regexp.new("[^[:ascii:]]", Regexp::IGNORECASE).match?(ch)
# `\W` holds neither letter and everything above ASCII, so it takes both
# with or without the fold; the negated form is what a fold would break.
assert_true Regexp.new("[\\W]", Regexp::IGNORECASE).match?(ch)
assert_false Regexp.new("[^\\W]", Regexp::IGNORECASE).match?(ch)
# Outside a class the shorthand never folded, and still does not.
assert_false Regexp.new("\\w", Regexp::IGNORECASE).match?(ch)
assert_true Regexp.new("\\W", Regexp::IGNORECASE).match?(ch)
# /i does not move either in or out of [:word:], whatever the set holds
# (this gem's is the ASCII word characters; CRuby's holds every Unicode
# word character, these two among them).
assert_equal Regexp.new("[[:word:]]").match?(ch),
Regexp.new("[[:word:]]", Regexp::IGNORECASE).match?(ch)
assert_equal Regexp.new("[^[:word:]]").match?(ch),
Regexp.new("[^[:word:]]", Regexp::IGNORECASE).match?(ch)
end
# A letter written out beside the shorthand folds as it does on its own:
# the class then holds it by name as well as through `\w`, and the name is
# what folds. Either case of the letter, in either order, and a range too.
assert_true Regexp.new("[\\ws]", Regexp::IGNORECASE).match?(long_s)
assert_true Regexp.new("[\\wS]", Regexp::IGNORECASE).match?(long_s)
assert_true Regexp.new("[k\\w]", Regexp::IGNORECASE).match?(kelvin)
assert_true Regexp.new("[\\wa-z]", Regexp::IGNORECASE).match?(long_s)
assert_false Regexp.new("[^\\ws]", Regexp::IGNORECASE).match?(long_s)
# Naming one letter folds that letter and no other.
assert_false Regexp.new("[\\wk]", Regexp::IGNORECASE).match?(long_s)
assert_false Regexp.new("[\\ws]", Regexp::IGNORECASE).match?(kelvin)
# The other direction is untouched: a member above ASCII still folds to the
# letter, and reaches the letter's other case through it.
assert_true Regexp.new("[\\w#{long_s}]", Regexp::IGNORECASE).match?("S")
assert_false Regexp.new("[^\\w#{long_s}]", Regexp::IGNORECASE).match?("S")
# The other POSIX brackets fold like a written range: [:lower:] holds `k`,
# so under /i it reaches U+212A, and [^[:alpha:]] rejects U+017F.
assert_true Regexp.new("[[:lower:]]", Regexp::IGNORECASE).match?(kelvin)
assert_true Regexp.new("[[:alpha:]]", Regexp::IGNORECASE).match?(long_s)
assert_false Regexp.new("[^[:alpha:]]", Regexp::IGNORECASE).match?(long_s)
end

assert("Regexp - repetition {n,m}") do
assert_equal "aaa", Regexp.new("a{3}").match("aaaa")[0]
assert_equal "aa", Regexp.new("a{2,3}").match("aa")[0]
Expand Down
Loading