Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions mrbgems/mruby-regexp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ simulation) with backtracking fallback.
Outside a character class the list form is a sequence rather than one
atom, so a quantifier after it repeats the last codepoint only:
`/\u{61 62}+/` is `ab+`. Inside a class every codepoint is a member of
its own, and the last one can still open a range: `/[\u{61 62}-z]/` is
`a` plus `b-z`.
its own, and the one next to a `-` still opens or closes a range: the
last of the list before it and the first after it, so `/[\u{61 62}-z]/`
is `a` plus `b-z` and `/[a-\u{63 7a}]/` is `a-c` plus `z`. A range
written backwards, `[b-a]` or `[b-\u{61 63}]`, raises `RegexpError` as
in CRuby.

### Anchors

Expand Down
42 changes: 30 additions & 12 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,9 @@ class_named_cp(re_compiler *c, re_charclass *cc, uint32_t cp, mrb_bool *is_byte)
advances c->p. *is_byte says which of the two the value is: TRUE for a
byte at or above 0x80 that starts no whole character, FALSE for ASCII, for
a decoded codepoint and for `\u`, which names a codepoint outright.
closes_range says the atom follows a `-`, which matters to a `\u{...}`
list alone: the codepoint next to the `-` is the range end, and the rest
of the list are members.

The question is the one the literal path already answers: emit_char_folded()
decodes and stands aside when the decode consumed one byte, so `\xB5` and a
Expand All @@ -680,7 +683,7 @@ class_named_cp(re_compiler *c, re_charclass *cc, uint32_t cp, mrb_bool *is_byte)
the pattern holds. A byte and a codepoint of the same number are different
members, which is what the tag on the stored value records. */
static uint32_t
read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte)
read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte, mrb_bool closes_range)
{
*is_byte = FALSE;
if (peek(c) == '\\') {
Expand All @@ -690,9 +693,20 @@ read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte)
mrb_bool more;
uint32_t cp = unicode_escape_first(c, &more);
uint32_t nx;
/* Every codepoint of a `\u{...}` list is a member of its own. All but
the last join the class here; the last is returned, so it can open a
range as any other atom would: `[\u{61 62}-z]` is `a` plus `b-z`. */
/* Every codepoint of a `\u{...}` list is a member of its own, apart
from the one next to a `-`, which is a range end as any other atom
would be. That is the last of the list before the `-` and the first
after it: `[\u{61 62}-z]` is `a` plus `b-z`, and `[a-\u{63 7a}]` is
`a-c` plus `z`. The rest join the class here. */
if (closes_range) {
uint32_t end = class_named_cp(c, cc, cp, is_byte);
while (unicode_escape_next(c, &more, &nx)) {
mrb_bool member_byte = FALSE;
uint32_t member = class_named_cp(c, cc, nx, &member_byte);
class_add_member(c, cc, member, member_byte);
}
return end;
}
while (unicode_escape_next(c, &more, &nx)) {
mrb_bool member_byte = FALSE;
uint32_t member = class_named_cp(c, cc, cp, &member_byte);
Expand Down Expand Up @@ -787,30 +801,34 @@ compile_charclass(re_compiler *c)
}

mrb_bool cp_byte;
uint32_t cp = read_class_atom(c, cc, &cp_byte);
uint32_t cp = read_class_atom(c, cc, &cp_byte, FALSE);

/* check for range a-z (or U+xxxx-U+yyyy) */
if (peek(c) == '-' && c->p + 1 < c->src_end && c->p[1] != ']') {
next_char(c); /* skip '-' */
mrb_bool hi_byte;
uint32_t hi = read_class_atom(c, cc, &hi_byte);
uint32_t hi = read_class_atom(c, cc, &hi_byte, TRUE);
/* An endpoint at or above 128 is a byte or a character, and a span from
one to the other names neither: [\x80-µ] would run from a byte to a
codepoint. ASCII belongs to both, so it pairs with either. */
if (cp >= 128 && hi >= 128 && cp_byte != hi_byte) {
compile_error(c, "character class range mixes a byte and a character");
}
/* A range written backwards holds nothing, and CRuby reports it rather
than compiling a class that silently lacks the span, or in the
negated form admits everything: [b-a] and [^b-a] both raise. The
numbers compare, since the check above leaves no byte paired with a
character and ASCII sits below either. */
if (cp > hi) compile_error(c, "empty range in char class");
/* A range that straddles the ASCII boundary is split in two: the
bitmap takes the half below 128 and the codepoint list the rest.
Neither half can hold the other, and class_match() picks the side
to read from the codepoint alone, so a span left whole in the
codepoint list is unreachable below 128. */
if (cp <= hi) {
if (cp < 128) class_set_range(cc, (uint8_t)cp, (uint8_t)(hi < 128 ? hi : 127));
if (hi >= 128) {
uint32_t tag = hi_byte ? RE_CLASS_BYTE : 0;
class_add_range(c, cc, tag | (cp < 128 ? 128 : cp), tag | hi);
}
if (cp < 128) class_set_range(cc, (uint8_t)cp, (uint8_t)(hi < 128 ? hi : 127));
if (hi >= 128) {
uint32_t tag = hi_byte ? RE_CLASS_BYTE : 0;
class_add_range(c, cc, tag | (cp < 128 ? 128 : cp), tag | hi);
}
}
else {
Expand Down
18 changes: 18 additions & 0 deletions mrbgems/mruby-regexp/test/regexp_syntax.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,24 @@
assert_equal "abc", md[0]
end

assert("Regexp - reversed character class range") do
# A range written backwards holds nothing. It used to compile to a class
# that silently lacked the span, or in the negated form admitted every
# character; CRuby raises for either.
assert_raise_with_message(RegexpError, "empty range in char class: /[b-a]/") do
Regexp.new("[b-a]")
end
assert_raise_with_message(RegexpError, "empty range in char class: /[^b-a]/") do
Regexp.new("[^b-a]")
end
assert_raise(RegexpError) { Regexp.new("[xz-ay]") }
# a range of one is not empty
assert_equal ["a"], "abc".scan(/[a-a]/)
# a '-' at either edge is a member, not a range
assert_equal ["-", "a"], "-ab".scan(/[-a]/)
assert_equal ["a", "-"], "abc-".scan(/[a-]/)
end

assert("Regexp - POSIX bracket classes") do
# ASCII semantics, like this gem's \w/\d shorthands.
assert_equal "abc", "123abc456".match(/[[:alpha:]]+/)[0]
Expand Down
53 changes: 50 additions & 3 deletions mrbgems/mruby-regexp/test/regexp_utf8.rb
Original file line number Diff line number Diff line change
Expand Up @@ -425,13 +425,58 @@
assert_nil (/[\u{3042}-\u{3044}]/ =~ "\xe3\x81\x85")
assert_equal 0, (/[a-\u{7a}]/ =~ "q")

# every codepoint of a list is a member of its own, and the last one can
# still open a range
# every codepoint of a list is a member of its own, and the one next to a
# '-' still opens or closes a range: the last before it, the first after it
assert_equal ["a", "b"], "abc".scan(/[\u{61 62}]/)
assert_equal ["a", "b", "c"], "abc-".scan(/[\u{61 62}-z]/)
assert_equal ["a", "b", "c", "z"], "abcdz-".scan(/[a-\u{63 7a}]/)
assert_equal ["c"], "abc".scan(/[^\u{61 62}]/)
end

assert("Regexp - a \\u list closing a character class range") do
# The codepoint next to the '-' is the range end and the rest of the list
# are members. The end used to be the last codepoint of the list, so the
# class held a different set from CRuby's in either direction: `a-z` plus
# `c` for /[a-\u{63 7a}]/, and `z` alone for /[a-\u{7a 41}]/, whose range
# ran from `a` down to `A` and was dropped.
assert_equal ["a", "b", "c", "z"], "abcdz".scan(/[a-\u{63 7a}]/)
assert_equal ["a", "A", "z"], "aAzB".scan(/[a-\u{7a 41}]/)
assert_equal ["a", "b", "c", "d"], "abcde".scan(/[\u{61 62}-\u{63 64}]/)
# which is also the codepoint a reversed range is reported for
assert_raise_with_message(RegexpError, "empty range in char class: /[b-\\u{61 63}]/") do
Regexp.new("[b-\\u{61 63}]")
end
if __ENCODING__ == "UTF-8"
assert_equal ["あ", "ぃ", "う"], "あぃぅう".scan(/[\u{3042}-\u{3044 3046}]/)
assert_equal ["あ", "ぅ"], "あぃぅ".scan(/[\u{3044}-\u{3046 3042}]/)
assert_raise(RegexpError) { Regexp.new("[\\u{3044}-\\u{3042 3046}]") }
end
end

assert("Regexp - reversed character class range through \\u") do
# `\u` can write a range backwards without the letters showing it. Such a
# range holds nothing, and it used to be dropped without a word: a positive
# class lacked the span and a negated one admitted everything.
assert_raise_with_message(RegexpError, "empty range in char class: /[\\u{62}-\\u{61}]/") do
Regexp.new("[\\u{62}-\\u{61}]")
end
assert_raise(RegexpError) { Regexp.new("[^\\u{62}-\\u{61}]") }
# the last codepoint of a list opens the range, so it is the one compared
assert_raise(RegexpError) { Regexp.new("[\\u{62 63}-a]") }
assert_equal ["a", "b"], "abc".scan(/[\u{62 61}-a]/)
# a range of one codepoint is not empty
assert_equal ["a"], "abc".scan(/[\u{61}-\u{61}]/)
assert_equal ["a"], "abc".scan(/[a-\u{61}]/)
# A range between two characters above ASCII compares their codepoints. On
# a build reading a String by byte the ends are the last bytes of their
# spellings, whose order is not the codepoints' order.
if __ENCODING__ == "UTF-8"
assert_raise(RegexpError) { Regexp.new("[\\u{3044}-\\u{3042}]") }
assert_raise(RegexpError) { Regexp.new("[\\u{100}-\\u{FF}]") }
assert_equal 0, (/[\u{3042}-\u{3042}]/ =~ "あ")
end
end

assert("Regexp - a \\u escape in a class names what spelling it out names") do
# A class member is one character, and on a build whose characters are single
# bytes a character above ASCII is not one: `[Ā]` holds the two bytes that
Expand All @@ -448,9 +493,11 @@
# is what the written out spelling answers too.
assert_true named.match?("\u{100}")
# An ASCII codepoint is a member of its own on either build, and a list still
# gives every codepoint a membership with the last one able to open a range.
# gives every codepoint a membership with the one next to a '-' able to open
# or close a range.
assert_equal ["a", "b"], "abc".scan(/[\u{61 62}]/)
assert_equal ["a", "b", "c"], "abc-".scan(/[\u{61 62}-z]/)
assert_equal ["a", "b", "c", "z"], "abcdz-".scan(/[a-\u{63 7a}]/)
end

assert("Regexp - malformed \\u escapes") do
Expand Down
Loading