Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 36 additions & 13 deletions mrbgems/mruby-regexp/src/re_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,12 @@ read_class_atom(re_compiler *c)
{
if (peek(c) == '\\') {
next_char(c);
return (uint32_t)parse_escape(c);
/* A backslash before a multibyte character has no escape meaning, so let
the decode below read the whole codepoint: [\Ā] is [Ā]. parse_escape()
returns one byte, which left the continuation byte as a class atom of
its own. A trailing backslash (peek < 0) still reaches parse_escape(),
which reports it. */
if (peek(c) < 0xC0) return (uint32_t)parse_escape(c);
}
uint8_t b = (uint8_t)*c->p;
if (b < 0xC0) {
Expand Down Expand Up @@ -643,6 +648,25 @@ parse_inline_flags(re_compiler *c, uint32_t base)
return (base | on) & ~off;
}

/* Emit every byte of the character whose lead byte `ch` was just consumed, so
the whole character is one atom. Leaving the continuation bytes to the parse
loop made each of them an atom of its own, and a quantifier binds to the last
atom emitted: /Ā+/ compiled as \xC4(\x80)+ and matched one Ā in "ĀĀ". An
invalid lead byte has a charlen of 1 and still emits alone. Every atom that
consumes a character has to emit all of its bytes before returning, since
what compile_quantified() repeats is the bytes that atom emitted. */
static void
emit_char_bytes(re_compiler *c, int ch)
{
int len = mrb_re_utf8_charlen(c->p - 1, c->src_end);
emit(c, RE_CHAR, (uint8_t)ch, 0);
for (int i = 1; i < len; i++) {
int b = next_char(c);
if (b < 0) break;
emit(c, RE_CHAR, (uint8_t)b, 0);
}
}

/* Compile a single atom (character, class, group, etc.) */
static void
compile_atom(re_compiler *c)
Expand Down Expand Up @@ -916,6 +940,16 @@ compile_atom(re_compiler *c)
emit(c, RE_BACKREF, (uint8_t)group, (c->flags & RE_FLAG_IGNORECASE) ? 1 : 0);
c->has_backref = TRUE;
}
else if (ch >= 0xC0) {
/* A backslash before a multibyte character has no escape meaning: \Ā is
Ā. parse_escape() returns one byte, which left the continuation bytes
to the parse loop as atoms of their own, so emit the whole character
here as the unescaped spelling does. The dispatch has to happen before
parse_escape() reads the letter, since \xNN and octal \NNN name a byte
rather than a character. */
next_char(c);
emit_char_bytes(c, ch);
}
else {
ch = parse_escape(c);
if (c->flags & RE_FLAG_IGNORECASE) {
Expand Down Expand Up @@ -978,18 +1012,7 @@ compile_atom(re_compiler *c)
}
}
if (ch >= 128) {
/* Emit every byte of a multibyte character here, so the whole character
is one atom. Leaving the continuation bytes to the parse loop made
each of them an atom of its own, and a quantifier binds to the last
atom emitted: /Ā+/ compiled as \xC4(\x80)+ and matched one Ā in "ĀĀ".
An invalid lead byte has a charlen of 1 and still emits alone. */
int len = mrb_re_utf8_charlen(c->p - 1, c->src_end);
emit(c, RE_CHAR, (uint8_t)ch, 0);
for (int i = 1; i < len; i++) {
int b = next_char(c);
if (b < 0) break;
emit(c, RE_CHAR, (uint8_t)b, 0);
}
emit_char_bytes(c, ch);
break;
}
emit(c, RE_CHAR, (uint8_t)ch, 0);
Expand Down
30 changes: 30 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,36 @@
assert_equal 0, "z".match(/Ā?/)[0].bytesize
end

assert("Regexp - quantifier on an escaped multibyte literal") do
# A backslash before a character with no escape meaning is just that
# character, so \Ā has to be one atom exactly like Ā. The escape path used
# to emit the lead byte alone and leave the continuation byte to the parse
# loop, so the quantifier bound to that byte instead.
# The /.../ spelling cannot show this, because the lexer drops the backslash
# before the gem sees the pattern: /\Ā/.source is the two bytes of Ā alone.
# A pattern built at runtime arrives through Regexp.new with the backslash
# still in it.
assert_equal 4, Regexp.new("\\Ā+").match("ĀĀ")[0].bytesize
assert_equal 6, Regexp.new("\\ĀĀĀ").match("ĀĀĀ")[0].bytesize
assert_true Regexp.new("\\Ā{2}").match?("ĀĀ")
assert_false Regexp.new("\\Ā{2}").match?("Ā")
assert_equal 6, Regexp.new("\\日+").match("日日")[0].bytesize
assert_equal 8, Regexp.new("\\𝕏+").match("𝕏𝕏")[0].bytesize
assert_equal 5, Regexp.new("a\\Ā+").match("aĀĀ")[0].bytesize
assert_equal 2, Regexp.new("\\Ā+?").match("ĀĀ")[0].bytesize
# Inside [...] the same escape has to read as one codepoint, or the class
# holds the lead byte and the continuation byte as two wrong members.
assert_true Regexp.new("[\\Ā]").match?("Ā")
assert_false Regexp.new("[\\Ā]").match?("Ä")
assert_true Regexp.new("[\\Ā-\\ā]").match?("ā")
assert_false Regexp.new("[\\Ā-\\ā]").match?("Ă")
# A raw byte escape names a byte rather than a character, so it keeps taking
# the parse_escape path and the quantifier binds to that one byte. CRuby
# joins byte escapes that spell a valid UTF-8 sequence into one character
# and matches four bytes here; closing that gap is a separate change.
assert_equal 2, Regexp.new("\\xC4\\x80+").match("ĀĀ")[0].bytesize
end

assert("Regexp - quantifier on an invalid multibyte literal") do
# A byte above 127 is one atom only while it starts a whole character. The
# sequences below never complete one, so each byte stands alone and the
Expand Down
Loading