Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions mrbgems/mruby-regexp/mrblib/string_regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -515,24 +515,38 @@ def index(*args)
md && md.begin(0)
end

# The last match that starts at or before `limit`, or nil, with the match
# globals left describing it. `limit` is a byte offset when `bytes` is
# true and a character offset otherwise, which is the whole of the
# difference between `byterindex` and `rindex`.
# The last match that starts at or before `limit`, a byte offset, or nil,
# with the match globals left describing it.
#
# The engine searches forward only, so this walks the subject from the
# start and keeps the last match that qualifies: linear in the number of
# positions a match starts at, where the backward search CRuby hands to
# Onig is not. Each step resumes one character past the match start and
# not at the match end, which is what keeps overlapping matches in view:
# Onig is not. Each step resumes one byte past the match start and not at
# the match end, which is what keeps overlapping matches in view:
# `"aaa".rindex(/aa/)` is 1, where resuming at the end would answer 0.
def __regexp_rsearch(pattern, limit, bytes)
#
# The walk is in byte space so that its length is what it looks like. A
# character offset is not a place the subject can be read from: every one
# handed to `Regexp.__search` is counted out from the start of the subject
# again, and every one read back off a match with `begin` is counted the
# same way, so a walk that speaks characters pays for the whole subject on
# every turn and takes quadratic time on a multibyte one. Nothing is given
# up by leaving them: a byte inside a character is not a position a match
# can start at, and the engine steps over one on its own rather than seed a
# match attempt there, so `+ 1` reaches the next character by itself.
#
# `Regexp.__byte_search` does not range check its position, so the walk
# stops itself at the end of the subject. An empty match there is the one
# that reaches it: it leaves `pos` one past the last byte.
def __regexp_rsearch(pattern, limit)
found = nil
pos = 0
while (md = Regexp.__search(pattern, self, pos))
break if (bytes ? md.__byte_begin(0) : md.begin(0)) > limit
size = self.bytesize
while pos <= size && (md = Regexp.__byte_search(pattern, self, pos))
start = md.__byte_begin(0)
break if start > limit
found = md
pos = md.begin(0) + 1
pos = start + 1
end
# The loop leaves behind whatever its last search published: the clear a
# failed one does, or a match past `limit` that is not the answer. Both
Expand Down Expand Up @@ -568,7 +582,14 @@ def rindex(*args)
pos = len
end
end
md = __regexp_rsearch(args[0], pos, false)
# The walk reads the subject by byte, so the character position it is to
# stop at has to be read as one here. A position at the end of the
# subject is the end of its bytes and needs no reading, which is the form
# `rindex` is called in when it is called with one argument at all; only
# a position named by the caller is measured, once, where the walk would
# otherwise have measured one on every turn.
byte_pos = pos == len ? self.bytesize : self[0, pos].bytesize
md = __regexp_rsearch(args[0], byte_pos)
md && md.begin(0)
end

Expand Down Expand Up @@ -614,7 +635,7 @@ def byterindex(*args)
pos = len
end
end
md = __regexp_rsearch(args[0], pos, true)
md = __regexp_rsearch(args[0], pos)
md && md.__byte_begin(0)
end

Expand All @@ -636,7 +657,7 @@ def rpartition(sep)
return __rpartition(sep) unless Regexp === sep
# The last match anywhere in the subject, so the limit is its end and the
# walk below never stops early.
md = __regexp_rsearch(sep, self.length, false)
md = __regexp_rsearch(sep, self.bytesize)
# No match puts the whole subject in the tail, which is the row this
# method is most often got wrong on.
return ["", "", self.byteslice(0, self.bytesize)] unless md
Expand Down
34 changes: 34 additions & 0 deletions mrbgems/mruby-regexp/test/string_index.rb
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,40 @@ def re.is_a?(klass); false; end
assert_nil "hello".rindex(/l/, -10)
end

assert("String#rindex bounds the match start in characters") do
# The position `rindex` takes is a character offset and the one
# `byterindex` takes is a byte offset. On a single-byte subject the two
# name the same place, so only a multibyte one says which is being read.
skip unless __ENCODING__ == "UTF-8"
str = "あいうあいう" # 6 characters, 18 bytes

assert_equal 4, str.rindex(/い/)
assert_equal 12, str.byterindex(/い/)

# 1 as a character offset is the second character, which the first `い` is;
# as a byte offset it is inside the first character and reaches no match at
# all
assert_equal 1, str.rindex(/い/, 1)
assert_nil str.rindex(/い/, 0)
assert_equal 3, str.byterindex(/い/, 3)
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# 4 as a character offset reaches the second `い`, where the same number of
# bytes is still short of the first
assert_equal 4, str.rindex(/い/, 4)
assert_equal 1, str.rindex(/い/, 3)

# a negative position counts back in the same space it counts forward in
assert_equal 4, str.rindex(/い/, -1)
assert_equal 1, str.rindex(/い/, -3)

# overlapping matches stay in view across a multibyte character, where a
# walk that resumed at the match end would answer 0
assert_equal 1, "あああ".rindex(/ああ/)
assert_equal 3, "あああ".byterindex(/ああ/)
assert_equal ["あ", "ああ", ""], "あああ".rpartition(/ああ/)
assert_equal ["あい", "うあ", "いう"], str.rpartition(/うあ/)
end

assert("String#index and String#rindex with regexp set the match globals") do
assert_equal 1, "abc".index(/(b)/)
assert_equal "b", $1
Expand Down
Loading