Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions include/mruby/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,12 @@ mrb_int mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi);
definition in string.c for what it reads and what it leaves behind. */
mrb_bool mrb_str_valid_encoding_p(mrb_state *mrb, mrb_value str);

/* Raise IndexError when `pos` lands inside a character of `str`, and return
otherwise. See the definition in string.c for which offsets are positions
the string has; a build without MRB_UTF8_STRING has one per byte, so this
is a no-op there. */
void mrb_str_check_byte_pos(mrb_state *mrb, mrb_value str, mrb_int pos);

/* Write the UTF-8 spelling of a codepoint into a buffer of at least four
bytes, and return how many it took (1-4), or 0 for a value that spells no
character. What counts as one, and why a surrogate does spell one here
Expand Down
13 changes: 9 additions & 4 deletions mrbgems/mruby-regexp/mrblib/string_regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -637,11 +637,13 @@ def byteindex(*args)
end
# `__byte_search` takes the position as given and does not range check
# it, where `Regexp.__search` answers nil for one outside the subject.
# Both ends are a miss here, as they are for `mrb_str_byteindex_m()`. An
# offset that lands inside a character is not an error: the C method does
# not check for one either, and on a build without MRB_UTF8_STRING there
# is nothing to check.
# Both ends are a miss here, as they are for `mrb_str_byteindex_m()`.
return Regexp.__search(args[0], nil) if pos < 0 || pos > len
# An offset that lands inside a character names no position the subject
# has, and the C method refuses one. It is asked after the range test,
# where the C method asks it too, so an offset outside the subject stays a
# miss rather than becoming an error.
Regexp.__check_byte_pos(self, pos)
md = Regexp.__byte_search(args[0], self, pos)
md && md.__byte_begin(0)
end
Expand All @@ -664,6 +666,9 @@ def byterindex(*args)
pos = len
end
end
# As in `byteindex` above, and after the same clamp: a position past the
# end of the subject has already been read as its end, which is a boundary.
Regexp.__check_byte_pos(self, pos)
md = __regexp_rsearch(args[0], pos)
md && md.__byte_begin(0)
end
Expand Down
27 changes: 27 additions & 0 deletions mrbgems/mruby-regexp/src/regexp.c
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,32 @@ regexp_check_encoding(mrb_state *mrb, mrb_value self)
return mrb_nil_value();
}

/*
* Regexp.__check_byte_pos(str, pos)
*
* Internal: `mrb_str_check_byte_pos()`, for the byte searches this gem takes
* over. `String#byteindex` and `String#byterindex` reach the C methods that
* ask this for every argument form but a Regexp, and the answer a search gives
* may not turn on which of the two it was reached through.
*/
static mrb_value
regexp_check_byte_pos(mrb_state *mrb, mrb_value self)
{
(void)self;
mrb_value str;
mrb_int pos;
mrb_get_args(mrb, "Si", &str, &pos);
/* The mrblib callers read the position against the byte length and answer
both ends themselves before asking this, so one outside the subject
reaches here only from a direct call. There is no boundary to ask about
at a position the subject does not have, and mrb_str_check_byte_pos()
would read behind RSTRING_PTR(str) looking for one. A backstop, as
check_regexp_arg() below is for a pattern. */
if (pos < 0 || pos > RSTRING_LEN(str)) return mrb_nil_value();
mrb_str_check_byte_pos(mrb, str, pos);
return mrb_nil_value();
}

/* Publish `obj` and the thirteen names derived from its offsets, the
counterpart of clear_match_globals(). Kept apart from create_matchdata() so
that an existing MatchData can be republished without rebuilding it. */
Expand Down Expand Up @@ -1449,6 +1475,7 @@ mrb_mruby_regexp_gem_init(mrb_state *mrb)
mrb_define_class_method(mrb, re, "__binary_string?", regexp_binary_string_p, MRB_ARGS_REQ(1));
mrb_define_class_method(mrb, re, "__check_encoding", regexp_check_encoding, MRB_ARGS_REQ(1));
mrb_define_class_method(mrb, re, "__check_pattern", regexp_check_pattern, MRB_ARGS_REQ(1));
mrb_define_class_method(mrb, re, "__check_byte_pos", regexp_check_byte_pos, MRB_ARGS_REQ(2));
mrb_define_class_method(mrb, re, "__search", regexp_s_search, MRB_ARGS_ARG(2, 2));
mrb_define_class_method(mrb, re, "__byte_search", regexp_s_byte_search, MRB_ARGS_ARG(2, 3));
mrb_define_class_method(mrb, re, "__search_p", regexp_s_search_p, MRB_ARGS_ARG(2, 1));
Expand Down
22 changes: 22 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,28 @@
assert_true Regexp.new(Regexp.escape("a # b"), Regexp::EXTENDED).match?("a # b")
end

assert("Regexp.__check_byte_pos passes a position the subject does not have") do
# `String#byteindex` and `String#byterindex` read the position against the
# byte length and answer both ends themselves before asking this, so one
# outside the subject arrives only from a direct call. A position the subject
# does not have sits on no boundary, and looking for one would read behind
# the subject.
s = "あいう"
assert_nil Regexp.__check_byte_pos(s, -1)
assert_nil Regexp.__check_byte_pos(s, -1000000)
assert_nil Regexp.__check_byte_pos(s, s.bytesize + 1)
assert_nil Regexp.__check_byte_pos(s, 1000000)

# the ones it does have are still asked
assert_nil Regexp.__check_byte_pos(s, 0)
assert_nil Regexp.__check_byte_pos(s, s.bytesize)
if __ENCODING__ == "UTF-8"
assert_raise(IndexError) { Regexp.__check_byte_pos(s, 1) }
else
assert_nil Regexp.__check_byte_pos(s, 1)
end
end

assert("Regexp#inspect") do
re = Regexp.new("abc", Regexp::IGNORECASE)
assert_equal "/abc/i", re.inspect
Expand Down
50 changes: 50 additions & 0 deletions mrbgems/mruby-regexp/test/string_index.rb
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,56 @@ def fake.match(str, pos = 0); raise "must not be called"; end
end
end

assert("String#byteindex and String#byterindex take an offset inside a character") do
# A byte offset that lands inside a character names no position the string
# has, and a byte search refuses one rather than search from the middle of a
# character, whichever form of it the search was reached through.
skip unless __ENCODING__ == "UTF-8"
str = "あいうあいう" # 6 characters, 18 bytes

assert_raise(IndexError) { str.byteindex("い", 1) }
assert_raise(IndexError) { str.byterindex("い", 1) }
assert_raise(IndexError) { str.byteindex(/い/, 1) }
assert_raise(IndexError) { str.byterindex(/い/, 1) }

# a negative offset is read against the byte length first, so where it lands
# is the question asked of it too
assert_raise(IndexError) { "あ".byteindex("x", -1) }
assert_raise(IndexError) { "あ".byterindex("x", -1) }
assert_raise(IndexError) { "あ".byteindex(/x/, -1) }
assert_raise(IndexError) { "あ".byterindex(/x/, -1) }

# an offset on a boundary is a position either way, and the two forms agree
assert_equal 3, str.byteindex("い", 3)
assert_equal 3, str.byteindex(/い/, 3)
assert_equal 3, str.byterindex("い", 3)
assert_equal 3, str.byterindex(/い/, 3)

# so is every offset of a string that indexes by byte, whatever its bytes
# spell, and both forms take one
bin = "あ".b
assert_nil bin.byteindex("x", 1)
assert_nil bin.byteindex(/x/, 1)
assert_nil bin.byterindex("x", 1)
assert_nil bin.byterindex(/x/, 1)

# an offset before the string is a miss before it is a position, for either
# form. The pattern is one the subject holds, so that a nil is the offset
# being answered rather than the search coming up empty on its own.
assert_nil "あ".byteindex("あ", -9)
assert_nil "あ".byteindex(/あ/, -9)
assert_nil "あ".byterindex("あ", -9)
assert_nil "あ".byterindex(/あ/, -9)

# past the far end the two methods part company, as they do for a String:
# `byteindex` misses, and `byterindex` reads the offset as the end it already
# searches back from
assert_nil "あ".byteindex("あ", 9)
assert_nil "あ".byteindex(/あ/, 9)
assert_equal 0, "あ".byterindex("あ", 9)
assert_equal 0, "あ".byterindex(/あ/, 9)
end

assert("String#byteindex and String#byterindex with regexp set the match globals") do
assert_equal 1, "abc".byteindex(/(b)/)
assert_equal "b", $1
Expand Down
8 changes: 4 additions & 4 deletions src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -2451,8 +2451,8 @@ mrb_str_include(mrb_state *mrb, mrb_value self)
one. A byte-indexed string has a position per byte, so every offset is one.
The boundaries are the ones String#length counts over, which is why a byte
no lead byte reaches is one of them. */
static void
str_check_byte_pos(mrb_state *mrb, mrb_value str, mrb_int pos)
void
mrb_str_check_byte_pos(mrb_state *mrb, mrb_value str, mrb_int pos)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
{
#ifdef MRB_UTF8_STRING
struct RString *s = mrb_str_ptr(str);
Expand Down Expand Up @@ -2482,7 +2482,7 @@ mrb_str_byteindex_m(mrb_state *mrb, mrb_value str)
}
}
if (pos > RSTRING_LEN(str)) return mrb_nil_value();
str_check_byte_pos(mrb, str, pos);
mrb_str_check_byte_pos(mrb, str, pos);
/* see str_index_str() */
if (!mrb_str_valid_encoding_p(mrb, sub)) return mrb_nil_value();
pos = str_index_str(mrb, str, sub, pos);
Expand Down Expand Up @@ -2780,7 +2780,7 @@ mrb_str_byterindex_m(mrb_state *mrb, mrb_value str)
}
if (pos > len) pos = len;
}
str_check_byte_pos(mrb, str, pos);
mrb_str_check_byte_pos(mrb, str, pos);
/* see str_index_str() */
if (!mrb_str_valid_encoding_p(mrb, sub)) return mrb_nil_value();
pos = str_byterindex(str, sub, pos);
Expand Down
Loading