mruby · takumin · Aug 12, 2026
@@ -228,6 +228,28 @@ re_binary_string_p(mrb_value str)
   return RSTR_BINARY_P(RSTRING(str));
 }
 
+/* CRuby refuses a search whose subject holds a byte that spells no character,
+   and mruby answers for it. Refuse it here too, so that a program moved from
+   one to the other is told about the subject rather than handed a result the
+   other would not have produced.
+
+   A binary string is exempt because it is indexed by byte throughout, so its
+   bytes make no claim that could be broken.
+
+   The check walks the whole subject, so every entry point below runs it on the
+   subject it is handed and the C loops over a subject run it before the first
+   turn. Two searches are driven from mrblib once per match rather than once per
+   call, `__byte_search` and the `__search` the backward search steps, and they
+   check too: core remembers a string it has read as valid UTF-8, so every turn
+   after the first costs a flag test and not a walk. */
+static void
+re_check_encoding(mrb_state *mrb, mrb_value str)
+{
+  if (!mrb_str_valid_encoding_p(mrb, str)) {
+    mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid byte sequence in UTF-8");
+  }
+}
+
 static mrb_value
 regexp_binary_string_p(mrb_state *mrb, mrb_value self)
 {
@@ -362,6 +384,7 @@ regexp_match(mrb_state *mrb, mrb_value self)
     return mrb_nil_value();
   }
 
+  re_check_encoding(mrb, str);
   md = exec_match(mrb, self, str, pos);
   if (!mrb_nil_p(md) && !mrb_nil_p(block)) {
     return mrb_yield(mrb, block, md);
@@ -410,6 +433,7 @@ regexp_s_search(mrb_state *mrb, mrb_value klass)
     clear_match_globals(mrb);
     return mrb_nil_value();
   }
+  re_check_encoding(mrb, str);
   return exec_match(mrb, re, str, pos);
 }
 
@@ -419,7 +443,8 @@ regexp_s_search(mrb_state *mrb, mrb_value klass)
  * Internal: the byte-offset search the mrblib loops of `gsub`, `split` and
  * `byteindex` drive themselves. No position normalization, because the
  * callers already work in byte space, and no operand conversion, because
- * they always pass a String.
+ * they always pass a String. The subject is the one the loop holds fixed, so
+ * the check reads the flag core left on it after the first turn.
  */
 static mrb_value
 regexp_s_byte_search(mrb_state *mrb, mrb_value klass)
@@ -429,6 +454,7 @@ regexp_s_byte_search(mrb_state *mrb, mrb_value klass)
 
   mrb_get_args(mrb, "oS|i", &re, &str, &pos);
   check_regexp_arg(mrb, re);
+  re_check_encoding(mrb, str);
   return exec_match(mrb, re, str, pos);
 }
 
@@ -445,6 +471,7 @@ exec_match_p(mrb_state *mrb, mrb_value re, mrb_value str, mrb_int pos)
 
   mrb_regexp_pattern *pat = DATA_GET_PTR(mrb, re, &regexp_type, mrb_regexp_pattern);
   if (!pat) mrb_raise(mrb, E_ARGUMENT_ERROR, "uninitialized Regexp");
+  re_check_encoding(mrb, str);
 
   int ncap = mrb_re_exec(mrb, pat, RSTRING_PTR(str), RSTRING_LEN(str), pos, NULL, 0,
                          re_binary_string_p(str));
@@ -493,6 +520,7 @@ regexp_match_op(mrb_state *mrb, mrb_value self)
     return mrb_nil_value();
   }
   str = match_operand(mrb, str);
+  re_check_encoding(mrb, str);
 
   mrb_value md = exec_match(mrb, self, str, 0);
   if (mrb_nil_p(md)) return mrb_nil_value();
@@ -516,6 +544,7 @@ regexp_case_match(mrb_state *mrb, mrb_value self)
 
   pat = DATA_GET_PTR(mrb, self, &regexp_type, mrb_regexp_pattern);
   if (!pat) return mrb_false_value();
+  re_check_encoding(mrb, str);
 
   md = exec_match(mrb, self, str, 0);
   return mrb_bool_value(!mrb_nil_p(md));
@@ -1082,6 +1111,7 @@ regexp_s_gsub_str(mrb_state *mrb, mrb_value klass)
 
   mrb_regexp_pattern *pat = DATA_GET_PTR(mrb, re, &regexp_type, mrb_regexp_pattern);
   if (!pat) mrb_raise(mrb, E_ARGUMENT_ERROR, "uninitialized Regexp");
+  re_check_encoding(mrb, str);
 
   const char *s = RSTRING_PTR(str);
   mrb_int slen = RSTRING_LEN(str);
@@ -1172,6 +1202,7 @@ regexp_s_sub_str(mrb_state *mrb, mrb_value klass)
 
   mrb_regexp_pattern *pat = DATA_GET_PTR(mrb, re, &regexp_type, mrb_regexp_pattern);
   if (!pat) mrb_raise(mrb, E_ARGUMENT_ERROR, "uninitialized Regexp");
+  re_check_encoding(mrb, str);
 
   const char *s = RSTRING_PTR(str);
   mrb_int slen = RSTRING_LEN(str);
@@ -1226,6 +1257,7 @@ regexp_s_scan(mrb_state *mrb, mrb_value klass)
 
   mrb_regexp_pattern *pat = DATA_GET_PTR(mrb, re, &regexp_type, mrb_regexp_pattern);
   if (!pat) mrb_raise(mrb, E_ARGUMENT_ERROR, "uninitialized Regexp");
+  re_check_encoding(mrb, str);
 
   const char *s = RSTRING_PTR(str);
   mrb_int slen = RSTRING_LEN(str);

@@ -801,14 +801,27 @@
   # lead bytes of the run alone made such a byte part of the character before
   # it, rewound too little, and the lookbehind then failed on text it
   # describes, or succeeded where the negative form describes it.
-  assert_equal 2, ("\x80ab" =~ /(?<=\x80a)b/)
-  assert_nil ("\x80ab" =~ /(?<!\x80a)b/)
+  #
+  # A subject whose bytes spell no character is refused wherever an encoding
+  # reads them, so the character rewind is asked of the build that reads none,
+  # and the byte rewind below puts the same question to the same bytes in
+  # either build.
+  if __ENCODING__ == "UTF-8"
+    assert_raise(ArgumentError) { "\x80ab" =~ /(?<=\x80a)b/ }
+    assert_raise(ArgumentError) { "\x80ab" =~ /(?<!\x80a)b/ }
+    assert_raise(ArgumentError) { "\xE3\x81ab" =~ /(?<=\xE3\x81a)b/ }
+    assert_raise(ArgumentError) { "\x80ab" =~ Regexp.new("(?<=\x80a)b") }
+  else
+    assert_equal 2, ("\x80ab" =~ /(?<=\x80a)b/)
+    assert_nil ("\x80ab" =~ /(?<!\x80a)b/)
+    # a sequence cut short spells no character either, so each of its bytes is
+    # one: E3 leads three bytes and only two follow it here
+    assert_equal 3, ("\xE3\x81ab" =~ /(?<=\xE3\x81a)b/)
+    # the same bytes written into the pattern rather than escaped
+    assert_equal 2, ("\x80ab" =~ Regexp.new("(?<=\x80a)b"))
+  end
+  # a subject that spells characters throughout is read in either build
   assert_nil ("ab" =~ /(?<=\x80a)b/)
-  # a sequence cut short spells no character either, so each of its bytes is
-  # one: E3 leads three bytes and only two follow it here
-  assert_equal 3, ("\xE3\x81ab" =~ /(?<=\xE3\x81a)b/)
-  # the same bytes written into the pattern rather than escaped
-  assert_equal 2, ("\x80ab" =~ Regexp.new("(?<=\x80a)b"))
   # a byte-indexed subject counts the same bytes, and rewinds by them
   assert_equal 2, ("\x80ab".b =~ Regexp.new("(?<=\x80a)b"))
   assert_equal 3, ("\xE3\x81ab".b =~ Regexp.new("(?<=\xE3\x81a)b"))

@@ -217,29 +217,38 @@
   assert_equal 1, (("x" + u) =~ Regexp.new("\u{1F600}"))
 end
 
-assert("Regexp - match positions on malformed UTF-8 agree with string indexing") do
-  # String indexing counts a byte no lead byte reaches as one character, but
-  # #begin used to count lead bytes only, so a stray continuation byte was
-  # zero width to it. Computing the length marks such a string single-byte
-  # and switches it to byte counting, so the same match reported one
-  # position before the length was known and another after.
+assert("Regexp - a subject whose bytes are not UTF-8 is refused whatever is known about it") do
+  # The refusal must not turn on what the string happens to have been asked
+  # about: the single-byte flag `#length` leaves behind says one byte per
+  # character, which a string of stray bytes satisfies too, so reading it in
+  # place of walking the bytes would make the same call answer one way before
+  # a length was taken and another after. That is what #begin used to do here,
+  # counting lead bytes only until the flag switched it to counting bytes.
   s = "a\x80b"
-  m = /b/.match(s)
-  before = m.begin(0)
-  assert_equal 3, s.length
-  assert_equal before, /b/.match(s).begin(0)
-  assert_equal 2, before
-  assert_equal "b", s[before]
-  assert_equal 3, m.end(0)
-  # A position argument walks the same characters, from either end. The
-  # fresh literal pins the walk on a string whose length is not known yet.
-  assert_equal "b", /b/.match(s, 2)[0]
-  assert_equal "a", /a/.match(s, -3)[0]
+  if __ENCODING__ == "UTF-8"
+    assert_raise(ArgumentError) { /b/.match(s) }
+    assert_equal 3, s.length
+    assert_raise(ArgumentError) { /b/.match(s) }
+    # A position argument is no way around it either, from either end. The
+    # fresh literal puts the question to a string whose length is not known.
+    assert_raise(ArgumentError) { /b/.match(s, 2) }
+    assert_raise(ArgumentError) { /a/.match("a\x80b", -3) }
+  else
+    # A build that reads no encoding for these bytes to break has nothing to
+    # refuse, and reports the byte positions the half below asserts in either
+    # build.
+    assert_equal 2, /b/.match(s).begin(0)
+    assert_equal 3, s.length
+    assert_equal 2, /b/.match(s).begin(0)
+    assert_equal "b", /b/.match(s, 2)[0]
+    assert_equal "a", /a/.match("a\x80b", -3)[0]
+  end
+  # A position outside the subject is settled before the bytes are read, so it
+  # answers nil in either build rather than raising.
   assert_nil /a/.match(s, -4)
-  assert_equal "a", /a/.match("a\x80b", -3)[0]
-  # Read as bytes the same subject reports byte offsets, which its own
-  # indexing agrees with too, and a position argument walks it from either
-  # end in bytes.
+  # Read as bytes the same subject makes no claim that could be broken, and
+  # every position it reports is a byte offset its own indexing agrees with,
+  # walked from either end.
   bs = "a\x80b".b
   bm = /b/.match(bs)
   assert_equal 2, bm.begin(0)
@@ -555,3 +564,48 @@
   assert_nil (re =~ utf8.call(0x8081))
   assert_nil (re =~ "A")
 end
+
+assert("Regexp - a subject whose bytes are not UTF-8 is refused") do
+  # CRuby raises ArgumentError for a subject holding a byte that stands for no
+  # character, and mruby answered for it, so a program moved from one to the
+  # other took a result CRuby would not have produced. `String#scrub` is how a
+  # subject like this becomes matchable.
+  skip unless __ENCODING__ == "UTF-8"
+  broken = "あ\x80b"     # "あ" followed by a lone continuation byte
+
+  assert_raise(ArgumentError) { broken =~ /b/ }
+  assert_raise(ArgumentError) { /b/ =~ broken }
+  assert_raise(ArgumentError) { /b/.match(broken) }
+  assert_raise(ArgumentError) { /b/.match?(broken) }
+  assert_raise(ArgumentError) { /b/ === broken }
+  assert_raise(ArgumentError) { broken.match(/b/) }
+  assert_raise(ArgumentError) { broken.match?(/b/) }
+  assert_raise(ArgumentError) { broken.index(/b/) }
+  assert_raise(ArgumentError) { broken.rindex(/b/) }
+  assert_raise(ArgumentError) { broken.byteindex(/b/) }
+  assert_raise(ArgumentError) { broken.byterindex(/b/) }
+  assert_raise(ArgumentError) { broken[/b/] }
+  assert_raise(ArgumentError) { broken.sub(/b/, "!") }
+  assert_raise(ArgumentError) { broken.sub(/b/) { "!" } }
+  assert_raise(ArgumentError) { broken.gsub(/b/, "!") }
+  assert_raise(ArgumentError) { broken.gsub(/b/) { "!" } }
+  assert_raise(ArgumentError) { broken.scan(/b/) }
+  assert_raise(ArgumentError) { broken.split(/b/) }
+  assert_raise(ArgumentError) { broken.partition(/b/) }
+  assert_raise(ArgumentError) { broken.rpartition(/b/) }
+  assert_raise(ArgumentError) { broken.start_with?(/b/) }
+  assert_raise(ArgumentError) { broken.dup.sub!(/b/, "!") }
+  assert_raise(ArgumentError) { broken.dup.gsub!(/b/, "!") }
+  assert_raise(ArgumentError) { broken.dup.slice!(/b/) }
+  assert_raise(ArgumentError) { broken.dup[/b/] = "!" }
+
+  # A byte-indexed subject is indexed by byte throughout, so its bytes make no
+  # claim that could be broken and it goes through as it always did.
+  assert_equal 4, (broken.b =~ /b/)
+  assert_equal 4, broken.b.match(/b/).begin(0)
+  assert_equal "あ\x80!".b, broken.b.sub(/b/, "!")
+
+  # A whole subject is untouched, including one the walk reads to the end.
+  assert_equal 2, ("あいb" =~ /b/)
+  assert_equal 1, ("a\u{10FFFF}b" =~ /b\z|\u{10FFFF}/)
+end