mruby · matz · Aug 12, 2026 · Aug 12, 2026 · Aug 12, 2026
@@ -168,6 +168,17 @@ mrb_bool mrb_strcasecmp_p(const char *s1, mrb_int len1, const char *s2, mrb_int
 uint32_t mrb_byte_hash(const uint8_t*, mrb_int);
 uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);
 
+/* Character-index/byte-offset conversion, honoring the string's own indexing
+   (single-byte and binary strings index by byte). mrb_str_char_to_byte returns
+   the byte length of `nchars` characters starting at byte offset `off`; when
+   the string ends before `nchars` characters, the remaining byte length plus
+   one is returned so out-of-range requests stay detectable. mrb_str_byte_to_char
+   returns the character index for byte offset `bi` counted from the start of
+   the string, or -1 when `bi` is past the end or inside a multi-byte character.
+   On non-UTF-8 builds a byte is a character and both are identity. */
+mrb_int mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int nchars);
+mrb_int mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi);
+
 mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp);
 #ifdef MRB_UTF8_STRING
 mrb_int mrb_utf8len(const char *str, const char *end);

@@ -183,33 +183,25 @@ re_byte_substr(mrb_state *mrb, mrb_value str, mrb_int beg, mrb_int len)
 }
 
 /* Convert a byte offset into str to a character offset, so MatchData#begin
-   and #end report character positions like CRuby. Counts UTF-8 lead bytes
-   (every byte that is not a 10xxxxxx continuation) in [0, byte_off). On
-   non-UTF-8 builds a byte is a character, so the offset is returned as-is. */
+   and #end report character positions like CRuby. Engine offsets normally
+   sit on character boundaries; one that does not (possible only on malformed
+   UTF-8) is backed up to the start of the containing character. Negative
+   offsets pass through for unmatched captures. */
 static mrb_int
 re_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int byte_off)
 {
-  (void)mrb;
   if (byte_off < 0) return byte_off;
 
-#ifdef MRB_UTF8_STRING
-  struct RString *s = RSTRING(str);
-  if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) return byte_off;
-
-  mrb_int len = RSTR_LEN(s);
+  mrb_int len = RSTRING_LEN(str);
   if (byte_off > len) byte_off = len;
 
-  const char *p = RSTR_PTR(s);
-  mrb_int chars = 0;
-  for (mrb_int i = 0; i < byte_off; i++) {
-    if (((unsigned char)p[i] & 0xC0) != 0x80) chars++;
+  mrb_int chars = mrb_str_byte_to_char(mrb, str, byte_off);
+  while (chars < 0 && byte_off > 0) {
+    chars = mrb_str_byte_to_char(mrb, str, --byte_off);
   }
   return chars;
-#else
-  (void)str;
-  return byte_off;
-#endif
 }
+
 /* Normalize Regexp#match positional argument for the regexp engine.
    For UTF-8 multibyte strings, Ruby's public pos is a character offset and
    must be converted to a byte offset. For single-byte or binary strings,
@@ -218,43 +210,16 @@ re_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int byte_off)
 static mrb_int
 re_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int char_off)
 {
-  (void)mrb;
-#ifdef MRB_UTF8_STRING
-  struct RString *s = RSTRING(str);
-  mrb_int len = RSTR_LEN(s);
-  if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
-    if (char_off < 0) char_off += len;
-    if (char_off < 0 || char_off > len) return -1;
-    return char_off;
-  }
-
-  const char *p = RSTR_PTR(s);
-  const char *e = p + len;
-  mrb_int chars = 0;
+  mrb_int len = RSTRING_LEN(str);
 
   if (char_off < 0) {
-    mrb_int char_len = re_byte_to_char(mrb, str, len);
-    char_off += char_len;
+    char_off += mrb_str_byte_to_char(mrb, str, len);
     if (char_off < 0) return -1;
   }
-  else if (char_off > len) {
-    return -1;
-  }
-  while (p < e && chars < char_off) {
-    p++;
-    while (p < e && (((unsigned char)*p & 0xC0) == 0x80)) {
-      p++;
-    }
-    chars++;
-  }
-  if (chars < char_off) return -1;
-  return (mrb_int)(p - RSTR_PTR(s));
-#else
-  mrb_int len = RSTRING_LEN(str);
-  if (char_off < 0) char_off += len;
-  if (char_off < 0 || char_off > len) return -1;
-  return char_off;
-#endif
+
+  mrb_int byte_off = mrb_str_char_to_byte(mrb, str, 0, char_off);
+  if (byte_off > len) return -1;
+  return byte_off;
 }
 
 static mrb_bool

@@ -196,6 +196,28 @@
   assert_equal 1, (("x" + u) =~ Regexp.new("\u{1F600}"))
 end
 
+assert("Regexp - match positions on malformed UTF-8 agree with string indexing") do
+  # String indexing counts a byte no lead byte reaches as one character, but
+  # #begin used to count lead bytes only, so a stray continuation byte was
+  # zero width to it. Computing the length marks such a string single-byte
+  # and switches it to byte counting, so the same match reported one
+  # position before the length was known and another after.
+  s = "a\x80b"
+  m = /b/.match(s)
+  before = m.begin(0)
+  assert_equal 3, s.length
+  assert_equal before, /b/.match(s).begin(0)
+  assert_equal 2, before
+  assert_equal "b", s[before]
+  assert_equal 3, m.end(0)
+  # A position argument walks the same characters, from either end. The
+  # fresh literal pins the walk on a string whose length is not known yet.
+  assert_equal "b", /b/.match(s, 2)[0]
+  assert_equal "a", /a/.match(s, -3)[0]
+  assert_nil /a/.match(s, -4)
+  assert_equal "a", /a/.match("a\x80b", -3)[0]
+end
+
 assert("Regexp - a match does not end inside a character") do
   # A pattern is compiled byte by byte and RE_CHAR consumes one byte, so a
   # pattern holding a byte that reaches no character ends its match in the
@@ -255,6 +277,8 @@
   assert_nil /い/.match("あいあ", 2)
   assert_nil /あ/.match("あいあ", 4)
   assert_nil /あ/.match("あいあ", -4)
+  # The position one past the last character is the end, not out of range.
+  assert_equal 3, //.match("あいあ", 3).begin(0)
   assert_true /あ/.match?("あいあ", 2)
   assert_false /い/.match?("あいあ", 2)
 end

@@ -533,8 +533,9 @@ utf8_strlen(mrb_value str)
   mrb_int byte_len = RSTR_LEN(s);
 
   /* A byte-indexed string has one position per byte, which is what
-     chars2bytes() and bytes2chars() already answer for it. Asked here only
-     about the single-byte flag, the same string was measured as UTF-8 and
+     mrb_str_char_to_byte() and mrb_str_byte_to_char() already answer for it.
+     Asked here only about the single-byte flag, the same string was measured
+     as UTF-8 and
      reported a length its own indexing did not agree with.
 
      The flag below is deliberately not set on the way out: it says the bytes
@@ -559,9 +560,10 @@ utf8_strlen(mrb_value str)
 #define RSTRING_CHAR_LEN(s) utf8_strlen(s)
 
 /* map character index to byte offset index */
-static mrb_int
-chars2bytes(mrb_value str, mrb_int off, mrb_int idx)
+mrb_int
+mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
 {
+  (void)mrb;
   struct RString *s = mrb_str_ptr(str);
   if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
     return idx;
@@ -598,9 +600,10 @@ chars2bytes(mrb_value str, mrb_int off, mrb_int idx)
 }
 
 /* map byte offset to character index */
-static mrb_int
-bytes2chars(mrb_value str, mrb_int bi)
+mrb_int
+mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi)
 {
+  (void)mrb;
   struct RString *s = mrb_str_ptr(str);
   if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
     return bi;
@@ -656,21 +659,36 @@ str_index_str_by_char(mrb_state *mrb, mrb_value str, mrb_value sub, mrb_int pos)
   mrb_int len = RSTRING_LEN(sub);
 
   if (pos > 0) {
-    pos = chars2bytes(str, 0, pos);
+    pos = mrb_str_char_to_byte(mrb, str, 0, pos);
   }
 
   pos = mrb_str_index(mrb, str, ptr, len, pos);
 
   if (pos > 0) {
-    pos = bytes2chars(str, pos);
+    pos = mrb_str_byte_to_char(mrb, str, pos);
   }
   return pos;
 }
 
 #else
 #define RSTRING_CHAR_LEN(s) RSTRING_LEN(s)
-#define chars2bytes(s, off, ci) (ci)
-#define bytes2chars(s, bi) (bi)
+/* a byte is a character here, so both conversions are identity */
+mrb_int
+mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
+{
+  (void)mrb;
+  (void)str;
+  (void)off;
+  return idx;
+}
+
+mrb_int
+mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi)
+{
+  (void)mrb;
+  (void)str;
+  return bi;
+}
 #define char_adjust(ptr, end) (ptr)
 #define char_backtrack(ptr, end) ((end) - 1)
 #define str_index_str_by_char(mrb, str, sub, pos) str_index_str((mrb), (str), (sub), (pos))
@@ -826,8 +844,8 @@ mrb_str_byte_subseq(mrb_state *mrb, mrb_value str, mrb_int beg, mrb_int len)
 static inline mrb_value
 str_subseq(mrb_state *mrb, mrb_value str, mrb_int beg, mrb_int len)
 {
-  beg = chars2bytes(str, 0, beg);
-  len = chars2bytes(str, beg, len);
+  beg = mrb_str_char_to_byte(mrb, str, 0, beg);
+  len = mrb_str_char_to_byte(mrb, str, beg, len);
   return mrb_str_byte_subseq(mrb, str, beg, len);
 }
 #else
@@ -1675,8 +1693,8 @@ mrb_str_aset(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value alen, mrb_v
       if (beg < 0 || beg > charlen) { str_out_of_index(mrb, idx); }
       /* fall through */
     case STR_CHAR_RANGE_CORRECTED:
-      beg = chars2bytes(str, 0, beg);
-      len = chars2bytes(str, beg, len);
+      beg = mrb_str_char_to_byte(mrb, str, 0, beg);
+      len = mrb_str_char_to_byte(mrb, str, beg, len);
       /* fall through */
     case STR_BYTE_RANGE_CORRECTED:
       if (mrb_int_add_overflow(beg, len, &len)) {
@@ -2482,7 +2500,7 @@ mrb_str_rindex_m(mrb_state *mrb, mrb_value str)
     pos = RSTRING_LEN(str);
   }
   else if (pos >= 0) {
-    pos = chars2bytes(str, 0, pos);
+    pos = mrb_str_char_to_byte(mrb, str, 0, pos);
   }
   else {
     const char *p = RSTRING_PTR(str);
@@ -2495,7 +2513,7 @@ mrb_str_rindex_m(mrb_state *mrb, mrb_value str)
   }
   pos = str_rindex(mrb, str, sub, pos);
   if (pos >= 0) {
-    pos = bytes2chars(str, pos);
+    pos = mrb_str_byte_to_char(mrb, str, pos);
     if (pos < 0) return mrb_nil_value();
     return mrb_int_value(mrb, pos);
   }
@@ -2615,7 +2633,7 @@ mrb_str_split_m(mrb_state *mrb, mrb_value str)
         if (end < 0) break;
       }
       else {
-        end = chars2bytes(str, idx, 1);
+        end = mrb_str_char_to_byte(mrb, str, idx, 1);
       }
       mrb_ary_push(mrb, result, mrb_str_byte_subseq(mrb, str, idx, end));
       mrb_gc_arena_restore(mrb, ai);