mruby · matz · Aug 12, 2026 · Aug 12, 2026 · Aug 12, 2026 · Aug 12, 2026
@@ -192,9 +192,9 @@ mrb_bool mrb_str_valid_encoding_p(mrb_state *mrb, mrb_value str);
 mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp);
 
 /* What a run of bytes spells is a question apart from whether String indexes
-   by character, so a gem that reads UTF-8 on its own asks for these two by
+   by character, so a gem that reads UTF-8 on its own asks for these by
    defining MRB_UTF8_SCAN (mruby-regexp does, from its mrbgem.rake). A build
-   with neither that gem nor MRB_UTF8_STRING carries neither function. */
+   with neither that gem nor MRB_UTF8_STRING carries none of them. */
 #if defined(MRB_UTF8_STRING) || defined(MRB_UTF8_SCAN)
 /* The byte length of the character at `str`, which has to be a byte of the
    string rather than `end` itself, and 1 for a run of bytes that spells no
@@ -205,6 +205,13 @@ mrb_int mrb_utf8len(const char *str, const char *end);
    already a character boundary. A continuation byte that no lead byte reaches
    is a boundary too. */
 const char *mrb_utf8_char_head(const char *beg, const char *p, const char *end);
+
+/* The codepoint of the character at `p`, which has to be a byte of the string
+   rather than `e` itself, with the byte length consumed always stored through
+   `lenp`. A run of bytes that spells no character comes back as its first
+   byte over one byte, so a value of 0x80 or above beside *lenp == 1 marks an
+   invalid sequence; whether that is an error is the caller's question. */
+uint32_t mrb_utf8_decode(const char *p, const char *e, mrb_int *lenp);
 #endif
 
 #ifdef MRB_UTF8_STRING

@@ -173,8 +173,7 @@ mrb_regexp_pattern* mrb_re_compile(mrb_state *mrb, const char *pattern, mrb_int
 /* Free a compiled pattern */
 void mrb_re_free(mrb_state *mrb, mrb_regexp_pattern *pat);
 
-/* UTF-8 helpers */
-uint32_t mrb_re_utf8_decode(const char *s, const char *end, int *len);
+/* Word character (\w) test */
 mrb_bool mrb_re_is_word_char(uint32_t c);
 
 /* The two foldings whose result is an ASCII letter. Every build carries them,
@@ -242,7 +241,10 @@ mrb_re_decode_char(const char *s, const char *end, int *len, mrb_bool binary)
     if (len) *len = 1;
     return (uint8_t)*s;
   }
-  return mrb_re_utf8_decode(s, end, len);
+  mrb_int n;
+  uint32_t cp = mrb_utf8_decode(s, end, &n);
+  if (len) *len = (int)n;
+  return cp;
 }
 
 /* TRUE when s points into the middle of a character that starts earlier in

@@ -6,7 +6,7 @@ MRuby::Gem::Specification.new('mruby-regexp') do |spec|
   spec.add_dependency 'mruby-string-ext', :core => 'mruby-string-ext'
 
   # The engine reads UTF-8 whatever a build's strings index by, so it asks core
-  # for the two functions that answer what a run of bytes spells. They wait
+  # for the functions that answer what a run of bytes spells. They wait
   # behind MRB_UTF8_STRING otherwise, and this build has no reason to set that:
   # mruby-encoding is what does, and the default gembox carries this gem
   # without it.

@@ -596,8 +596,8 @@ read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte)
   }
   /* Multi-byte UTF-8 leader: decode the full codepoint. An invalid leader
      decodes as itself over one byte, so it is a byte like the rest. */
-  int len = 0;
-  uint32_t cp = mrb_re_utf8_decode(c->p, c->src_end, &len);
+  mrb_int len = 0;
+  uint32_t cp = mrb_utf8_decode(c->p, c->src_end, &len);
   c->p += len;
   if (len == 1) *is_byte = TRUE;
   return cp;
@@ -1015,8 +1015,8 @@ static mrb_bool
 emit_char_folded(re_compiler *c, int ch)
 {
   if (ch < 128 || !(c->flags & RE_FLAG_IGNORECASE)) return FALSE;
-  int len = 0;
-  uint32_t cp = mrb_re_utf8_decode(c->p - 1, c->src_end, &len);
+  mrb_int len = 0;
+  uint32_t cp = mrb_utf8_decode(c->p - 1, c->src_end, &len);
   if (len == 1) return FALSE;
   if (!emit_cp_folded(c, cp)) return FALSE;
   c->p += len - 1;

@@ -1,44 +1,11 @@
 /*
-** re_utf8.c - UTF-8 utility functions for regexp engine
+** re_utf8.c - case folding and word characters for regexp engine
 **
 ** See Copyright Notice in mruby.h
 */
 
 #include "re_internal.h"
 
-/* Decode a UTF-8 character and return its codepoint.
-   *len is set to the byte length consumed. mrb_utf8len() answers 1 for every
-   sequence it rejects, so those consume a single byte and come back as the
-   lead byte itself. */
-uint32_t
-mrb_re_utf8_decode(const char *s, const char *end, int *len)
-{
-  uint8_t c = (uint8_t)s[0];
-  uint32_t cp;
-  int n = (int)mrb_utf8len(s, end);
-
-  *len = n;
-  switch (n) {
-  case 2:
-    cp = (c & 0x1f) << 6;
-    cp |= ((uint8_t)s[1] & 0x3f);
-    return cp;
-  case 3:
-    cp = (c & 0x0f) << 12;
-    cp |= ((uint8_t)s[1] & 0x3f) << 6;
-    cp |= ((uint8_t)s[2] & 0x3f);
-    return cp;
-  case 4:
-    cp = (c & 0x07) << 18;
-    cp |= ((uint8_t)s[1] & 0x3f) << 12;
-    cp |= ((uint8_t)s[2] & 0x3f) << 6;
-    cp |= ((uint8_t)s[3] & 0x3f);
-    return cp;
-  default:
-    return c;  /* ASCII, or invalid/truncated byte returned as-is */
-  }
-}
-
 /* Check if character is a "word" character (\w): [a-zA-Z0-9_] */
 mrb_bool
 mrb_re_is_word_char(uint32_t c)

@@ -967,25 +967,18 @@ str_succ(mrb_state *mrb, mrb_value self)
 
 #ifdef MRB_UTF8_STRING
 /* Decodes the UTF-8 character starting at p, storing its byte length through
-   lenp when that is not NULL. mrb_utf8len() answers 1 for every sequence it
-   rejects, so a lead byte measured as one byte is invalid. */
+   lenp when that is not NULL. mrb_utf8_decode() hands back a rejected
+   sequence as its lead byte over one byte; String treats that as an error. */
 MRB_INLINE mrb_int
 utf8code(mrb_state* mrb, const unsigned char* p, const unsigned char *e, mrb_int *lenp)
 {
-  mrb_int len = mrb_utf8len((const char*)p, (const char*)e);
+  mrb_int len;
+  uint32_t cp = mrb_utf8_decode((const char*)p, (const char*)e, &len);
   if (lenp) *lenp = len;
-  if (len == 1) {
-    if (p[0] >= 0x80) {
-      mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid UTF-8 byte sequence");
-    }
-    return p[0];
-  }
-
-  mrb_int cp = p[0] & (0xff >> (len + 1));
-  for (mrb_int i = 1; i < len; i++) {
-    cp = (cp << 6) | (p[i] & 0x3f);
+  if (len == 1 && p[0] >= 0x80) {
+    mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid UTF-8 byte sequence");
   }
-  return cp;
+  return (mrb_int)cp;
 }
 
 static mrb_value

@@ -441,6 +441,39 @@ mrb_utf8_char_head(const char *beg, const char *p, const char *end)
   return p;
 }
 
+/* Decode a UTF-8 character and return its codepoint.
+   *lenp is set to the byte length consumed. mrb_utf8len() answers 1 for every
+   sequence it rejects, so those consume a single byte and come back as the
+   lead byte itself. */
+uint32_t
+mrb_utf8_decode(const char *p, const char *e, mrb_int *lenp)
+{
+  uint8_t c = (uint8_t)p[0];
+  uint32_t cp;
+  mrb_int n = mrb_utf8len(p, e);
+
+  *lenp = n;
+  switch (n) {
+  case 2:
+    cp = (c & 0x1f) << 6;
+    cp |= ((uint8_t)p[1] & 0x3f);
+    return cp;
+  case 3:
+    cp = (c & 0x0f) << 12;
+    cp |= ((uint8_t)p[1] & 0x3f) << 6;
+    cp |= ((uint8_t)p[2] & 0x3f);
+    return cp;
+  case 4:
+    cp = (c & 0x07) << 18;
+    cp |= ((uint8_t)p[1] & 0x3f) << 12;
+    cp |= ((uint8_t)p[2] & 0x3f) << 6;
+    cp |= ((uint8_t)p[3] & 0x3f);
+    return cp;
+  default:
+    return c;  /* ASCII, or invalid/truncated byte returned as-is */
+  }
+}
+
 #endif  /* MRB_UTF8_STRING || MRB_UTF8_SCAN */
 
 #ifdef MRB_UTF8_STRING