mruby · matz · Aug 14, 2026 · Aug 13, 2026 · Aug 13, 2026 · Aug 13, 2026
@@ -201,11 +201,11 @@ void mrb_str_check_byte_pos(mrb_state *mrb, mrb_value str, mrb_int pos);
    while mrb_utf8len() says it does not, is in the definition in string.c. */
 mrb_int mrb_utf8_to_buf(char *buf, mrb_int cp);
 
-/* What a run of bytes spells is a question apart from whether String indexes
-   by character, so a gem that reads UTF-8 on its own asks for these by
-   defining MRB_UTF8_SCAN (mruby-regexp does, from its mrbgem.rake). A build
-   with neither that gem nor MRB_UTF8_STRING carries none of them. */
-#if defined(MRB_UTF8_STRING) || defined(MRB_UTF8_SCAN)
+/* UTF-8: what a run of bytes spells, and how many characters a string holds.
+   Only a build that indexes strings by character has to answer either, so a
+   build without MRB_UTF8_STRING carries none of them. What has to read a
+   string whatever the build encodes it in asks through mrb_enc_* below. */
+#ifdef MRB_UTF8_STRING
 /* The byte length of the character at `str`, which has to be a byte of the
    string rather than `end` itself, and 1 for a run of bytes that spells no
    character. See the definition in string.c for what it rejects. */
@@ -222,11 +222,65 @@ const char *mrb_utf8_char_head(const char *beg, const char *p, const char *end);
    byte over one byte, so a value of 0x80 or above beside *lenp == 1 marks an
    invalid sequence; whether that is an error is the caller's question. */
 uint32_t mrb_utf8_decode(const char *p, const char *e, mrb_int *lenp);
+
+mrb_int mrb_utf8_strlen(const char *str, mrb_int byte_len);
 #endif
 
+/* Whether more than one byte can spell one character in what this build
+   reads. The three functions below answer what a given run of bytes spells,
+   which is what a reader wants; this is for the few places that have to know
+   the shape of the answer before they have bytes to ask about, such as
+   whether a set of single characters can hold a named codepoint at all. */
 #ifdef MRB_UTF8_STRING
-mrb_int mrb_utf8_strlen(const char *str, mrb_int byte_len);
+# define MRB_ENC_MULTIBYTE_P 1
+#else
+# define MRB_ENC_MULTIBYTE_P 0
+#endif
+
+/* What a run of bytes spells, in whatever a build's strings are encoded in.
+   These are the three above where the build reads UTF-8, and one byte per
+   character where it does not, which is what a String is there. Anything that
+   has to read a string whatever the build indexes it by asks through these,
+   so that adding a codec is a change here rather than in every caller. The
+   spelling of a codepoint has no such answer and stays UTF-8: see
+   mrb_utf8_to_buf() above.
+
+   The byte-per-character answers are inline because a matcher asks them once
+   per byte; where the build reads bytes each call folds into the constant it
+   returns and the branch around it goes away. */
+static inline mrb_int
+mrb_enc_charlen(const char *p, const char *e)
+{
+#ifdef MRB_UTF8_STRING
+  return mrb_utf8len(p, e);
+#else
+  (void)p; (void)e;
+  return 1;
 #endif
+}
+
+static inline const char *
+mrb_enc_char_head(const char *beg, const char *p, const char *end)
+{
+#ifdef MRB_UTF8_STRING
+  return mrb_utf8_char_head(beg, p, end);
+#else
+  (void)beg; (void)end;
+  return p;  /* every byte starts a character of its own */
+#endif
+}
+
+static inline uint32_t
+mrb_enc_decode(const char *p, const char *e, mrb_int *lenp)
+{
+#ifdef MRB_UTF8_STRING
+  return mrb_utf8_decode(p, e, lenp);
+#else
+  (void)e;
+  *lenp = 1;
+  return (uint8_t)*p;
+#endif
+}
 
 /* attr accessor bodies (class.c); the VM compares function pointers against
    these to run attr calls without a full method-call frame */

@@ -158,20 +158,28 @@ pattern analysis.
 
 ## Limitations
 
+- **UTF-8 only where the build reads it**: the engine reads a pattern and a
+  subject the way the build's `String` reads them, so everything below about
+  characters holds on a build that defines `MRB_UTF8_STRING` (mruby-encoding
+  is what defines it). Where it is not defined a string is bytes and so is the
+  engine: `/./` matches one byte, `/Ā/` is two atoms of one byte each, and
+  `/i` folds ASCII letters and nothing else. A binary (`ASCII-8BIT`) subject
+  reads by byte on either build.
 - **Fixed-length lookbehind only**: `(?<=...)` and `(?<!...)`
   require a fixed-length pattern (no `*`, `+`, `?`, or alternation).
   Maximum 255 bytes.
 - **No Unicode properties**: `\p{Alpha}`, `\p{L}`, etc. are not
   supported.
 - **No `\x{...}` hex escape**: the hex escape is `\xHH`, so it reaches
   `0xff` at most. Write `\u{...}` for a codepoint above that.
-- **No encodings**: a pattern is a byte string read as UTF-8, and there is no
-  encoding to consult about a byte that starts no whole character. Such a byte
-  is that byte, inside a character class as much as outside one: `[\xB5]` and
-  `\xB5` both hold the byte `0xB5`, and neither matches `µ`, which is `C2 B5`.
-  CRuby settles the same question with the pattern's encoding and raises
-  `RegexpError` for either spelling. A range whose ends are a byte and a
-  character (`[\x80-µ]`) names neither and raises `RegexpError`.
+- **No encodings**: a pattern is a byte string read the way the build reads a
+  String, and there is no encoding to consult about a byte that starts no
+  whole character. Such a byte is that byte, inside a character class as much
+  as outside one: `[\xB5]` and `\xB5` both hold the byte `0xB5`, and neither
+  matches `µ`, which is `C2 B5`. CRuby settles the same question with the
+  pattern's encoding and raises `RegexpError` for either spelling. A range
+  whose ends are a byte and a character (`[\x80-µ]`) names neither and raises
+  `RegexpError`.
 - **ASCII case folding by default**: The `i` flag handles ASCII letters
   only unless the build defines `MRB_REGEXP_UNICODE_CASE`, which adds the
   Unicode foldings that pair one codepoint with one other. Without the

@@ -228,10 +228,17 @@ void mrb_re_case_unfold_range(uint32_t lo, uint32_t hi,
                               void (*add)(void *, uint32_t, uint32_t), void *user);
 #endif
 
+/* The byte length of the character at `s`, and its codepoint: every read of a
+   run of bytes in this gem goes through these two. A subject handed over as
+   binary is one character per byte; everything else is whatever core says a
+   run of bytes spells, which is one character per byte too on a build that
+   indexes Strings by byte. So the engine reads pattern and subject the way
+   the build reads a String, and the compiler passes FALSE for `binary`, a
+   pattern being read that same way. */
 static inline int
 mrb_re_charlen(const char *s, const char *end, mrb_bool binary)
 {
-  return binary ? 1 : (int)mrb_utf8len(s, end);
+  return binary ? 1 : (int)mrb_enc_charlen(s, end);
 }
 
 static inline uint32_t
@@ -242,7 +249,7 @@ mrb_re_decode_char(const char *s, const char *end, int *len, mrb_bool binary)
     return (uint8_t)*s;
   }
   mrb_int n;
-  uint32_t cp = mrb_utf8_decode(s, end, &n);
+  uint32_t cp = mrb_enc_decode(s, end, &n);
   if (len) *len = (int)n;
   return cp;
 }
@@ -255,10 +262,10 @@ mrb_re_decode_char(const char *s, const char *end, int *len, mrb_bool binary)
    so keep the answer for a byte that starts a character here rather than in
    the call. */
 static inline mrb_bool
-mrb_re_utf8_interior_p(const char *str, const char *s, const char *end)
+mrb_re_char_interior_p(const char *str, const char *s, const char *end)
 {
   if (s >= end || ((uint8_t)*s & 0xC0) != 0x80) return FALSE;
-  return mrb_utf8_char_head(str, s, end) != s;
+  return mrb_enc_char_head(str, s, end) != s;
 }
 
 /* Execute a match.

@@ -5,13 +5,6 @@ MRuby::Gem::Specification.new('mruby-regexp') do |spec|
 
   spec.add_dependency 'mruby-string-ext', :core => 'mruby-string-ext'
 
-  # The engine reads UTF-8 whatever a build's strings index by, so it asks core
-  # for the functions that answer what a run of bytes spells. They wait
-  # behind MRB_UTF8_STRING otherwise, and this build has no reason to set that:
-  # mruby-encoding is what does, and the default gembox carries this gem
-  # without it.
-  spec.build.defines << 'MRB_UTF8_SCAN'
-
   # Enumerator is optional: only String#gsub without a block reaches `to_enum`,
   # and without mruby-enumerator that is core Kernel#to_enum, which raises
   # NotImplementedError -- the same deal as Kernel#loop and String#each_char

@@ -546,6 +546,33 @@ class_add_member(re_compiler *c, re_charclass *cc, uint32_t cp, mrb_bool is_byte
   else class_add_codepoint(c, cc, (is_byte ? RE_CLASS_BYTE : 0) | cp);
 }
 
+/* What a `\u` escape names, in the members a class can hold. On a build whose
+   characters are single bytes, a codepoint above ASCII is the bytes that spell
+   it, which is already what a character written out in the class comes to
+   there: read_class_atom() decodes one byte at a time, so `[Ā]` holds `\xC4`
+   and `\x80`. Naming the same character rather than spelling it out cannot
+   mean something else, so the escape contributes those bytes too. All but the
+   last join the class here, and the last is returned, so it can open a range
+   as any other atom would.
+
+   A range so opened is a range of bytes, since that is what both ends are.
+   The written out spelling reaches byte ends by its own route and comes to a
+   different span, which is what a range between two characters neither
+   spelling can express comes to on a build like this. */
+static uint32_t
+class_named_cp(re_compiler *c, re_charclass *cc, uint32_t cp, mrb_bool *is_byte)
+{
+  if (MRB_ENC_MULTIBYTE_P || cp < 0x80) return cp;
+
+  char buf[4];
+  int len = (int)mrb_utf8_to_buf(buf, (mrb_int)cp);
+  for (int i = 0; i < len - 1; i++) {
+    class_add_member(c, cc, (uint8_t)buf[i], TRUE);
+  }
+  *is_byte = TRUE;
+  return (uint8_t)buf[len - 1];
+}
+
 /* Read one character class atom: either an ASCII byte (0-127), a
    `\escape`, or a full multi-byte UTF-8 codepoint. Returns the value and
    advances c->p. *is_byte says which of the two the value is: TRUE for a
@@ -573,10 +600,12 @@ read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte)
          the last join the class here; the last is returned, so it can open a
          range as any other atom would: `[\u{61 62}-z]` is `a` plus `b-z`. */
       while (unicode_escape_next(c, &more, &nx)) {
-        class_add_member(c, cc, cp, FALSE);
+        mrb_bool member_byte = FALSE;
+        uint32_t member = class_named_cp(c, cc, cp, &member_byte);
+        class_add_member(c, cc, member, member_byte);
         cp = nx;
       }
-      return cp;
+      return class_named_cp(c, cc, cp, is_byte);
     }
     /* A backslash before a multibyte character has no escape meaning, so let
        the decode below read the whole codepoint: [\Ā] is [Ā]. parse_escape()
@@ -598,8 +627,8 @@ read_class_atom(re_compiler *c, re_charclass *cc, mrb_bool *is_byte)
   }
   /* Multi-byte UTF-8 leader: decode the full codepoint. An invalid leader
      decodes as itself over one byte, so it is a byte like the rest. */
-  mrb_int len = 0;
-  uint32_t cp = mrb_utf8_decode(c->p, c->src_end, &len);
+  int len = 0;
+  uint32_t cp = mrb_re_decode_char(c->p, c->src_end, &len, FALSE);
   c->p += len;
   if (len == 1) *is_byte = TRUE;
   return cp;
@@ -843,8 +872,8 @@ compute_fixed_len(re_compiler *c, uint32_t start, uint32_t end, int *chars_out)
     case RE_CHAR: {
       /* A multibyte literal is a run of one-byte RE_CHAR instructions, and
          what a byte spells depends on the bytes after it, so hand the run to
-         mrb_utf8len rather than read the lead bit alone: a continuation byte
-         no lead reaches is a character of its own, which is the rule the
+         mrb_re_charlen() rather than read the lead bit alone: a continuation
+         byte no lead reaches is a character of its own, which is the rule the
          executor rewinds by. Four bytes is the longest character there is,
          and a run never splits one. */
       char buf[4];
@@ -853,7 +882,7 @@ compute_fixed_len(re_compiler *c, uint32_t start, uint32_t end, int *chars_out)
         buf[n] = (char)c->code[pc + n].a;
         n++;
       }
-      int clen = (int)mrb_utf8len(buf, buf + n);
+      int clen = mrb_re_charlen(buf, buf + n, FALSE);
       len += clen;
       chars += 1;
       pc += (uint32_t)clen;
@@ -971,7 +1000,7 @@ emit_char(re_compiler *c, uint8_t ch)
 static void
 emit_char_bytes(re_compiler *c, int ch)
 {
-  int len = (int)mrb_utf8len(c->p - 1, c->src_end);
+  int len = mrb_re_charlen(c->p - 1, c->src_end, FALSE);
   emit(c, RE_CHAR, (uint8_t)ch, 0);
   for (int i = 1; i < len; i++) {
     int b = next_char(c);
@@ -1028,8 +1057,8 @@ static mrb_bool
 emit_char_folded(re_compiler *c, int ch)
 {
   if (ch < 128 || !(c->flags & RE_FLAG_IGNORECASE)) return FALSE;
-  mrb_int len = 0;
-  uint32_t cp = mrb_utf8_decode(c->p - 1, c->src_end, &len);
+  int len = 0;
+  uint32_t cp = mrb_re_decode_char(c->p - 1, c->src_end, &len, FALSE);
   if (len == 1) return FALSE;
   if (!emit_cp_folded(c, cp)) return FALSE;
   c->p += len - 1;
@@ -1050,9 +1079,20 @@ emit_codepoint(re_compiler *c, uint32_t cp)
     emit_char(c, (uint8_t)cp);
     return;
   }
-  if ((c->flags & RE_FLAG_IGNORECASE) && emit_cp_folded(c, cp)) return;
   char buf[4];
   int len = (int)mrb_utf8_to_buf(buf, (mrb_int)cp);
+  /* Fold only a spelling the engine reads back as the one character it spells.
+     What the fold emits is a class, and a class compares one decoded
+     character; where the build decodes bytes it never sees this one, so the
+     class would answer for a lone byte of the same number rather than for the
+     character the pattern names. The bytes below name it on either build,
+     which is the fallback a character the pattern spells already takes there,
+     through the length emit_char_folded() reads. */
+  if ((c->flags & RE_FLAG_IGNORECASE) &&
+      mrb_re_charlen(buf, buf + len, FALSE) == len &&
+      emit_cp_folded(c, cp)) {
+    return;
+  }
   for (int i = 0; i < len; i++) {
     emit(c, RE_CHAR, (uint8_t)buf[i], 0);
   }

@@ -272,7 +272,7 @@ add_thread(pike_state *s, re_threadlist *list,
          seeding loop applies to where a match opens. Killing the thread
          rather than the attempt lets a longer branch match instead. */
       if (inst.offset == 1 && !s->binary && sp < s->str_end &&
-          mrb_re_utf8_interior_p(s->str, sp, s->str_end)) {
+          mrb_re_char_interior_p(s->str, sp, s->str_end)) {
         return;
       }
       if (!s->match_only) {
@@ -436,7 +436,7 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
          Threads seeded earlier are still stepped at this position, so the
          test guards the seeding alone and never skips the iteration. */
       if (s.binary || sp >= str_end ||
-          !mrb_re_utf8_interior_p(str, sp, str_end)) {
+          !mrb_re_char_interior_p(str, sp, str_end)) {
         int slot = match_only ? 0 : pool_alloc(&s);
         if (!match_only) memset(CAP(&s, slot), -1, sizeof(int) * ncap);
         advance_gen(&s);
@@ -584,7 +584,7 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
  * Where the lookbehind at pc starts matching from: sp rewound by the byte
  * count in the opcode for a binary subject, and otherwise by the character
  * count in the RE_LB_WIDTH that follows it. The backward walk steps over
- * continuation bytes with mrb_re_utf8_interior_p(), which keeps it on the
+ * continuation bytes with mrb_re_char_interior_p(), which keeps it on the
  * boundaries the forward decode uses, broken input included. Returns NULL
  * when the text before sp runs out first.
  */
@@ -601,7 +601,7 @@ lookbehind_start(const mrb_regexp_pattern *pat, const char *str,
   while (nchars > 0) {
     if (sp <= str) return NULL;
     sp--;
-    while (sp > str && mrb_re_utf8_interior_p(str, sp, str_end)) sp--;
+    while (sp > str && mrb_re_char_interior_p(str, sp, str_end)) sp--;
     nchars--;
   }
   return sp;
@@ -685,7 +685,7 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
            (see the Pike VM case). Failing here backtracks into the other
            branches, so a longer one can still match. */
         if (slot == 1 && !binary && sp < str_end &&
-            mrb_re_utf8_interior_p(str, sp, str_end)) {
+            mrb_re_char_interior_p(str, sp, str_end)) {
           return FALSE;
         }
         if (slot < ncap) {
@@ -824,7 +824,7 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
       while (sp < str_end && !FIRST_BYTE_OK(pat, (uint8_t)*sp)) sp++;
       if (sp > str_end) break;
     }
-    if (!binary && sp < str_end && mrb_re_utf8_interior_p(str, sp, str_end)) {
+    if (!binary && sp < str_end && mrb_re_char_interior_p(str, sp, str_end)) {
       continue;
     }
     memset(caps, -1, sizeof(int) * ncap);
@@ -856,13 +856,13 @@ literal_exec(const mrb_regexp_pattern *pat,
   while (sp + plen <= str_end) {
     const char *found = (const char*)memchr(sp, pat->prefix[0], str_end - sp);
     if (!found || found + plen > str_end) return 0;
-    if (!binary && mrb_re_utf8_interior_p(str, found, str_end)) {
+    if (!binary && mrb_re_char_interior_p(str, found, str_end)) {
       sp = found + 1;  /* not a char boundary, same rule as the other engines */
       continue;
     }
     if (plen == 1 || memcmp(found + 1, pat->prefix + 1, plen - 1) == 0) {
       if (!binary && found + plen < str_end &&
-          mrb_re_utf8_interior_p(str, found + plen, str_end)) {
+          mrb_re_char_interior_p(str, found + plen, str_end)) {
         sp = found + 1;  /* ends inside a character, same rule as the end of
                             group 0 in the other engines */
         continue;