mruby · matz · Aug 12, 2026 · Aug 12, 2026 · Aug 12, 2026
@@ -181,6 +181,16 @@ uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);
 mrb_int mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int nchars);
 mrb_int mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi);
 
+/* Whether a string's bytes read as the encoding it is taken to have. A binary
+   string claims no encoding and is valid whatever its bytes are; any other
+   string is walked, and one that turns out to hold a byte standing for no
+   character is not valid. The single-byte flag is not read on the way in, so
+   the answer does not depend on whether the string has been measured before;
+   a string found to have one byte per character is marked on the way out. On
+   non-UTF-8 builds a string is bytes with no encoding to disagree with, and
+   this is always TRUE. */
+mrb_bool mrb_str_valid_encoding_p(mrb_state *mrb, mrb_value str);
+
 mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp);
 #ifdef MRB_UTF8_STRING
 mrb_int mrb_utf8len(const char *str, const char *end);

@@ -17,27 +17,7 @@
 static mrb_value
 str_valid_enc_p(mrb_state *mrb, mrb_value str)
 {
-  struct RString *s = mrb_str_ptr(str);
-  /* MRB_STR_SINGLE_BYTE says one byte per character, which every other reader
-     uses to index without decoding. A string of stray bytes has that property
-     too, so it cannot stand in for "valid" here: String#size sets the flag on
-     "a\x80" and this used to answer true for it afterwards. Only the loop
-     below decides. */
-  if (RSTR_BINARY_P(s)) return mrb_true_value();
-
-  mrb_int byte_len = RSTR_LEN(s);
-  mrb_int utf8_len = 0;
-  const char *p = RSTR_PTR(s);
-  const char *e = p + byte_len;
-  while (p < e) {
-    mrb_int len = mrb_utf8len(p, e);
-
-    if (len == 1 && (*p & 0x80)) return mrb_false_value();
-    p += len;
-    utf8_len++;
-  }
-  if (byte_len == utf8_len) RSTR_SET_SINGLE_BYTE_FLAG(s);
-  return mrb_true_value();
+  return mrb_bool_value(mrb_str_valid_encoding_p(mrb, str));
 }
 
 static mrb_value

@@ -41,6 +41,29 @@
   end
 end
 
+assert('String#valid_encoding? after a run of ASCII') do
+  # The walk skips ASCII a word at a time and decodes only where a byte leaves
+  # that range, so a broken byte has to be caught after such a run as well as
+  # at the head of the string.
+  if UTF8STRING
+    assert_true ("a" * 40).valid_encoding?
+    assert_true ("a" * 40 + "あ").valid_encoding?
+    assert_false ("a" * 40 + "\xfe").valid_encoding?
+    assert_false ("a" * 40 + "\xe3\x81").valid_encoding?  # 3-byte sequence cut short
+    assert_true ("a" * 40 + "\xe3\x81").b.valid_encoding?
+  end
+end
+
+assert('String#valid_encoding? of a shared substring') do
+  # A substring too long to embed shares the parent's buffer, so the walk has
+  # to stop where the substring ends rather than where the parent's bytes do.
+  if UTF8STRING
+    parent = "あ" * 40 + "\xfe"
+    assert_true parent.byteslice(0, 120).valid_encoding?
+    assert_false parent.byteslice(0, 121).valid_encoding?
+  end
+end
+
 assert('String#length of a binary string counts bytes') do
   # A byte-indexed string has one position per byte, which is what indexing it
   # already answered. Measuring it read the same string as UTF-8, so the length

@@ -525,8 +525,12 @@ static inline uint32_t popcount(bitint x)
 }
 #endif
 
-mrb_int
-mrb_utf8_strlen(const char *str, mrb_int byte_len)
+/* Counts characters, and when `validp` is given also reports whether every
+   sequence decoded as one character. The walk stops at the first broken
+   sequence, so the returned count is a character count only while `*validp`
+   stays TRUE. */
+static mrb_int
+utf8_strlen_check(const char *str, mrb_int byte_len, mrb_bool *validp)
 {
   const char *p = str;
   const char *e = str + byte_len;
@@ -539,13 +543,28 @@ mrb_utf8_strlen(const char *str, mrb_int byte_len)
     if (np == e) break;
     p = np;
     while (p < e && NOASCII(*p)) {
-      p += mrb_utf8len(p, e);
+      mrb_int clen = mrb_utf8len(p, e);
+
+      /* mrb_utf8len() answers 1 for a byte that leads no valid sequence. The
+         byte here is known to be non-ASCII, so a length of 1 means the string
+         carries a byte that stands for no character. */
+      if (validp && clen == 1) {
+        *validp = FALSE;
+        return len;
+      }
+      p += clen;
       len++;
     }
   }
   return len;
 }
 
+mrb_int
+mrb_utf8_strlen(const char *str, mrb_int byte_len)
+{
+  return utf8_strlen_check(str, byte_len, NULL);
+}
+
 static mrb_int
 utf8_strlen(mrb_value str)
 {
@@ -578,6 +597,27 @@ utf8_strlen(mrb_value str)
 
 #define RSTRING_CHAR_LEN(s) utf8_strlen(s)
 
+/* whether a string's bytes read as the encoding it is taken to have */
+mrb_bool
+mrb_str_valid_encoding_p(mrb_state *mrb, mrb_value str)
+{
+  (void)mrb;
+  struct RString *s = mrb_str_ptr(str);
+  /* A byte-indexed string makes no such claim, so it is valid whatever its
+     bytes are. MRB_STR_SINGLE_BYTE is deliberately not read here: it says one
+     byte per character, which a string of stray bytes satisfies too, so only
+     the walk below decides. */
+  if (RSTR_BINARY_P(s)) return TRUE;
+
+  mrb_int byte_len = RSTR_LEN(s);
+  mrb_bool valid = TRUE;
+  mrb_int utf8_len = utf8_strlen_check(RSTR_PTR(s), byte_len, &valid);
+
+  if (!valid) return FALSE;
+  if (byte_len == utf8_len) RSTR_SET_SINGLE_BYTE_FLAG(s);
+  return TRUE;
+}
+
 /* map character index to byte offset index */
 mrb_int
 mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
@@ -711,6 +751,15 @@ mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi)
 #define char_adjust(ptr, end) (ptr)
 #define char_backtrack(ptr, end) ((end) - 1)
 #define str_index_str_by_char(mrb, str, sub, pos) str_index_str((mrb), (str), (sub), (pos))
+
+/* a string is bytes here, with no encoding to disagree with */
+mrb_bool
+mrb_str_valid_encoding_p(mrb_state *mrb, mrb_value str)
+{
+  (void)mrb;
+  (void)str;
+  return TRUE;
+}
 #endif
 
 /* memsearch_swar (SWAR stands for SIMD within a register)                 */