Skip to content

mruby-string-ext: validate UTF-8 sequences with mrb_utf8len #7105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 3 commits into from
Aug 12, 2026
Merged

mruby-string-ext: validate UTF-8 sequences with mrb_utf8len #7105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6764935
mruby-string-ext: pin what `String#ord` and `String#codepoints` reject
takumin Aug 12, 2026
a354343
mruby-string-ext: measure a scrubbed character with `mrb_utf8len`
takumin Aug 12, 2026
c7362cb
mruby-string-ext: decode a character on top of `mrb_utf8len`
takumin Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 22 additions & 50 deletions mrbgems/mruby-string-ext/src/string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -968,36 +968,26 @@ str_succ(mrb_state *mrb, mrb_value self)
#ifdef MRB_UTF8_STRING
extern const char mrb_utf8len_table[];

/* Decodes the UTF-8 character starting at p, storing its byte length through
lenp when that is not NULL. mrb_utf8len() answers 1 for every sequence it
rejects, so a lead byte measured as one byte is invalid. */
MRB_INLINE mrb_int
utf8code(mrb_state* mrb, const unsigned char* p, const unsigned char *e)
{
if (p[0] < 0x80) return p[0];

mrb_int len = mrb_utf8len_table[p[0]>>3];
mrb_int cp = -1;
if (p+len <= e && len > 1 && (p[1] & 0xc0) == 0x80) {
if (len == 2)
cp = ((p[0] & 0x1f) << 6) + (p[1] & 0x3f);
else if ((p[2] & 0xc0) == 0x80) {
if (len == 3)
cp = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f);
else if (len == 4 && (p[3] & 0xc0) == 0x80) {
cp = ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12)
+ ((p[2] & 0x3f) << 6) + (p[3] & 0x3f);
}
utf8code(mrb_state* mrb, const unsigned char* p, const unsigned char *e, mrb_int *lenp)
{
mrb_int len = mrb_utf8len((const char*)p, (const char*)e);
if (lenp) *lenp = len;
if (len == 1) {
if (p[0] >= 0x80) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid UTF-8 byte sequence");
}
return p[0];
}
/* Reject overlong sequences, UTF-16 surrogates, and code points above
U+10FFFF (RFC 3629, Unicode D93b). */
if (cp >= 0 &&
((len == 2 && cp >= 0x80) ||
(len == 3 && cp >= 0x800 && (cp < 0xD800 || 0xDFFF < cp)) ||
(len == 4 && cp >= 0x10000 && cp <= 0x10FFFF))) {
return cp;

mrb_int cp = p[0] & (0xff >> (len + 1));
for (mrb_int i = 1; i < len; i++) {
cp = (cp << 6) | (p[i] & 0x3f);
}
mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid UTF-8 byte sequence");
/* not reached */
return -1;
return cp;
}

static mrb_value
Expand All @@ -1015,7 +1005,7 @@ str_ord(mrb_state* mrb, mrb_value str)
c = p[0];
}
else {
c = utf8code(mrb, p, e);
c = utf8code(mrb, p, e, NULL);
}
return mrb_fixnum_value(c);
}
Expand All @@ -1027,27 +1017,8 @@ str_ord(mrb_state* mrb, mrb_value str)
static mrb_int
str_scrub_char_len(const unsigned char *p, const unsigned char *e)
{
if (p[0] < 0x80) return 1;
mrb_int len = mrb_utf8len_table[p[0]>>3];
if (len < 2 || len > e - p) return -1;
for (mrb_int i = 1; i < len; i++) {
if ((p[i] & 0xc0) != 0x80) return -1;
}
mrb_int cp;
if (len == 2) {
cp = ((p[0] & 0x1f) << 6) | (p[1] & 0x3f);
if (cp < 0x80) return -1;
}
else if (len == 3) {
cp = ((p[0] & 0x0f) << 12) | ((p[1] & 0x3f) << 6) | (p[2] & 0x3f);
if (cp < 0x800) return -1;
if (cp >= 0xD800 && cp <= 0xDFFF) return -1;
}
else { /* len == 4 */
cp = ((p[0] & 0x07) << 18) | ((p[1] & 0x3f) << 12)
| ((p[2] & 0x3f) << 6) | (p[3] & 0x3f);
if (cp < 0x10000 || cp > 0x10FFFF) return -1;
}
mrb_int len = mrb_utf8len((const char*)p, (const char*)e);
if (len == 1 && p[0] >= 0x80) return -1;
return len;
}

Expand Down Expand Up @@ -1175,9 +1146,10 @@ str_codepoints(mrb_state *mrb, mrb_value str)
}
else {
while (p < e) {
mrb_int c = utf8code(mrb, p, e);
mrb_int len;
mrb_int c = utf8code(mrb, p, e, &len);
mrb_ary_push(mrb, result, mrb_int_value(mrb, c));
p += mrb_utf8len_table[p[0]>>3];
p += len;
}
}
return result;
Expand Down
20 changes: 20 additions & 0 deletions mrbgems/mruby-string-ext/test/string.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,26 @@ def assert_upto(exp, receiver, *args)
assert_equal expect, cp
end if UTF8STRING

assert('String#codepoints rejects malformed sequences') do
assert_raise(ArgumentError) { "\x80".codepoints } # stray continuation
assert_raise(ArgumentError) { "\xE3\x81".codepoints } # truncated
assert_raise(ArgumentError) { "\xC0\xAF".codepoints } # overlong "/"
assert_raise(ArgumentError) { "\xED\xA0\x80".codepoints } # surrogate U+D800
assert_raise(ArgumentError) { "\xF4\x90\x80\x80".codepoints } # > U+10FFFF
assert_raise(ArgumentError) { "\xF8\x88\x80\x80\x80".codepoints }
# the walk keeps its place: characters after a 4-byte one still decode
assert_equal [128169, 97], "\u{1F4A9}a".codepoints
assert_raise(ArgumentError) { "\u{1F4A9}\xC0\xAF".codepoints }
end if UTF8STRING

assert('String#ord rejects malformed sequences') do
assert_raise(ArgumentError) { "\x80".ord }
assert_raise(ArgumentError) { "\xE3\x81".ord }
assert_raise(ArgumentError) { "\xC0\xAF".ord }
assert_raise(ArgumentError) { "\xED\xA0\x80".ord }
assert_raise(ArgumentError) { "\xF4\x90\x80\x80".ord }
end if UTF8STRING

assert('String#each_codepoint') do
expect = [104, 101, 108, 108, 111, 33]
cp = []
Expand Down
Loading