Skip to content

string.c: ask a string what encoding it is, not whether a bit is set #7157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matz merged 1 commit into from
Aug 14, 2026
Merged

string.c: ask a string what encoding it is, not whether a bit is set #7157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions include/mruby/string.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,32 @@ struct RStringEmbed {
# define RSTR_COPY_BROKEN_ENC_FLAG(dst, src) (void)0
#endif
#define RSTR_SET_ASCII_FLAG(s) RSTR_SET_SINGLE_BYTE_FLAG(s)
#define RSTR_BINARY_P(s) ((s)->flags & MRB_STR_BINARY)
#define RSTR_SET_BINARY_FLAG(s) ((s)->flags |= MRB_STR_BINARY)
#define RSTR_UNSET_BINARY_FLAG(s) ((s)->flags &= ~MRB_STR_BINARY)
/* A copy of a string is byte-indexed exactly when the string it copies is, so
the flag travels with the bytes rather than being left behind on the
original. */
#define RSTR_COPY_BINARY_FLAG(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_BINARY) | RSTR_BINARY_P(src))
/* The encoding a string's bytes are read as, named as an index into the set of
encodings the build carries. Index 0 is that build's default, which is what
a string that says nothing else holds: the literals the parser hands over,
the strings numbers and times spell themselves with. The zero a fresh string
is filled with therefore already says the default, and the path every string
is made on stores nothing.

A build without MRB_UTF8_STRING carries no UTF-8, so it names none: writing
the name there is a compile error rather than a quiet no-op. Such a build
still tells a byte-read string from a default one, since String#b and
Integer#chr mark one there too. */
#define MRB_STR_ENCODING_DEFAULT 0
#define MRB_STR_ENCODING_BINARY 1
#ifdef MRB_UTF8_STRING
# define MRB_STR_ENCODING_UTF8 MRB_STR_ENCODING_DEFAULT
#endif

#define RSTR_ENCODING(s) \
(((s)->flags & MRB_STR_BINARY) ? MRB_STR_ENCODING_BINARY : MRB_STR_ENCODING_DEFAULT)
#define RSTR_ENCODING_SET(s, e) \
((s)->flags = ((s)->flags & ~MRB_STR_BINARY) | \
(((e) == MRB_STR_ENCODING_BINARY) ? MRB_STR_BINARY : 0))
#define RSTR_BINARY_P(s) (RSTR_ENCODING(s) == MRB_STR_ENCODING_BINARY)
/* A copy of a string is read the way the string it copies is, so the encoding
travels with the bytes rather than being left behind on the original. */
#define RSTR_ENC_COPY(dst, src) RSTR_ENCODING_SET(dst, RSTR_ENCODING(src))

/**
* Returns a pointer from a Ruby string
Expand Down
4 changes: 2 additions & 2 deletions mrbgems/mruby-encoding/src/encoding.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ str_force_encoding(mrb_state *mrb, mrb_value self)
struct RString *s = mrb_str_ptr(self);
if (MRB_STR_CASECMP_P(enc, ENC_ASCII_8BIT) ||
MRB_STR_CASECMP_P(enc, ENC_BINARY)) {
RSTR_SET_BINARY_FLAG(s);
RSTR_ENCODING_SET(s, MRB_STR_ENCODING_BINARY);
}
else if (MRB_STR_CASECMP_P(enc, ENC_UTF8)) {
RSTR_UNSET_BINARY_FLAG(s);
RSTR_ENCODING_SET(s, MRB_STR_ENCODING_UTF8);
}
else {
mrb_raisef(mrb, E_ARGUMENT_ERROR, "unknown encoding name - %v", enc);
Expand Down
2 changes: 1 addition & 1 deletion mrbgems/mruby-regexp/src/regexp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1205,7 +1205,7 @@ re_mark_spliced(mrb_value result, mrb_value subject, mrb_value replacement,
while (p < e && !(*p & 0x80)) p++;
if (p == e) return;
}
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}

/*
Expand Down
4 changes: 2 additions & 2 deletions mrbgems/mruby-sprintf/src/sprintf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ mark_written_bytes(mrb_value result, mrb_value src)
const char *p = RSTR_PTR(s);
const char *e = p + RSTR_LEN(s);
while (p < e && !(*p & 0x80)) p++;
if (p < e) RSTR_SET_BINARY_FLAG(r);
if (p < e) RSTR_ENCODING_SET(r, MRB_STR_ENCODING_BINARY);
}

static mrb_value
Expand Down Expand Up @@ -423,7 +423,7 @@ mrb_str_format(mrb_state *mrb, mrb_int argc, const mrb_value *argv, mrb_value fm
built out of them is read the way the format string was read, ASCII bytes
and all: the reading a receiver holds, which nothing written into it
lifts. */
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(result), mrb_str_ptr(fmt));
RSTR_ENC_COPY(mrb_str_ptr(result), mrb_str_ptr(fmt));
buf = RSTRING_PTR(result);
memset(buf, 0, bsiz);

Expand Down
2 changes: 1 addition & 1 deletion mrbgems/mruby-string-bitops/src/string_bitops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ static mrb_value
bitop_result_str(mrb_state *mrb, mrb_int len)
{
mrb_value result = mrb_str_new(mrb, NULL, len);
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
return result;
}

Expand Down
18 changes: 9 additions & 9 deletions mrbgems/mruby-string-ext/src/string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ int_chr_binary(mrb_state *mrb, mrb_value num)
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
}
else {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(str));
RSTR_ENCODING_SET(mrb_str_ptr(str), MRB_STR_ENCODING_BINARY);
}
return str;
}
Expand Down Expand Up @@ -125,7 +125,7 @@ str_mark_spliced_bytes(mrb_value recv, mrb_value src)
const char *p = RSTR_PTR(s);
const char *e = p + RSTR_LEN(s);
while (p < e && !(*p & 0x80)) p++;
if (p < e) RSTR_SET_BINARY_FLAG(r);
if (p < e) RSTR_ENCODING_SET(r, MRB_STR_ENCODING_BINARY);
}

static void
Expand Down Expand Up @@ -1459,7 +1459,7 @@ str_lines(mrb_state *mrb, mrb_value self)
/* a line of a byte-read string is a subrange of its bytes, read the
same way */
mrb_value line = mrb_str_new(mrb, t, len);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(line), mrb_str_ptr(self));
RSTR_ENC_COPY(mrb_str_ptr(line), mrb_str_ptr(self));
mrb_ary_push(mrb, result, line);
mrb_gc_arena_restore(mrb, ai);
}
Expand Down Expand Up @@ -1522,7 +1522,7 @@ static mrb_value
str_b(mrb_state *mrb, mrb_value self)
{
mrb_value str = mrb_str_dup(mrb, self);
RSTR_SET_BINARY_FLAG(mrb_str_ptr(str));
RSTR_ENCODING_SET(mrb_str_ptr(str), MRB_STR_ENCODING_BINARY);
return str;
}

Expand Down Expand Up @@ -1806,7 +1806,7 @@ str_chars_ary(mrb_state *mrb, mrb_value self)
way, so the marking goes with each one */
while (p < e) {
mrb_value piece = mrb_str_new(mrb, p, 1);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(piece), s);
RSTR_ENC_COPY(mrb_str_ptr(piece), s);
mrb_ary_push(mrb, result, piece);
p++;
}
Expand Down Expand Up @@ -1914,7 +1914,7 @@ str_rjust_core(mrb_state *mrb, mrb_value self)
/* the padded string is the receiver's bytes in wider clothes, so it is
read the way the receiver was, ASCII bytes and all */
if (RSTR_BINARY_P(mrb_str_ptr(self))) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}
return result;
}
Expand Down Expand Up @@ -1988,7 +1988,7 @@ str_center_core(mrb_state *mrb, mrb_value self)
/* the padded string is the receiver's bytes in wider clothes, so it is
read the way the receiver was, ASCII bytes and all */
if (RSTR_BINARY_P(mrb_str_ptr(self))) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}
return result;
}
Expand Down Expand Up @@ -2058,7 +2058,7 @@ mrb_str_slice_bang(mrb_state *mrb, mrb_value self)
/* the piece cut out is a subrange of the receiver's bytes, read the same
way; copied rather than shared, since the memmove below would slide the
receiver's remaining bytes through a shared buffer */
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(result), str);
RSTR_ENC_COPY(mrb_str_ptr(result), str);

mrb_str_modify(mrb, str);
ptr = RSTRING_PTR(self);
Expand Down Expand Up @@ -2093,7 +2093,7 @@ static mrb_value
str_cut_piece(mrb_state *mrb, mrb_value str, const char *p, mrb_int len)
{
mrb_value piece = mrb_str_new(mrb, p, len);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(piece), mrb_str_ptr(str));
RSTR_ENC_COPY(mrb_str_ptr(piece), mrb_str_ptr(str));
return piece;
}

Expand Down
14 changes: 7 additions & 7 deletions src/string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1010,7 +1010,7 @@ mrb_str_byte_subseq(mrb_state *mrb, mrb_value str, mrb_int beg, mrb_int len)
cutting can leave a character in pieces, and it can also cut away the
piece that spelled none, so a subrange inherits validity in neither
direction. */
RSTR_COPY_BINARY_FLAG(s, orig);
RSTR_ENC_COPY(s, orig);
return mrb_obj_value(s);
}

Expand Down Expand Up @@ -1107,7 +1107,7 @@ str_replace(mrb_state *mrb, struct RString *s1, struct RString *s2)
RSTR_COPY_SINGLE_BYTE_FLAG(s1, s2);
RSTR_COPY_VALID_ENC_FLAG(s1, s2);
RSTR_COPY_BROKEN_ENC_FLAG(s1, s2);
RSTR_COPY_BINARY_FLAG(s1, s2);
RSTR_ENC_COPY(s1, s2);
if (RSTR_SHARED_P(s1)) {
str_decref(mrb, s1->as.heap.aux.shared);
}
Expand Down Expand Up @@ -1419,7 +1419,7 @@ mrb_str_plus(mrb_state *mrb, mrb_value a, mrb_value b)
if ((RSTR_BINARY_P(s) && RSTR_BINARY_P(s2)) ||
(RSTR_BINARY_P(s) && !str_ascii_p(s)) ||
(RSTR_BINARY_P(s2) && !str_ascii_p(s2))) {
RSTR_SET_BINARY_FLAG(t);
RSTR_ENCODING_SET(t, MRB_STR_ENCODING_BINARY);
}

return mrb_obj_value(t);
Expand Down Expand Up @@ -1505,7 +1505,7 @@ mrb_str_times(mrb_state *mrb, mrb_value self)
RSTR_COPY_VALID_ENC_FLAG(str2, mrb_str_ptr(self));
/* a repetition of a byte-read string holds nothing but its bytes over
again, so it is read the same way */
RSTR_COPY_BINARY_FLAG(str2, mrb_str_ptr(self));
RSTR_ENC_COPY(str2, mrb_str_ptr(self));
/* A repetition of broken bytes reaches the same broken place the first copy
does, so it is broken too. Nought copies keep none of the bytes, and an
empty string is not broken whatever it was made from. */
Expand Down Expand Up @@ -1851,7 +1851,7 @@ str_replace_partial(mrb_state *mrb, mrb_value src, mrb_int pos, mrb_int end, mrb
their reading over, ASCII bytes move nothing */
struct RString *repp = mrb_str_ptr(rep);
if (!RSTR_BINARY_P(str) && RSTR_BINARY_P(repp) && !str_ascii_p(repp)) {
RSTR_SET_BINARY_FLAG(str);
RSTR_ENCODING_SET(str, MRB_STR_ENCODING_BINARY);
}
}
RSTR_SET_LEN(str, newlen);
Expand Down Expand Up @@ -3638,7 +3638,7 @@ mrb_str_cat_str(mrb_state *mrb, mrb_value str, mrb_value str2)
mrb_bool binary = !RSTR_BINARY_P(s) && RSTR_BINARY_P(s2) && !str_ascii_p(s2);
mrb_value ret = mrb_str_cat(mrb, str, RSTRING_PTR(str2), RSTRING_LEN(str2));
if (binary) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(ret));
RSTR_ENCODING_SET(mrb_str_ptr(ret), MRB_STR_ENCODING_BINARY);
}
return ret;
}
Expand Down Expand Up @@ -3879,7 +3879,7 @@ sub_replace(mrb_state *mrb, mrb_value self)
if ((RSTR_BINARY_P(mrb_str_ptr(replace)) && !str_ascii_p(mrb_str_ptr(replace))) ||
(match_taken && RSTR_BINARY_P(mrb_str_ptr(pat)) && !str_ascii_p(mrb_str_ptr(pat))) ||
(self_taken && RSTR_BINARY_P(mrb_str_ptr(self)) && !str_ascii_p(mrb_str_ptr(self)))) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}
return result;
}
Expand Down
Loading