Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 74 additions & 45 deletions include/mruby/string.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,54 +85,83 @@ struct RStringEmbed {
#define RSTR_FSHARED_P(s) ((s)->flags & MRB_STR_FSHARED)
#define RSTR_NOFREE_P(s) ((s)->flags & MRB_STR_NOFREE)

/* What reading the bytes as the encoding they carry has come back with: not
asked yet, nothing but ASCII, read whole and sound, or read and found
broken. The four are exclusive and cover every answer there is, so a string
keeps one of them rather than a flag per answer. UNKNOWN is 0, which is what
a fresh string is already filled with. */
#define MRB_STR_CODERANGE_UNKNOWN 0
#define MRB_STR_CODERANGE_7BIT 1
#define MRB_STR_CODERANGE_VALID 2
#define MRB_STR_CODERANGE_BROKEN 3

#ifdef MRB_UTF8_STRING
# define RSTR_SINGLE_BYTE_P(s) ((s)->flags & MRB_STR_SINGLE_BYTE)
# define RSTR_SET_SINGLE_BYTE_FLAG(s) ((s)->flags |= MRB_STR_SINGLE_BYTE)
# define RSTR_UNSET_SINGLE_BYTE_FLAG(s) ((s)->flags &= ~MRB_STR_SINGLE_BYTE)
# define RSTR_WRITE_SINGLE_BYTE_FLAG(s, v) (RSTR_UNSET_SINGLE_BYTE_FLAG(s), (s)->flags |= v)
# define RSTR_COPY_SINGLE_BYTE_FLAG(dst, src) RSTR_WRITE_SINGLE_BYTE_FLAG(dst, RSTR_SINGLE_BYTE_P(src))
/* Set once a walk has read the whole string as UTF-8, so a later walk can be
skipped. Unlike MRB_STR_SINGLE_BYTE this is not a property a byte subrange
inherits, since a subrange can cut a character in half; copy it only where
the destination ends up holding exactly the source's bytes. */
# define RSTR_VALID_ENC_P(s) ((s)->flags & MRB_STR_VALID_ENC)
# define RSTR_SET_VALID_ENC_FLAG(s) ((s)->flags |= MRB_STR_VALID_ENC)
# define RSTR_UNSET_VALID_ENC_FLAG(s) ((s)->flags &= ~MRB_STR_VALID_ENC)
# define RSTR_COPY_VALID_ENC_FLAG(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_VALID_ENC) | RSTR_VALID_ENC_P(src))
/* The other answer the same walk can come back with, kept so that a string
already read as broken is not read again on every later question. It is
forgotten wherever MRB_STR_VALID_ENC is, since a write to the bytes unmakes
either answer, and it travels only where the whole of the bytes travels. */
# define RSTR_BROKEN_ENC_P(s) ((s)->flags & MRB_STR_BROKEN_ENC)
# define RSTR_SET_BROKEN_ENC_FLAG(s) ((s)->flags |= MRB_STR_BROKEN_ENC)
# define RSTR_UNSET_BROKEN_ENC_FLAG(s) ((s)->flags &= ~MRB_STR_BROKEN_ENC)
# define RSTR_COPY_BROKEN_ENC_FLAG(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_BROKEN_ENC) | RSTR_BROKEN_ENC_P(src))
/* Kept for now in the three bits that held the answers one at a time, one bit
per answer. Nothing writes two of them, so the order below only decides what
a combination no writer makes would read as. 7BIT says more than VALID: a
string of nothing but ASCII reads as UTF-8 as it stands, and it is also one
character per byte, which is the part every index on it wants. */
# define RSTR_CODERANGE(s) \
(((s)->flags & MRB_STR_BROKEN_ENC) ? MRB_STR_CODERANGE_BROKEN : \
((s)->flags & MRB_STR_SINGLE_BYTE) ? MRB_STR_CODERANGE_7BIT : \
((s)->flags & MRB_STR_VALID_ENC) ? MRB_STR_CODERANGE_VALID : \
MRB_STR_CODERANGE_UNKNOWN)
# define RSTR_CODERANGE_SET(s, cr) \
((s)->flags = ((s)->flags & \
~(MRB_STR_SINGLE_BYTE|MRB_STR_VALID_ENC|MRB_STR_BROKEN_ENC)) | \
(((cr) == MRB_STR_CODERANGE_7BIT) ? MRB_STR_SINGLE_BYTE : \
((cr) == MRB_STR_CODERANGE_VALID) ? MRB_STR_VALID_ENC : \
((cr) == MRB_STR_CODERANGE_BROKEN) ? MRB_STR_BROKEN_ENC : 0))
#else
# define RSTR_SINGLE_BYTE_P(s) TRUE
# define RSTR_SET_SINGLE_BYTE_FLAG(s) (void)0
# define RSTR_UNSET_SINGLE_BYTE_FLAG(s) (void)0
# define RSTR_WRITE_SINGLE_BYTE_FLAG(s, v) (void)0
# define RSTR_COPY_SINGLE_BYTE_FLAG(dst, src) (void)0
# define RSTR_VALID_ENC_P(s) TRUE
# define RSTR_SET_VALID_ENC_FLAG(s) (void)0
# define RSTR_UNSET_VALID_ENC_FLAG(s) (void)0
# define RSTR_COPY_VALID_ENC_FLAG(dst, src) (void)0
# define RSTR_BROKEN_ENC_P(s) FALSE
# define RSTR_SET_BROKEN_ENC_FLAG(s) (void)0
# define RSTR_UNSET_BROKEN_ENC_FLAG(s) (void)0
# define RSTR_COPY_BROKEN_ENC_FLAG(dst, src) (void)0
/* A build that indexes by byte hands every byte back as a character and asks
the bytes nothing, so every string in it stands where 7BIT stands and there
is nothing to record. */
# define RSTR_CODERANGE(s) MRB_STR_CODERANGE_7BIT
# define RSTR_CODERANGE_SET(s, cr) ((void)0)
#endif
#define RSTR_SET_ASCII_FLAG(s) RSTR_SET_SINGLE_BYTE_FLAG(s)
#define RSTR_BINARY_P(s) ((s)->flags & MRB_STR_BINARY)
#define RSTR_SET_BINARY_FLAG(s) ((s)->flags |= MRB_STR_BINARY)
#define RSTR_UNSET_BINARY_FLAG(s) ((s)->flags &= ~MRB_STR_BINARY)
/* A copy of a string is byte-indexed exactly when the string it copies is, so
the flag travels with the bytes rather than being left behind on the
original. */
#define RSTR_COPY_BINARY_FLAG(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_BINARY) | RSTR_BINARY_P(src))

/* The encoding a string's bytes are read as, named as an index into the set of
encodings the build carries. Index 0 is that build's default, which is what
a string that says nothing else holds: the literals the parser hands over,
the strings numbers and times spell themselves with. The zero a fresh string
is filled with therefore already says the default, and the path every string
is made on stores nothing.

A build without MRB_UTF8_STRING carries no UTF-8, so it names none: writing
the name there is a compile error rather than a quiet no-op. Such a build
still tells a byte-read string from a default one, since String#b and
Integer#chr mark one there too. */
#define MRB_STR_ENCODING_DEFAULT 0
#define MRB_STR_ENCODING_BINARY 1
#ifdef MRB_UTF8_STRING
# define MRB_STR_ENCODING_UTF8 MRB_STR_ENCODING_DEFAULT
#endif

#define RSTR_ENCODING(s) \
(((s)->flags & MRB_STR_BINARY) ? MRB_STR_ENCODING_BINARY : MRB_STR_ENCODING_DEFAULT)
#define RSTR_ENCODING_SET(s, e) \
((s)->flags = ((s)->flags & ~MRB_STR_BINARY) | \
(((e) == MRB_STR_ENCODING_BINARY) ? MRB_STR_BINARY : 0))
#define RSTR_BINARY_P(s) (RSTR_ENCODING(s) == MRB_STR_ENCODING_BINARY)
/* A copy of a string is read the way the string it copies is, so the encoding
travels with the bytes rather than being left behind on the original. */
#define RSTR_ENC_COPY(dst, src) RSTR_ENCODING_SET(dst, RSTR_ENCODING(src))
/* A copy that ends up holding exactly the source's bytes reads them the same
way and stands exactly where the source stands, so the two answers travel
together. Splitting them apart would let a copy keep one and drop the
other, which is the way flags went missing when there was a macro per
flag. */
#define RSTR_ENC_CR_COPY(dst, src) \
(RSTR_ENC_COPY(dst, src), RSTR_CODERANGE_SET(dst, RSTR_CODERANGE(src)))
/* A subrange holds bytes of the source, so it is read the same way, but a cut
can leave a character in pieces and can also cut away the piece that spelled
none: it inherits neither soundness nor brokenness. Nothing but ASCII is
what survives being cut anywhere, so that is the one answer it carries
over. */
#define RSTR_ENC_CR_COPY_FOR_SUBSTR(dst, src) \
(RSTR_ENC_COPY(dst, src), \
RSTR_CODERANGE_SET(dst, (RSTR_CODERANGE(src) == MRB_STR_CODERANGE_7BIT) \
? MRB_STR_CODERANGE_7BIT : MRB_STR_CODERANGE_UNKNOWN))

/**
* Returns a pointer from a Ruby string
Expand Down
4 changes: 2 additions & 2 deletions mrbgems/mruby-encoding/src/encoding.c
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ str_force_encoding(mrb_state *mrb, mrb_value self)
struct RString *s = mrb_str_ptr(self);
if (MRB_STR_CASECMP_P(enc, ENC_ASCII_8BIT) ||
MRB_STR_CASECMP_P(enc, ENC_BINARY)) {
RSTR_SET_BINARY_FLAG(s);
RSTR_ENCODING_SET(s, MRB_STR_ENCODING_BINARY);
}
else if (MRB_STR_CASECMP_P(enc, ENC_UTF8)) {
RSTR_UNSET_BINARY_FLAG(s);
RSTR_ENCODING_SET(s, MRB_STR_ENCODING_UTF8);
}
else {
mrb_raisef(mrb, E_ARGUMENT_ERROR, "unknown encoding name - %v", enc);
Expand Down
2 changes: 1 addition & 1 deletion mrbgems/mruby-regexp/src/regexp.c
Original file line number Diff line number Diff line change
Expand Up @@ -1205,7 +1205,7 @@ re_mark_spliced(mrb_value result, mrb_value subject, mrb_value replacement,
while (p < e && !(*p & 0x80)) p++;
if (p == e) return;
}
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}

/*
Expand Down
4 changes: 2 additions & 2 deletions mrbgems/mruby-sprintf/src/sprintf.c
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ mark_written_bytes(mrb_value result, mrb_value src)
const char *p = RSTR_PTR(s);
const char *e = p + RSTR_LEN(s);
while (p < e && !(*p & 0x80)) p++;
if (p < e) RSTR_SET_BINARY_FLAG(r);
if (p < e) RSTR_ENCODING_SET(r, MRB_STR_ENCODING_BINARY);
}

static mrb_value
Expand Down Expand Up @@ -423,7 +423,7 @@ mrb_str_format(mrb_state *mrb, mrb_int argc, const mrb_value *argv, mrb_value fm
built out of them is read the way the format string was read, ASCII bytes
and all: the reading a receiver holds, which nothing written into it
lifts. */
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(result), mrb_str_ptr(fmt));
RSTR_ENC_COPY(mrb_str_ptr(result), mrb_str_ptr(fmt));
buf = RSTRING_PTR(result);
memset(buf, 0, bsiz);

Expand Down
2 changes: 1 addition & 1 deletion mrbgems/mruby-string-bitops/src/string_bitops.c
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ static mrb_value
bitop_result_str(mrb_state *mrb, mrb_int len)
{
mrb_value result = mrb_str_new(mrb, NULL, len);
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
return result;
}

Expand Down
36 changes: 18 additions & 18 deletions mrbgems/mruby-string-ext/src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ int_chr_binary(mrb_state *mrb, mrb_value num)
handing the byte back as a character. What it is instead is a string read
by bytes, which is what MRB_STR_BINARY says. */
if (cp < 0x80) {
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
}
else {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(str));
RSTR_ENCODING_SET(mrb_str_ptr(str), MRB_STR_ENCODING_BINARY);
}
return str;
}
Expand All @@ -53,7 +53,7 @@ int_chr_utf8(mrb_state *mrb, mrb_value num)
}
str = mrb_str_new(mrb, utf8, len);
if (len == 1) {
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
}
return str;
}
Expand Down Expand Up @@ -125,7 +125,7 @@ str_mark_spliced_bytes(mrb_value recv, mrb_value src)
const char *p = RSTR_PTR(s);
const char *e = p + RSTR_LEN(s);
while (p < e && !(*p & 0x80)) p++;
if (p < e) RSTR_SET_BINARY_FLAG(r);
if (p < e) RSTR_ENCODING_SET(r, MRB_STR_ENCODING_BINARY);
}

static void
Expand Down Expand Up @@ -1033,7 +1033,7 @@ str_ord(mrb_state* mrb, mrb_value str)
if (p == e) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "empty string");
}
if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
c = p[0];
}
else {
Expand Down Expand Up @@ -1123,7 +1123,7 @@ str_scrub_core(mrb_state *mrb, mrb_value self)
}

struct RString *s = mrb_str_ptr(self);
if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
return mrb_str_dup(mrb, self);
}

Expand Down Expand Up @@ -1166,7 +1166,7 @@ str_scrub_chunks(mrb_state *mrb, mrb_value self)
{
mrb_value ary = mrb_ary_new(mrb);
struct RString *s = mrb_str_ptr(self);
if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
mrb_ary_push(mrb, ary, mrb_str_dup(mrb, self));
return ary;
}
Expand Down Expand Up @@ -1201,7 +1201,7 @@ str_codepoints(mrb_state *mrb, mrb_value str)

mrb->c->ci->mid = 0;
mrb_value result = mrb_ary_new(mrb);
if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
while (p < e) {
mrb_ary_push(mrb, result, mrb_int_value(mrb, (mrb_int)*p));
p++;
Expand Down Expand Up @@ -1459,7 +1459,7 @@ str_lines(mrb_state *mrb, mrb_value self)
/* a line of a byte-read string is a subrange of its bytes, read the
same way */
mrb_value line = mrb_str_new(mrb, t, len);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(line), mrb_str_ptr(self));
RSTR_ENC_COPY(mrb_str_ptr(line), mrb_str_ptr(self));
mrb_ary_push(mrb, result, line);
mrb_gc_arena_restore(mrb, ai);
}
Expand Down Expand Up @@ -1513,7 +1513,7 @@ str_ascii_only_p(mrb_state *mrb, mrb_value str)
if (*p & 0x80) return mrb_false_value();
p++;
}
RSTR_SET_ASCII_FLAG(s);
RSTR_CODERANGE_SET(s, MRB_STR_CODERANGE_7BIT);
return mrb_true_value();
}

Expand All @@ -1522,7 +1522,7 @@ static mrb_value
str_b(mrb_state *mrb, mrb_value self)
{
mrb_value str = mrb_str_dup(mrb, self);
RSTR_SET_BINARY_FLAG(mrb_str_ptr(str));
RSTR_ENCODING_SET(mrb_str_ptr(str), MRB_STR_ENCODING_BINARY);
return str;
}

Expand Down Expand Up @@ -1788,11 +1788,11 @@ str_chars_ary(mrb_state *mrb, mrb_value self)
const char *p = RSTR_PTR(s);
const char *e = p + RSTR_LEN(s);
/* the count comes first: it is the exact capacity, and where every byte is
ASCII it also settles the single-byte flag the walk reads */
ASCII it also settles at 7BIT the answer the walk reads */
mrb_value result = mrb_ary_new_capa(mrb, mrb_str_char_len(mrb, self));

#ifdef MRB_UTF8_STRING
if (!RSTR_SINGLE_BYTE_P(s) && !RSTR_BINARY_P(s)) {
if (RSTR_CODERANGE(s) != MRB_STR_CODERANGE_7BIT && !RSTR_BINARY_P(s)) {
while (p < e) {
mrb_int char_len = mrb_utf8len(p, e);
mrb_ary_push(mrb, result, mrb_str_new(mrb, p, char_len));
Expand All @@ -1806,7 +1806,7 @@ str_chars_ary(mrb_state *mrb, mrb_value self)
way, so the marking goes with each one */
while (p < e) {
mrb_value piece = mrb_str_new(mrb, p, 1);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(piece), s);
RSTR_ENC_COPY(mrb_str_ptr(piece), s);
mrb_ary_push(mrb, result, piece);
p++;
}
Expand Down Expand Up @@ -1914,7 +1914,7 @@ str_rjust_core(mrb_state *mrb, mrb_value self)
/* the padded string is the receiver's bytes in wider clothes, so it is
read the way the receiver was, ASCII bytes and all */
if (RSTR_BINARY_P(mrb_str_ptr(self))) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}
return result;
}
Expand Down Expand Up @@ -1988,7 +1988,7 @@ str_center_core(mrb_state *mrb, mrb_value self)
/* the padded string is the receiver's bytes in wider clothes, so it is
read the way the receiver was, ASCII bytes and all */
if (RSTR_BINARY_P(mrb_str_ptr(self))) {
RSTR_SET_BINARY_FLAG(mrb_str_ptr(result));
RSTR_ENCODING_SET(mrb_str_ptr(result), MRB_STR_ENCODING_BINARY);
}
return result;
}
Expand Down Expand Up @@ -2058,7 +2058,7 @@ mrb_str_slice_bang(mrb_state *mrb, mrb_value self)
/* the piece cut out is a subrange of the receiver's bytes, read the same
way; copied rather than shared, since the memmove below would slide the
receiver's remaining bytes through a shared buffer */
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(result), str);
RSTR_ENC_COPY(mrb_str_ptr(result), str);

mrb_str_modify(mrb, str);
ptr = RSTRING_PTR(self);
Expand Down Expand Up @@ -2093,7 +2093,7 @@ static mrb_value
str_cut_piece(mrb_state *mrb, mrb_value str, const char *p, mrb_int len)
{
mrb_value piece = mrb_str_new(mrb, p, len);
RSTR_COPY_BINARY_FLAG(mrb_str_ptr(piece), mrb_str_ptr(str));
RSTR_ENC_COPY(mrb_str_ptr(piece), mrb_str_ptr(str));
return piece;
}

Expand Down
2 changes: 1 addition & 1 deletion mrbgems/mruby-time/src/time.c
Original file line number Diff line number Diff line change
Expand Up @@ -1489,7 +1489,7 @@ time_to_s(mrb_state *mrb, mrb_value self)
#endif
}
mrb_value str = mrb_str_new(mrb, buf, len);
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down
4 changes: 2 additions & 2 deletions src/numeric.c
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ flo_to_s(mrb_state *mrb, mrb_value flt)
str = mrb_float_to_str(mrb, flt, NULL);
}

RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down Expand Up @@ -2038,7 +2038,7 @@ mrb_integer_to_str(mrb_state *mrb, mrb_value x, mrb_int base)
const char *p = mrb_int_to_cstr(buf, sizeof(buf), val, base);
mrb_assert(p != NULL);
mrb_value str = mrb_str_new_cstr(mrb, p);
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down
8 changes: 4 additions & 4 deletions src/object.c
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,15 @@ static mrb_value
nil_to_s(mrb_state *mrb, mrb_value obj)
{
mrb_value str = mrb_str_new_frozen(mrb, NULL, 0);
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

static mrb_value
nil_inspect(mrb_state *mrb, mrb_value obj)
{
mrb_value str = mrb_str_new_lit_frozen(mrb, "nil");
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down Expand Up @@ -224,7 +224,7 @@ static mrb_value
true_to_s(mrb_state *mrb, mrb_value obj)
{
mrb_value str = mrb_str_new_lit_frozen(mrb, "true");
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down Expand Up @@ -333,7 +333,7 @@ static mrb_value
false_to_s(mrb_state *mrb, mrb_value obj)
{
mrb_value str = mrb_str_new_lit_frozen(mrb, "false");
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
RSTR_CODERANGE_SET(mrb_str_ptr(str), MRB_STR_CODERANGE_7BIT);
return str;
}

Expand Down
Loading
Loading