Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions include/mruby/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,74 @@ size_t mrb_gc_mark_range(mrb_state *mrb, struct RRange *r);
#endif

/* string */

/* Writing what a string's bytes are read as, and what reading them came back
with. mruby/string.h hands both fields back to anyone who asks, since what
they hold is a fact about the string and reading a fact costs the string
nothing. Writing one is the other thing: it is a claim, and a claim the
bytes do not support is caught nowhere. A wrong encoding index has the bytes
read as something they are not, and a string wrongly saying it reads whole
and sound walks straight through the check a regexp makes of its subject. So
the writes are offered where they can be answered for, which is inside the
library, rather than to whoever includes a header.

The values themselves stay in mruby/string.h: naming an answer is reading,
and what reads MRB_STR_CODERANGE_7BIT off a string has to be able to say
it. */
#ifdef MRB_UTF8_STRING
/* An answer is masked to the field's width on the way in, as an encoding index
is, so a fifth one lands wrong rather than reaching the bits beside it. Here
those bits are the encoding index rather than free ones, so an unmasked
write would not merely be a wrong answer: it would have the bytes read as
another encoding. What is written is one of the four either way, spelled
outright or read back out of another string's field, so nothing is left of
this at -O3. */
# define RSTR_CODERANGE_SET(s, cr) \
((s)->flags = ((s)->flags & ~MRB_STR_CODERANGE_MASK) | \
(((cr) & ((1 << MRB_STR_CODERANGE_BITS) - 1)) << MRB_STR_CODERANGE_SHIFT))
#else
/* A build that indexes by byte hands every byte back as a character and asks
the bytes nothing, so every string in it stands where 7BIT stands and there
is nothing to record. */
# define RSTR_CODERANGE_SET(s, cr) ((void)0)
#endif

/* The index is masked to the width of the field it goes into, so an index the
field is too narrow for lands wrong rather than reaching the bits beside it.
Widening MRB_STR_ENCODING_BITS is what a build carrying that many encodings
needs; until then this keeps the mistake where it can be seen. Both operands
are constants at every call, so nothing is left of this at -O3. */
#define RSTR_ENCODING_SET(s, e) \
((s)->flags = ((s)->flags & ~MRB_STR_ENCODING_MASK) | \
(((e) & ((1 << MRB_STR_ENCODING_BITS) - 1)) << MRB_STR_ENCODING_SHIFT))
/* A copy of a string is read the way the string it copies is, so the encoding
travels with the bytes rather than being left behind on the original. */
#define RSTR_ENC_COPY(dst, src) RSTR_ENCODING_SET(dst, RSTR_ENCODING(src))
/* A copy that ends up holding exactly the source's bytes reads them the same
way and stands exactly where the source stands, so the two answers travel
together. Splitting them apart would let a copy keep one and drop the
other, which is the way flags went missing when there was a macro per
flag.

The two fields sit side by side, so one mask spells both and the pair
crosses in a single read and a single write rather than one of each per
field. A build that indexes by byte keeps no coderange and writes those
bits nowhere, so what the mask carries across there is the zeros they
hold. */
#define MRB_STR_ENC_CR_MASK (MRB_STR_ENCODING_MASK|MRB_STR_CODERANGE_MASK)
#define RSTR_ENC_CR_COPY(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_ENC_CR_MASK) | \
((src)->flags & MRB_STR_ENC_CR_MASK))
/* A subrange holds bytes of the source, so it is read the same way, but a cut
can leave a character in pieces and can also cut away the piece that spelled
none: it inherits neither soundness nor brokenness. Nothing but ASCII is
what survives being cut anywhere, so that is the one answer it carries
over. */
#define RSTR_ENC_CR_COPY_FOR_SUBSTR(dst, src) \
(RSTR_ENC_COPY(dst, src), \
RSTR_CODERANGE_SET(dst, (RSTR_CODERANGE(src) == MRB_STR_CODERANGE_7BIT) \
? MRB_STR_CODERANGE_7BIT : MRB_STR_CODERANGE_UNKNOWN))

void mrb_gc_free_str(mrb_state*, struct RString*);
uint32_t mrb_str_hash(mrb_state *mrb, mrb_value str);
mrb_value mrb_str_dump(mrb_state *mrb, mrb_value str);
Expand Down
50 changes: 11 additions & 39 deletions include/mruby/string.h
Original file line number Diff line number Diff line change
Expand Up @@ -141,15 +141,11 @@ struct RStringEmbed {
this at -O3. */
# define RSTR_CODERANGE(s) \
(((s)->flags & MRB_STR_CODERANGE_MASK) >> MRB_STR_CODERANGE_SHIFT)
# define RSTR_CODERANGE_SET(s, cr) \
((s)->flags = ((s)->flags & ~MRB_STR_CODERANGE_MASK) | \
(((cr) & ((1 << MRB_STR_CODERANGE_BITS) - 1)) << MRB_STR_CODERANGE_SHIFT))
#else
/* A build that indexes by byte hands every byte back as a character and asks
the bytes nothing, so every string in it stands where 7BIT stands and there
is nothing to record. */
# define RSTR_CODERANGE(s) MRB_STR_CODERANGE_7BIT
# define RSTR_CODERANGE_SET(s, cr) ((void)0)
#endif

/* The encoding a string's bytes are read as, named as an index into the set of
Expand All @@ -171,42 +167,18 @@ struct RStringEmbed {

#define RSTR_ENCODING(s) \
(((s)->flags & MRB_STR_ENCODING_MASK) >> MRB_STR_ENCODING_SHIFT)
/* The index is masked to the width of the field it goes into, so an index the
field is too narrow for lands wrong rather than reaching the bits beside it.
Widening MRB_STR_ENCODING_BITS is what a build carrying that many encodings
needs; until then this keeps the mistake where it can be seen. Both operands
are constants at every call, so nothing is left of this at -O3. */
#define RSTR_ENCODING_SET(s, e) \
((s)->flags = ((s)->flags & ~MRB_STR_ENCODING_MASK) | \
(((e) & ((1 << MRB_STR_ENCODING_BITS) - 1)) << MRB_STR_ENCODING_SHIFT))
#define RSTR_BINARY_P(s) (RSTR_ENCODING(s) == MRB_STR_ENCODING_BINARY)
/* A copy of a string is read the way the string it copies is, so the encoding
travels with the bytes rather than being left behind on the original. */
#define RSTR_ENC_COPY(dst, src) RSTR_ENCODING_SET(dst, RSTR_ENCODING(src))
/* A copy that ends up holding exactly the source's bytes reads them the same
way and stands exactly where the source stands, so the two answers travel
together. Splitting them apart would let a copy keep one and drop the
other, which is the way flags went missing when there was a macro per
flag.

The two fields sit side by side, so one mask spells both and the pair
crosses in a single read and a single write rather than one of each per
field. A build that indexes by byte keeps no coderange and writes those
bits nowhere, so what the mask carries across there is the zeros they
hold. */
#define MRB_STR_ENC_CR_MASK (MRB_STR_ENCODING_MASK|MRB_STR_CODERANGE_MASK)
#define RSTR_ENC_CR_COPY(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_ENC_CR_MASK) | \
((src)->flags & MRB_STR_ENC_CR_MASK))
/* A subrange holds bytes of the source, so it is read the same way, but a cut
can leave a character in pieces and can also cut away the piece that spelled
none: it inherits neither soundness nor brokenness. Nothing but ASCII is
what survives being cut anywhere, so that is the one answer it carries
over. */
#define RSTR_ENC_CR_COPY_FOR_SUBSTR(dst, src) \
(RSTR_ENC_COPY(dst, src), \
RSTR_CODERANGE_SET(dst, (RSTR_CODERANGE(src) == MRB_STR_CODERANGE_7BIT) \
? MRB_STR_CODERANGE_7BIT : MRB_STR_CODERANGE_UNKNOWN))
/* Whether a character index into this string is already a byte index: every
byte of it stands for a character of its own. That is so where the bytes are
nothing but ASCII, and so where they are read as bytes to begin with. The
two arrive at it from different sides, which is why this is derived from
what the string carries rather than carried alongside it. */
#define RSTR_SINGLE_BYTE_P(s) \
(RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s))

/* Writing either field is in mruby/internal.h. What is read back here is what
the bytes were found to be; what is written there is a claim about them,
which only what can make good on it should be spelling. */

/**
* Returns a pointer from a Ruby string
Expand Down
10 changes: 5 additions & 5 deletions mrbgems/mruby-string-ext/src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -1033,7 +1033,7 @@ str_ord(mrb_state* mrb, mrb_value str)
if (p == e) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "empty string");
}
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
c = p[0];
}
else {
Expand Down Expand Up @@ -1123,7 +1123,7 @@ str_scrub_core(mrb_state *mrb, mrb_value self)
}

struct RString *s = mrb_str_ptr(self);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
return mrb_str_dup(mrb, self);
}

Expand Down Expand Up @@ -1166,7 +1166,7 @@ str_scrub_chunks(mrb_state *mrb, mrb_value self)
{
mrb_value ary = mrb_ary_new(mrb);
struct RString *s = mrb_str_ptr(self);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
mrb_ary_push(mrb, ary, mrb_str_dup(mrb, self));
return ary;
}
Expand Down Expand Up @@ -1201,7 +1201,7 @@ str_codepoints(mrb_state *mrb, mrb_value str)

mrb->c->ci->mid = 0;
mrb_value result = mrb_ary_new(mrb);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
while (p < e) {
mrb_ary_push(mrb, result, mrb_int_value(mrb, (mrb_int)*p));
p++;
Expand Down Expand Up @@ -1799,7 +1799,7 @@ str_chars_ary(mrb_state *mrb, mrb_value self)
int ai = mrb_gc_arena_save(mrb);

#ifdef MRB_UTF8_STRING
if (RSTR_CODERANGE(s) != MRB_STR_CODERANGE_7BIT && !RSTR_BINARY_P(s)) {
if (!RSTR_SINGLE_BYTE_P(s)) {
while (p < e) {
mrb_int char_len = mrb_utf8len(p, e);
mrb_ary_push(mrb, result, mrb_str_new(mrb, p, char_len));
Expand Down
11 changes: 5 additions & 6 deletions src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -728,7 +728,7 @@ mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
{
(void)mrb;
struct RString *s = mrb_str_ptr(str);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
return idx;
}

Expand Down Expand Up @@ -769,7 +769,7 @@ mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi)
(void)mrb;
struct RString *s = mrb_str_ptr(str);
if (bi < 0 || RSTR_LEN(s) < bi) return -1;
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
return bi;
}

Expand Down Expand Up @@ -2125,8 +2125,7 @@ mrb_str_chomp_bang(mrb_state *mrb, mrb_value str)
a character of its own, and cutting there would leave a string that is
not UTF-8: "あ".chomp("\x82") is the whole of the last byte of a
three-byte character. CRuby reads that as no match. */
if (!RSTR_BINARY_P(s) && RSTR_CODERANGE(s) != MRB_STR_CODERANGE_7BIT &&
mrb_utf8_char_head(p, pp, p + len) != pp) {
if (!RSTR_SINGLE_BYTE_P(s) && mrb_utf8_char_head(p, pp, p + len) != pp) {
return mrb_nil_value();
}
#endif
Expand Down Expand Up @@ -2437,7 +2436,7 @@ mrb_str_check_byte_pos(mrb_state *mrb, mrb_value str, mrb_int pos)
{
#ifdef MRB_UTF8_STRING
struct RString *s = mrb_str_ptr(str);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) return;
if (RSTR_SINGLE_BYTE_P(s)) return;

const char *b = RSTR_PTR(s);
const char *p = b + pos;
Expand Down Expand Up @@ -2791,7 +2790,7 @@ static mrb_value
mrb_str_rindex_m(mrb_state *mrb, mrb_value str)
{
struct RString *s = mrb_str_ptr(str);
if (RSTR_CODERANGE(s) == MRB_STR_CODERANGE_7BIT || RSTR_BINARY_P(s)) {
if (RSTR_SINGLE_BYTE_P(s)) {
return mrb_str_byterindex_m(mrb, str);
}

Expand Down
Loading