Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions include/mruby/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,9 @@ mrb_bool mrb_strcasecmp_p(const char *s1, mrb_int len1, const char *s2, mrb_int
uint32_t mrb_byte_hash(const uint8_t*, mrb_int);
uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);

/* Character-index/byte-offset conversion, honoring the string's own indexing
(single-byte and binary strings index by byte). mrb_str_char_to_byte returns
/* Character count and character-index/byte-offset conversion, honoring the
string's own indexing (single-byte and binary strings index by byte, so
their character count is the byte length). mrb_str_char_to_byte returns
the byte length of `nchars` characters starting at byte offset `off`; when
the string ends before `nchars` characters, the remaining byte length plus
one is returned so out-of-range requests stay detectable. mrb_str_byte_to_char
Expand All @@ -178,6 +179,7 @@ uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);
character. On non-UTF-8 builds a byte is a character, so the conversions are
identity within the string, and mrb_str_byte_to_char still rejects an offset
outside it. */
mrb_int mrb_str_char_len(mrb_state *mrb, mrb_value str);
mrb_int mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int nchars);
mrb_int mrb_str_byte_to_char(mrb_state *mrb, mrb_value str, mrb_int bi);

Expand Down
121 changes: 15 additions & 106 deletions mrbgems/mruby-string-ext/src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -1741,26 +1741,6 @@ str_strip_bang(mrb_state *mrb, mrb_value self)
return self;
}

/* Internal helper to count UTF-8 characters in a string using mruby's standard function */
static mrb_int
str_char_count(mrb_value str)
{
#ifdef MRB_UTF8_STRING
struct RString *s = mrb_str_ptr(str);

if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
/* ASCII/Binary: each byte is a character */
return RSTR_LEN(s);
}

/* UTF-8: use mruby's standard UTF-8 character counting function */
return mrb_utf8_strlen(RSTR_PTR(s), RSTR_LEN(s));
#else
/* Non-UTF8 build: treat as single bytes */
return RSTRING_LEN(str);
#endif
}

/* Internal fast path for String#chars - returns array of individual characters */
static mrb_value
str_chars_ary(mrb_state *mrb, mrb_value self)
Expand Down Expand Up @@ -1845,13 +1825,13 @@ str_ljust_core(mrb_state *mrb, mrb_value self)
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}

mrb_int char_len = str_char_count(self);
mrb_int char_len = mrb_str_char_len(mrb, self);
if (width <= char_len) {
return mrb_str_dup(mrb, self);
}

mrb_int padsize = width - char_len;
mrb_int pad_char_len = str_char_count(padstr);
mrb_int pad_char_len = mrb_str_char_len(mrb, padstr);
if (pad_char_len == 0) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}
Expand Down Expand Up @@ -1895,13 +1875,13 @@ str_rjust_core(mrb_state *mrb, mrb_value self)
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}

mrb_int char_len = str_char_count(self);
mrb_int char_len = mrb_str_char_len(mrb, self);
if (width <= char_len) {
return mrb_str_dup(mrb, self);
}

mrb_int padsize = width - char_len;
mrb_int pad_char_len = str_char_count(padstr);
mrb_int pad_char_len = mrb_str_char_len(mrb, padstr);
if (pad_char_len == 0) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}
Expand Down Expand Up @@ -1945,7 +1925,7 @@ str_center_core(mrb_state *mrb, mrb_value self)
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}

mrb_int char_len = str_char_count(self);
mrb_int char_len = mrb_str_char_len(mrb, self);
if (width <= char_len) {
return mrb_str_dup(mrb, self);
}
Expand All @@ -1954,7 +1934,7 @@ str_center_core(mrb_state *mrb, mrb_value self)
mrb_int left_pad = total_pad / 2;
mrb_int right_pad = total_pad - left_pad;

mrb_int pad_char_len = str_char_count(padstr);
mrb_int pad_char_len = mrb_str_char_len(mrb, padstr);
if (pad_char_len == 0) {
mrb_raise(mrb, E_ARGUMENT_ERROR, "zero width padding");
}
Expand Down Expand Up @@ -1993,67 +1973,6 @@ str_center_core(mrb_state *mrb, mrb_value self)
return mrb_str_cat_str(mrb, result, right_padding);
}

#ifdef MRB_UTF8_STRING
/*
* Given a character index, find the byte offset in a UTF-8 string.
* Returns -1 if the character index is out of bounds.
*/
static mrb_int
str_char_to_byte_offset(mrb_value str, mrb_int char_index)
{
struct RString *s = mrb_str_ptr(str);
const char *p = RSTR_PTR(s);
mrb_int byte_len = RSTR_LEN(s);

if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
return char_index;
}

if (char_index < 0) return -1;

mrb_int byte_offset = 0;
mrb_int current_char_index = 0;
while (byte_offset < byte_len && current_char_index < char_index) {
mrb_int char_len = mrb_utf8len(p + byte_offset, p + byte_len - byte_offset);
if (char_len == 0) break;
byte_offset += char_len;
current_char_index++;
}

if (current_char_index < char_index) return -1;
return byte_offset;
}

/*
* Given a starting character index and a character length, find the byte length.
*/
static mrb_int
str_chars_to_byte_len(mrb_value str, mrb_int char_start, mrb_int char_len)
{
struct RString *s = mrb_str_ptr(str);
const char *p = RSTR_PTR(s);
mrb_int str_byte_len = RSTR_LEN(s);

if (RSTR_SINGLE_BYTE_P(s) || RSTR_BINARY_P(s)) {
return char_len;
}

mrb_int start_byte_offset = str_char_to_byte_offset(str, char_start);
if (start_byte_offset == -1) return 0;

mrb_int byte_offset = start_byte_offset;
mrb_int current_char_len = 0;
while (byte_offset < str_byte_len && current_char_len < char_len) {
mrb_int cl = mrb_utf8len(p + byte_offset, p + str_byte_len - byte_offset);
if (cl == 0) break;
byte_offset += cl;
current_char_len++;
}

return byte_offset - start_byte_offset;
}
#endif

static mrb_value
mrb_str_slice_bang(mrb_state *mrb, mrb_value self)
{
Expand All @@ -2065,25 +1984,20 @@ mrb_str_slice_bang(mrb_state *mrb, mrb_value self)
struct RString *str = mrb_str_ptr(self);
const char *ptr = RSTRING_PTR(self);

#ifdef MRB_UTF8_STRING
mrb_int str_len = str_char_count(self);
#else
mrb_int str_len = RSTRING_LEN(self);
#endif
mrb_int str_len = mrb_str_char_len(mrb, self);

mrb_int beg, len;

if (argc == 1) {
if (mrb_string_p(arg1)) {
mrb_int pos = mrb_str_index(mrb, self, RSTRING_PTR(arg1), RSTRING_LEN(arg1), 0);
if (pos == -1) return mrb_nil_value();
#ifdef MRB_UTF8_STRING
beg = str_char_count(mrb_str_substr(mrb, self, 0, pos));
len = str_char_count(arg1);
#else
beg = pos;
len = RSTRING_LEN(arg1);
#endif
/* The search runs over bytes, so a match may start inside a character;
mrb_str_byte_to_char() answers -1 there, and a match no character
index reaches is no match. */
beg = mrb_str_byte_to_char(mrb, self, pos);
if (beg < 0) return mrb_nil_value();
len = mrb_str_char_len(mrb, arg1);
}
else if (mrb_range_p(arg1)) {
if (mrb_range_beg_len(mrb, arg1, &beg, &len, str_len, TRUE) != MRB_RANGE_OK) {
Expand Down Expand Up @@ -2111,13 +2025,8 @@ mrb_str_slice_bang(mrb_state *mrb, mrb_value self)
}
if (len < 0) len = 0;

#ifdef MRB_UTF8_STRING
mrb_int byte_beg = str_char_to_byte_offset(self, beg);
mrb_int byte_len = str_chars_to_byte_len(self, beg, len);
#else
mrb_int byte_beg = beg;
mrb_int byte_len = len;
#endif
mrb_int byte_beg = mrb_str_char_to_byte(mrb, self, 0, beg);
mrb_int byte_len = mrb_str_char_to_byte(mrb, self, byte_beg, len);

if (byte_beg < 0 || byte_beg > RSTRING_LEN(self) || byte_beg + byte_len > RSTRING_LEN(self)) {
return mrb_nil_value();
Expand Down
46 changes: 46 additions & 0 deletions mrbgems/mruby-string-ext/test/string.rb
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,52 @@ def assert_upto(exp, receiver, *args)
assert_raise(ArgumentError) { "foo".slice! }
end

assert('String#slice! with multibyte characters') do
a = "あいうえお"
assert_equal "えお", a.slice!(3, 2)
assert_equal "あいう", a

a = "あいう"
assert_equal "いう", a.slice!(1..2)
assert_equal "あ", a

a = "あいう"
assert_equal "う", a.slice!(-1)
assert_equal "あい", a

a = "aあいb"
assert_equal "あい", a.slice!(1, 2)
assert_equal "ab", a

a = "あい"
assert_equal "あい", a.slice!(0, 2)
assert_equal "", a

a = "あい"
assert_equal "", a.slice!(2, 1)
assert_equal "あい", a
end if UTF8STRING

assert('String#slice! with a multibyte match') do
a = "あいう"
assert_equal "い", a.slice!("い")
assert_equal "あう", a

a = "あいう"
assert_equal "う", a.slice!("う")
assert_equal "あい", a

a = "あいう"
assert_nil a.slice!("え")
assert_equal "あいう", a

# the search runs over bytes, so a match starting inside a character is
# not a match
a = "あ"
assert_nil a.slice!("\x81\x82")
assert_equal "あ", a
end if UTF8STRING

assert('String#succ') do
assert_equal "", "".succ
assert_equal "1", "0".succ
Expand Down
31 changes: 19 additions & 12 deletions src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -546,9 +546,11 @@ mrb_utf8_strlen(const char *str, mrb_int byte_len)
return len;
}

static mrb_int
utf8_strlen(mrb_value str)
/* count the characters of a string */
mrb_int
mrb_str_char_len(mrb_state *mrb, mrb_value str)
{
(void)mrb;
struct RString *s = mrb_str_ptr(str);
mrb_int byte_len = RSTR_LEN(s);

Expand Down Expand Up @@ -576,8 +578,6 @@ utf8_strlen(mrb_value str)
}
}

#define RSTRING_CHAR_LEN(s) utf8_strlen(s)

/* map character index to byte offset index */
mrb_int
mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
Expand Down Expand Up @@ -690,8 +690,15 @@ str_index_str_by_char(mrb_state *mrb, mrb_value str, mrb_value sub, mrb_int pos)
}

#else
#define RSTRING_CHAR_LEN(s) RSTRING_LEN(s)
/* a byte is a character here, so both conversions are identity */
/* a byte is a character here, so the count is the byte length and both
conversions are identity */
mrb_int
mrb_str_char_len(mrb_state *mrb, mrb_value str)
{
(void)mrb;
return RSTRING_LEN(str);
}

mrb_int
mrb_str_char_to_byte(mrb_state *mrb, mrb_value str, mrb_int off, mrb_int idx)
{
Expand Down Expand Up @@ -890,7 +897,7 @@ mrb_str_beg_len(mrb_int str_len, mrb_int *begp, mrb_int *lenp)
static mrb_value
str_substr(mrb_state *mrb, mrb_value str, mrb_int beg, mrb_int len)
{
return mrb_str_beg_len(RSTRING_CHAR_LEN(str), &beg, &len) ?
return mrb_str_beg_len(mrb_str_char_len(mrb, str), &beg, &len) ?
str_subseq(mrb, str, beg, len) : mrb_nil_value();
}

Expand Down Expand Up @@ -1222,7 +1229,7 @@ mrb_str_plus_m(mrb_state *mrb, mrb_value self)
static mrb_value
mrb_str_size(mrb_state *mrb, mrb_value self)
{
mrb_int len = RSTRING_CHAR_LEN(self);
mrb_int len = mrb_str_char_len(mrb, self);
return mrb_int_value(mrb, len);
}

Expand Down Expand Up @@ -1463,7 +1470,7 @@ str_convert_range(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value alen,
return STR_BYTE_RANGE_CORRECTED;

case MRB_TT_RANGE:
*len = RSTRING_CHAR_LEN(str);
*len = mrb_str_char_len(mrb, str);
switch (mrb_range_beg_len(mrb, idx, beg, len, *len, TRUE)) {
case MRB_RANGE_OK:
return STR_CHAR_RANGE_CORRECTED;
Expand Down Expand Up @@ -1707,7 +1714,7 @@ mrb_str_aset(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value alen, mrb_v
if (len < 0) {
mrb_raisef(mrb, E_INDEX_ERROR, "negative length %v", alen);
}
charlen = RSTRING_CHAR_LEN(str);
charlen = mrb_str_char_len(mrb, str);
if (beg < 0) { beg += charlen; }
if (beg < 0 || beg > charlen) { str_out_of_index(mrb, idx); }
/* fall through */
Expand Down Expand Up @@ -2223,7 +2230,7 @@ mrb_str_index_m(mrb_state *mrb, mrb_value str)
pos = 0;
}
else if (pos < 0) {
mrb_int clen = RSTRING_CHAR_LEN(str);
mrb_int clen = mrb_str_char_len(mrb, str);
pos += clen;
if (pos < 0) {
return mrb_nil_value();
Expand Down Expand Up @@ -2403,7 +2410,7 @@ mrb_str_reverse_bang(mrb_state *mrb, mrb_value str)
char *p, *e;

#ifdef MRB_UTF8_STRING
mrb_int utf8_len = RSTRING_CHAR_LEN(str);
mrb_int utf8_len = mrb_str_char_len(mrb, str);
mrb_int len = RSTR_LEN(s);

if (utf8_len < 2) return str;
Expand Down
Loading