Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions include/mruby/string.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ struct RStringEmbed {
#endif
#define RSTR_SET_ASCII_FLAG(s) RSTR_SET_SINGLE_BYTE_FLAG(s)
#define RSTR_BINARY_P(s) ((s)->flags & MRB_STR_BINARY)
/* A copy of a string is byte-indexed exactly when the string it copies is, so
the flag travels with the bytes rather than being left behind on the
original. */
#define RSTR_COPY_BINARY_FLAG(dst, src) \
((dst)->flags = ((dst)->flags & ~MRB_STR_BINARY) | RSTR_BINARY_P(src))

/**
* Returns a pointer from a Ruby string
Expand Down
21 changes: 21 additions & 0 deletions mrbgems/mruby-encoding/test/string.rb
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,24 @@
assert_equal Encoding::BINARY, a.encoding
end
end

assert('String#encoding survives a copy') do
# A copy holds the same bytes, so it is byte-indexed exactly when the string
# it copies is. The copy used to come back UTF-8, which made `size` and every
# offset computed from it read the bytes as characters again.
if UTF8STRING
a = "\u{1F600}".b # F0 9F 98 80: four bytes, one character
assert_equal Encoding::BINARY, a.dup.encoding
assert_equal Encoding::BINARY, a.clone.encoding
assert_equal Encoding::BINARY, a.freeze.dup.encoding
b = "hello"
b.replace(a)
assert_equal Encoding::BINARY, b.encoding
# and a copy of a UTF-8 string is still UTF-8: the flag is copied, not set
c = "\u{1F600}"
assert_equal Encoding::UTF_8, c.dup.encoding
d = "x".b
d.replace(c)
assert_equal Encoding::UTF_8, d.encoding
end
end
18 changes: 18 additions & 0 deletions mrbgems/mruby-regexp/test/regexp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -844,6 +844,24 @@
assert_equal 2, ("x" + "\xb5").match(/.?[µ]/)[0].bytesize
end

assert("Regexp - a byte-indexed subject is reported in bytes") do
# `String#b` marks the subject byte-indexed, and MatchData snapshots it with
# a copy. The copy came back as if it were UTF-8, so #begin counted the
# characters of a string that has none, and disagreed with #pre_match, which
# counts the same span in bytes.
s = "\u{1F600}".b # F0 9F 98 80: four bytes, one character
assert_equal 3, (s =~ Regexp.new("\x80"))
assert_equal 3, s.byteindex(Regexp.new("\x80"))
md = s.match(Regexp.new("\x80"))
assert_equal 3, md.begin(0)
assert_equal 4, md.end(0)
assert_equal md.pre_match.bytesize, md.begin(0)
# the same subject read as UTF-8 counts characters, as it always has
u = "\u{1F600}"
assert_equal 0, (u =~ /./)
assert_equal 1, (("x" + u) =~ Regexp.new("\u{1F600}"))
end

assert("Regexp - a match does not end inside a character") do
# A pattern is compiled byte by byte and RE_CHAR consumes one byte, so a
# pattern holding a byte that reaches no character ends its match in the
Expand Down
1 change: 1 addition & 0 deletions src/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,7 @@ str_replace(mrb_state *mrb, struct RString *s1, struct RString *s2)
mrb_check_frozen(mrb, s1);
if (s1 == s2) return mrb_obj_value(s1);
RSTR_COPY_SINGLE_BYTE_FLAG(s1, s2);
RSTR_COPY_BINARY_FLAG(s1, s2);
if (RSTR_SHARED_P(s1)) {
str_decref(mrb, s1->as.heap.aux.shared);
}
Expand Down
Loading