Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions bench/bench_string_casemap.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# frozen_string_literal: true
require 'benchmark/ips'

# gem install benchmark-ips
# Compare Rubies using benchmark-ips' saved results (one file per case):
# ruby bench/bench_string_casemap.rb baseline /tmp/casemap
# jruby bench/bench_string_casemap.rb candidate /tmp/casemap
# Optional: CASE=capitalize_ascii_1000 WARMUP=2 TIME=5
label = ARGV.fetch(0, RUBY_DESCRIPTION)
save_prefix = ARGV[1]

inputs = {
'ascii_1' => 'a',
'ascii_100' => ('hello World ' * 9)[0, 100],
'ascii_1000' => ('hello World ' * 84)[0, 1000],
'binary_100' => ("hello W\xC3\xB6rld " * 8).byteslice(0, 100).b,
'utf8_100' => ("héllo Wörld " * 7) + 'ab', # 100 bytes
}

%w[upcase downcase capitalize swapcase].each do |op|
bang = :"#{op}!"
inputs.each do |input, str|
name = "#{op}_#{input}"
next if ENV['CASE'] && ENV['CASE'] != name
puts "\n#{name}"
Benchmark.ips do |x|
x.config(warmup: Float(ENV.fetch('WARMUP', '2')), time: Float(ENV.fetch('TIME', '5')))
x.report("#{label} #{op}") { str.public_send(op) }
# Includes the dup needed to keep the input unmodified.
x.report("#{label} #{bang}") { str.dup.public_send(bang) }
x.save! "#{save_prefix}-#{name}.json" if save_prefix
x.compare!
end
end
end
112 changes: 68 additions & 44 deletions core/src/main/java/org/jruby/RubyString.java
Original file line number Diff line number Diff line change
Expand Up @@ -2113,16 +2113,63 @@ private int caseMap(ThreadContext context, int flags, Encoding enc) {
IntHolder flagsP = new IntHolder();
flagsP.value = flags;
if ((flags & Config.CASE_ASCII_ONLY) != 0) {
StringSupport.asciiOnlyCaseMap(context, value, flagsP);
StringSupport.asciiOnlyCaseMap(context, value, flagsP, enc);
} else {
value = StringSupport.caseMap(context, value, flagsP, enc);
}
return flagsP.value;
}

/** rb_str_upcase / rb_str_upcase_bang
*
*/
@CRuby(value = "case_option_single_p", file = "string.c")
private boolean caseOptionSingle(int flags, Encoding enc) {
if ((flags & Config.CASE_ASCII_ONLY) != 0 && (enc == UTF8Encoding.INSTANCE || enc.maxLength() == 1)) return true;
return (flags & Config.CASE_FOLD_TURKISH_AZERI) == 0 &&
(getCodeRange() == CR_7BIT || enc == ASCIIEncoding.INSTANCE);
}

// Only ASCII letters change, so the code range stays as it is.
@CRuby(value = "upcase_single", file = "string.c")
private static boolean upcaseSingle(byte[] bytes, int s, int end) {
boolean modified = false;
for (; s < end; s++) {
int c = bytes[s] & 0xff;
if ('a' <= c && c <= 'z') {
bytes[s] = (byte) ('A' + (c - 'a'));
modified = true;
}
}
return modified;
}

@CRuby(value = "downcase_single", file = "string.c")
private static boolean downcaseSingle(byte[] bytes, int s, int end) {
boolean modified = false;
for (; s < end; s++) {
int c = bytes[s] & 0xff;
if ('A' <= c && c <= 'Z') {
bytes[s] = (byte) ('a' + (c - 'A'));
modified = true;
}
}
return modified;
}

// titlecase the first byte, then the rest is a plain downcase loop
@CRuby(value = "capitalize_single", file = "string.c")
private static boolean capitalizeSingle(byte[] bytes, int s, int end) {
boolean modified = false;
if (s < end) {
int c = bytes[s] & 0xff;
if ('a' <= c && c <= 'z') {
bytes[s] = (byte) ('A' + (c - 'a'));
modified = true;
}
s++;
}
return downcaseSingle(bytes, s, end) | modified;
}

@CRuby(value = "rb_str_upcase", file = "string.c")
@JRubyMethod(name = "upcase")
public RubyString upcase(ThreadContext context) {
RubyString str = dupString(context, this);
Expand Down Expand Up @@ -2159,22 +2206,12 @@ public IRubyObject upcase_bang(ThreadContext context, IRubyObject arg0, IRubyObj
return upcase_bang(context, StringSupport.checkCaseMapOptions(context, arg0, arg1, Config.CASE_UPCASE));
}

@CRuby(value = "rb_str_upcase_bang", file = "string.c")
private IRubyObject upcase_bang(ThreadContext context, int flags) {
modifyAndKeepCodeRange();
Encoding enc = checkDummyEncoding();
if (((flags & Config.CASE_ASCII_ONLY) != 0 && (enc.isUTF8() || enc.maxLength() == 1)) ||
(flags & Config.CASE_FOLD_TURKISH_AZERI) == 0 && getCodeRange() == CR_7BIT) {
int s = value.getBegin();
int end = s + value.getRealSize();
byte[]bytes = value.getUnsafeBytes();
while (s < end) {
int c = bytes[s] & 0xff;
if (Encoding.isAscii(c) && 'a' <= c && c <= 'z') {
bytes[s] = (byte)('A' + (c - 'a'));
flags |= Config.CASE_MODIFIED;
}
s++;
}
if (caseOptionSingle(flags, enc)) {
if (upcaseSingle(value.getUnsafeBytes(), value.getBegin(), value.getBegin() + value.getRealSize())) flags |= Config.CASE_MODIFIED;
} else {
flags = caseMap(context, flags, enc);
if ((flags & Config.CASE_MODIFIED) != 0) clearCodeRange();
Expand All @@ -2183,9 +2220,7 @@ private IRubyObject upcase_bang(ThreadContext context, int flags) {
return ((flags & Config.CASE_MODIFIED) != 0) ? this : context.nil;
}

/** rb_str_downcase / rb_str_downcase_bang
*
*/
@CRuby(value = "rb_str_downcase", file = "string.c")
@JRubyMethod(name = "downcase")
public RubyString downcase(ThreadContext context) {
RubyString str = dupString(context, this);
Expand Down Expand Up @@ -2222,22 +2257,12 @@ public IRubyObject downcase_bang(ThreadContext context, IRubyObject arg0, IRubyO
return downcase_bang(context, StringSupport.checkCaseMapOptions(context, arg0, arg1, Config.CASE_DOWNCASE));
}

@CRuby(value = "rb_str_downcase_bang", file = "string.c")
private IRubyObject downcase_bang(ThreadContext context, int flags) {
modifyAndKeepCodeRange();
Encoding enc = checkDummyEncoding();
if (((flags & Config.CASE_ASCII_ONLY) != 0 && (enc.isUTF8() || enc.maxLength() == 1)) ||
(flags & Config.CASE_FOLD_TURKISH_AZERI) == 0 && getCodeRange() == CR_7BIT) {
int s = value.getBegin();
int end = s + value.getRealSize();
byte[]bytes = value.getUnsafeBytes();
while (s < end) {
int c = bytes[s] & 0xff;
if (Encoding.isAscii(c) && 'A' <= c && c <= 'Z') {
bytes[s] = (byte)('a' + (c - 'A'));
flags |= Config.CASE_MODIFIED;
}
s++;
}
if (caseOptionSingle(flags, enc)) {
if (downcaseSingle(value.getUnsafeBytes(), value.getBegin(), value.getBegin() + value.getRealSize())) flags |= Config.CASE_MODIFIED;
} else {
flags = caseMap(context, flags, enc);
if ((flags & Config.CASE_MODIFIED) != 0) clearCodeRange();
Expand All @@ -2246,9 +2271,7 @@ private IRubyObject downcase_bang(ThreadContext context, int flags) {
return ((flags & Config.CASE_MODIFIED) != 0) ? this : context.nil;
}

/** rb_str_swapcase / rb_str_swapcase_bang
*
*/
@CRuby(value = "rb_str_swapcase", file = "string.c")
@JRubyMethod(name = "swapcase")
public RubyString swapcase(ThreadContext context) {
RubyString str = dupString(context, this);
Expand Down Expand Up @@ -2285,6 +2308,7 @@ public IRubyObject swapcase_bang(ThreadContext context, IRubyObject arg0, IRubyO
return swapcase_bang(context, StringSupport.checkCaseMapOptions(context, arg0, arg1, Config.CASE_UPCASE | Config.CASE_DOWNCASE));
}

@CRuby(value = "rb_str_swapcase_bang", file = "string.c")
private IRubyObject swapcase_bang(ThreadContext context, int flags) {
modifyAndKeepCodeRange();
flags = caseMap(context, flags, checkDummyEncoding());
Expand All @@ -2296,9 +2320,7 @@ private IRubyObject swapcase_bang(ThreadContext context, int flags) {
return context.nil;
}

/** rb_str_capitalize / rb_str_capitalize_bang
*
*/
@CRuby(value = "rb_str_capitalize", file = "string.c")
@JRubyMethod(name = "capitalize")
public RubyString capitalize(ThreadContext context) {
RubyString str = dupString(context, this);
Expand Down Expand Up @@ -2335,6 +2357,7 @@ public IRubyObject capitalize_bang(ThreadContext context, IRubyObject arg0, IRub
return capitalize_bang(context, StringSupport.checkCaseMapOptions(context, arg0, arg1, Config.CASE_UPCASE | Config.CASE_TITLECASE));
}

@CRuby(value = "rb_str_capitalize_bang", file = "string.c")
private IRubyObject capitalize_bang(ThreadContext context, int flags) {
modifyAndKeepCodeRange();
Encoding enc = checkDummyEncoding();
Expand All @@ -2344,13 +2367,14 @@ private IRubyObject capitalize_bang(ThreadContext context, int flags) {
return context.nil;
}

flags = caseMap(context, flags, enc);
if ((flags & Config.CASE_MODIFIED) != 0) {
clearCodeRange();
return this;
if (caseOptionSingle(flags, enc)) {
if (capitalizeSingle(value.getUnsafeBytes(), value.getBegin(), value.getBegin() + value.getRealSize())) flags |= Config.CASE_MODIFIED;
} else {
flags = caseMap(context, flags, enc);
if ((flags & Config.CASE_MODIFIED) != 0) clearCodeRange();
}

return context.nil;
return ((flags & Config.CASE_MODIFIED) != 0) ? this : context.nil;
}

@Deprecated(since = "10.0.0.0")
Expand Down
37 changes: 29 additions & 8 deletions core/src/main/java/org/jruby/util/StringSupport.java
Original file line number Diff line number Diff line change
Expand Up @@ -2925,19 +2925,40 @@ public static ByteList caseMap(ThreadContext context, ByteList src, IntHolder fl

@Deprecated(since = "10.0.0.0")
public static void asciiOnlyCaseMap(Ruby runtime, ByteList value, IntHolder flags, Encoding enc) {
asciiOnlyCaseMap(runtime.getCurrentContext(), value, flags);
asciiOnlyCaseMap(runtime.getCurrentContext(), value, flags, enc);
}

@Deprecated(since = "10.1.3.0")
public static void asciiOnlyCaseMap(ThreadContext context, ByteList value, IntHolder flags) {
asciiOnlyCaseMap(context, value, flags, ASCIIEncoding.INSTANCE);
}

// Walks characters in enc, so invalid bytes raise and capitalize titlecases the first character, not byte.
// ASCII letters keep their length in every encoding, so the mapping is done in place.
@CRuby(value = "rb_str_ascii_casemap", file = "string.c")
@CRuby(value = "onigenc_ascii_only_case_map", file = "regenc.c")
public static void asciiOnlyCaseMap(ThreadContext context, ByteList value, IntHolder flagsP, Encoding enc) {
if (value.getRealSize() == 0) return;
int s = value.getBegin();
int end = s + value.getRealSize();
byte[]bytes = value.getUnsafeBytes();
int p = value.getBegin();
int end = p + value.getRealSize();
byte[] bytes = value.getUnsafeBytes();
int flags = flagsP.value;

IntHolder pp = new IntHolder();
pp.value = s;
int len = ASCIIEncoding.INSTANCE.caseMap(flags, bytes, pp, end, bytes, s, end);
if (len < 0) throw argumentError(context, "input string invalid");
while (p < end) {
int n = preciseLength(enc, bytes, p, end);
if (!MBCLEN_CHARFOUND_P(n)) throw argumentError(context, "input string invalid");
int code = enc.mbcToCode(bytes, p, end);
if ('a' <= code && code <= 'z' && (flags & Config.CASE_UPCASE) != 0) {
flags |= Config.CASE_MODIFIED;
enc.codeToMbc(code - ('a' - 'A'), bytes, p);
} else if ('A' <= code && code <= 'Z' && (flags & (Config.CASE_DOWNCASE | Config.CASE_FOLD)) != 0) {
flags |= Config.CASE_MODIFIED;
enc.codeToMbc(code + ('a' - 'A'), bytes, p);
}
p += MBCLEN_CHARFOUND_LEN(n);
if ((flags & Config.CASE_TITLECASE) != 0) flags ^= Config.CASE_UPCASE | Config.CASE_DOWNCASE | Config.CASE_TITLECASE;
}
flagsP.value = flags;
}

public static int encCoderangeClean(int cr) {
Expand Down
49 changes: 49 additions & 0 deletions spec/ruby/core/string/capitalize_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@
"abcdef"[1...-1].capitalize.should == "Bcde"
end

it "changes only ASCII letters in a binary String" do
"aB\xE4\xC4\0cD".b.capitalize.should == "Ab\xE4\xC4\0cd".b
end

it "downcases the rest of a binary String when the first byte is not ASCII" do
"\xE4AB".b.capitalize.should == "\xE4ab".b
end

describe "full Unicode case mapping" do
it "works for all of Unicode with no option" do
"äöÜ".capitalize.should == "Äöü"
Expand All @@ -33,13 +41,35 @@
end

describe "ASCII-only case mapping" do
it "raises ArgumentError for an invalid multibyte String" do
-> { "aB\xA4".dup.force_encoding(Encoding::EUC_JP).capitalize(:ascii) }.should.raise(ArgumentError, "input string invalid")
end

it "raises ArgumentError for invalid UTF8-MAC" do
-> { "\xFFaB".dup.force_encoding(Encoding::UTF8_MAC).capitalize(:ascii) }.should.raise(ArgumentError, "input string invalid")
end

it "does not change ASCII bytes inside multibyte characters" do
"\x83AaB".dup.force_encoding(Encoding::Shift_JIS).capitalize(:ascii).should == "\x83Aab".dup.force_encoding(Encoding::Shift_JIS)
end

it "capitalizes the first character in UTF-16 and UTF-32 encodings" do
[Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE].each do |encoding|
"aBc".encode(encoding).capitalize(:ascii).should == "Abc".encode(encoding)
end
end

it "does not capitalize non-ASCII characters" do
"ßet".capitalize(:ascii).should == "ßet"
end

it "handles non-ASCII substrings properly" do
"garçon"[1...-1].capitalize(:ascii).should == "Arço"
end

it "downcases the rest when the first character is not ASCII" do
"éAB".capitalize(:ascii).should == "éab"
end
end

describe "full Unicode case mapping adapted for Turkic languages" do
Expand Down Expand Up @@ -189,6 +219,25 @@
-> { a = "abc"; a.capitalize!(:invalid_option) }.should.raise(ArgumentError)
end

it "changes only ASCII letters in a binary String" do
a = "aB\xE4\xC4\0cD".b
a.capitalize!.should.equal?(a)
a.should == "Ab\xE4\xC4\0cd".b
a.ascii_only?.should == false
end

it "returns nil for a binary String with no ASCII letters" do
"\xE4\xC4\0".b.capitalize!.should == nil
end

it "does not modify the String it was sliced from" do
source = +"xxaBcDxx"
slice = source[2, 4]
slice.capitalize!
slice.should == "Abcd"
source.should == "xxaBcDxx"
end

it "returns nil when no changes are made" do
a = +"Hello"
a.capitalize!.should == nil
Expand Down
1 change: 1 addition & 0 deletions spec/ruby/core/string/casecmp_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@
it "case folds Latin-1 characters but not binary non-ASCII bytes" do
"\u00c4".encode('ISO-8859-1').casecmp?("\u00e4".encode('ISO-8859-1')).should == true
"\xc4".b.casecmp?("\xe4".b).should == false
"A\xc4".b.casecmp?("a\xc4".b).should == true
end

it "case folds ASCII letters in UTF-16 and UTF-32 encodings" do
Expand Down
Loading
Loading