Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions bench/bench_string_casecmp_unicode.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# frozen_string_literal: true
require 'benchmark/ips'

# gem install benchmark-ips
# ruby bench/bench_string_casecmp_unicode.rb LABEL [SAVE_PREFIX]
# Optional: CASE=unicode_short WARMUP=2 TIME=5
# Saved results allow benchmark-ips to compare separate Ruby processes.
label = ARGV.fetch(0, RUBY_DESCRIPTION)
save_prefix = ARGV[1]
utf8 = Encoding::UTF_8
lower = [*"\u{e0}".."\u{ff}"].join('') * 100
upper = [*"\u{c0}".."\u{df}"].join('') * 100
changed = "ÄΣЖĞŇÅÖÜ" * 400

cases = [
['ascii_known', :warm, 'HelloWorldAbCd00', 'hELLOwORLDaBcD00', true],
['ascii_fresh', :fresh, ('ab' * 16).bytes, ('AB' * 16).bytes, true],
['unicode_short', :warm, 'Straße', 'STRASSE', true],
['unicode_fresh', :fresh, 'Straße'.bytes, 'STRASSE'.bytes, true],
['unicode_mismatch_6200', :warm, lower, upper, false],
['unicode_identical_6200', :warm, lower, lower.bytes.pack('C*').force_encoding(utf8), true],
['unicode_changed_6400', :warm, changed, changed.downcase(:fold), true],
['unicode_expansions', :warm, 'ΐßİffiΣ' * 32, "ι\u0308\u0301ssi\u0307ffiσ" * 32, true],
]

cases.each do |name, mode, left, right, expected|
next if ENV['CASE'] && ENV['CASE'] != name
[left, right].each(&:ascii_only?) if mode == :warm
sample_left, sample_right = mode == :fresh ? [left.pack('C*').force_encoding(utf8), right.pack('C*').force_encoding(utf8)] : [left, right]
raise "incorrect comparison for #{name}" unless sample_left.casecmp?(sample_right) == expected
puts "\n#{name}"
Benchmark.ips do |x|
x.config(warmup: Float(ENV.fetch('WARMUP', '2')), time: Float(ENV.fetch('TIME', '5')))
if mode == :fresh
# Includes creation of new strings with unknown coderanges.
x.report(label) { left.pack('C*').force_encoding(utf8).casecmp?(right.pack('C*').force_encoding(utf8)) }
else
x.report(label) { left.casecmp?(right) }
end
x.save! "#{save_prefix}-#{name}.json" if save_prefix
x.compare!
end
end
8 changes: 8 additions & 0 deletions core/src/main/java/org/jruby/RubyString.java
Original file line number Diff line number Diff line change
Expand Up @@ -1970,6 +1970,14 @@ public IRubyObject casecmp_p(ThreadContext context, IRubyObject other) {
&& value.caseInsensitiveCmp(otherStr.value) == 0);
}

if (getEncoding().isUTF8() && otherStr.getEncoding().isUTF8()
&& (!isCodeRangeAsciiOnly() || !otherStr.isCodeRangeAsciiOnly())) {
if (scanForCodeRange() == CR_BROKEN || otherStr.scanForCodeRange() == CR_BROKEN) {
throw argumentError(context, "input string invalid");
}
return asBoolean(context, StringSupport.utf8CaseFoldEquals(getEncoding(), value, otherStr.value));
}

int flags = Config.CASE_FOLD;
RubyString down = dupString(context, this);
down.downcase_bang(context, flags);
Expand Down
44 changes: 44 additions & 0 deletions core/src/main/java/org/jruby/util/StringSupport.java
Original file line number Diff line number Diff line change
Expand Up @@ -2879,6 +2879,49 @@ private static final class MappingBuffer {
}
}

// Compare full folds of UTF-8 operands. Callers must validate both first: an early mismatch
// must not hide invalid bytes later in either original string.
public static boolean utf8CaseFoldEquals(Encoding enc, ByteList left, ByteList right) {
byte[] leftBytes = left.getUnsafeBytes();
byte[] rightBytes = right.getUnsafeBytes();
IntHolder leftInput = new IntHolder();
IntHolder rightInput = new IntHolder();
leftInput.value = left.getBegin();
rightInput.value = right.getBegin();
int leftEnd = leftInput.value + left.getRealSize();
int rightEnd = rightInput.value + right.getRealSize();
byte[] leftFold = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN];
byte[] rightFold = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN];
int leftPosition = 0, leftLength = 0;
int rightPosition = 0, rightLength = 0;

while (true) {
if (leftPosition == leftLength) {
if (leftInput.value == leftEnd) {
return rightInput.value == rightEnd && rightPosition == rightLength;
}
leftPosition = 0;
if (Encoding.isAscii(leftBytes[leftInput.value])) {
leftFold[0] = Encoding.asciiToLower(leftBytes[leftInput.value++]);
leftLength = 1;
} else {
leftLength = enc.mbcCaseFold(Config.CASE_FOLD, leftBytes, leftInput, leftEnd, leftFold);
}
}
if (rightPosition == rightLength) {
if (rightInput.value == rightEnd) return false;
rightPosition = 0;
if (Encoding.isAscii(rightBytes[rightInput.value])) {
rightFold[0] = Encoding.asciiToLower(rightBytes[rightInput.value++]);
rightLength = 1;
} else {
rightLength = enc.mbcCaseFold(Config.CASE_FOLD, rightBytes, rightInput, rightEnd, rightFold);
}
}
if (leftFold[leftPosition++] != rightFold[rightPosition++]) return false;
}
}

private static final int CASE_MAPPING_ADDITIONAL_LENGTH = 20;

@Deprecated(since = "10.0.0.0")
Expand Down Expand Up @@ -2910,6 +2953,7 @@ public static ByteList caseMap(ThreadContext context, ByteList src, IntHolder fl
tgt = new ByteList(buffer.bytes, 0, buffer.used, enc, false);
} else {
tgt = new ByteList(tgtLen);
tgt.setRealSize(tgtLen);
tgt.setEncoding(enc);
buffer = root.next;
int tgtPos = 0;
Expand Down
38 changes: 38 additions & 0 deletions spec/ruby/core/string/casecmp_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,44 @@
[left, right].should == originals
end

it "matches Unicode folds across expanded character boundaries" do
[
["ßßa", "ssssA"],
["İİ", "i\u0307i\u0307"],
["ffiß", "ffiss"],
["Σς", "σσ"],
["\u{10400}", "\u{10428}"],
["ΐ" * 4, "ι\u0308\u0301" * 4],
].each do |left, right|
left.freeze.casecmp?(right.freeze).should == true
right.casecmp?(left).should == true
end
"ßßx".casecmp?("ssssY").should == false
end

it "raises for invalid UTF-8 after an unequal prefix" do
[["a\xff", "z"], ["a", "z\xff"], ["", "z\xff"]].each do |left, right|
-> { left.freeze.casecmp?(right.freeze) }.should.raise(ArgumentError)
end
end

it "raises for identical invalid UTF-8 operands" do
invalid = "\xff".freeze
-> { invalid.casecmp?(invalid) }.should.raise(ArgumentError)
end

it "folds frozen Unicode substrings without modifying their carriers" do
text = "aÄßİΣKffi\u{10400}\0" * 64
fold = "aässi\u0307σkffi\u{10428}\0" * 64
left_carrier = ('p' * 1024 + text + 'q' * 512).freeze
right_carrier = ('r' * 768 + fold + 's' * 512).freeze
left = left_carrier.byteslice(1024, text.bytesize).freeze
right = right_carrier.byteslice(768, fold.bytesize).freeze
left.casecmp?(right).should == true
left_carrier.should == 'p' * 1024 + text + 'q' * 512
right_carrier.should == 'r' * 768 + fold + 's' * 512
end

it 'returns true when equal to other' do
'abc'.casecmp?('abc').should == true
'abc'.casecmp?('ABC').should == true
Expand Down
15 changes: 15 additions & 0 deletions spec/ruby/core/string/downcase_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@
end

describe "case folding" do
it "preserves all characters when several Unicode folds expand" do
[4, 10, 1000].each do |count|
source = ("\u0390" * count).freeze
source.downcase(:fold).should == "\u03b9\u0308\u0301" * count
end
end

it "case folds special characters" do
"ß".downcase.should == "ß"
"ß".downcase(:fold).should == "ss"
Expand Down Expand Up @@ -164,6 +171,14 @@
end

describe "case folding" do
it "preserves all characters when several Unicode folds expand" do
[4, 10, 1000].each do |count|
source = "\u0390" * count
source.downcase!(:fold)
source.should == "\u03b9\u0308\u0301" * count
end
end

it "case folds special characters" do
a = "ß"
a.downcase!
Expand Down
Loading