mruby · matz · Aug 15, 2026 · Aug 15, 2026 · Aug 15, 2026 · Aug 15, 2026
@@ -36,4 +36,5 @@ mruby-source-*.gem
 node_modules
 perf.data*
 tags
+/tools/unicode/data
 !Gemfile.lock
@@ -52,6 +52,7 @@ load "#{MRUBY_ROOT}/tasks/benchmark.rake"
 load "#{MRUBY_ROOT}/tasks/doc.rake"
 load "#{MRUBY_ROOT}/tasks/install.rake"
 load "#{MRUBY_ROOT}/tasks/amalgam.rake"
+load "#{MRUBY_ROOT}/tasks/unicode.rake"
 
 ##############################
 # generic build targets, rules

@@ -1,8 +1,8 @@
 /*
 ** re_cased.h - codepoints /i cannot answer without Unicode case data
 **
-** Generated by tools/gen_cased.rb from Unicode 17.0.0
-** as carried by ruby 4.0.6. Do not edit by hand.
+** Generated by mrbgems/mruby-regexp/tools/gen_cased.rb from the Unicode
+** 17.0.0 character database. Do not edit by hand.
 **
 ** A build without MRB_UNICODE_CASE refuses to compile an /i pattern
 ** holding one of these, rather than folding ASCII and answering wrongly.

@@ -1,7 +1,13 @@
 # Generate the codepoints /i cannot answer for, from the Unicode simple case
-# folding data that the host CRuby carries.
+# folding data.
 #
-#   ruby mrbgems/mruby-regexp/tools/gen_cased.rb mrbgems/mruby-regexp/src
+#   ruby mrbgems/mruby-regexp/tools/gen_cased.rb mrbgems/mruby-regexp/src [UCDDIR]
+#
+# UCDDIR holds UnicodeData.txt, SpecialCasing.txt and CaseFolding.txt as
+# published under https://www.unicode.org/Public/<version>/ucd/, and defaults
+# to the copy tools/unicode/case_data.rb names, the same one core's
+# tools/gen_unicase.rb reads. The files are not in the repository: they are
+# read when the table is regenerated and nowhere else.
 #
 # re_cased.h holds the ranges a build without the foldings must refuse under
 # /i, and is compiled only into such a build. A build with them reads them off
@@ -11,31 +17,39 @@
 # refusals are written out, since what has to be refused is defined by what
 # the other build folds: a codepoint is here exactly when it is missing there.
 
-require 'rbconfig'
 require 'set'
+require_relative '../../../tools/unicode/case_data'
+
+outdir = ARGV[0] or abort "usage: #{$0} OUTDIR [UCDDIR]"
+data = Unicode::CaseData.load(ARGV[1])
+version = data.version
 
-outdir = ARGV[0] or abort "usage: #{$0} OUTDIR"
+# The lower case mapping, which is the full one where the character has one:
+# the counterpart hunted below is the one a lower case away, and a character
+# whose lower case is several characters has none. Folding is the full one,
+# which is the folding core carries and therefore the one whose reach is at
+# stake here.
+lower = data.lower
+fold = data.fold
 
 pairs = []      # [source, counterpart] for the foldings that pair one with one
 skipped = []    # [source, fold] where no single counterpart exists
 cased = Set.new # every codepoint that either folds or is folded to
 
-(0x80..0x10FFFF).each do |cp|
-  next if cp.between?(0xD800, 0xDFFF)
-  c = begin; cp.chr("UTF-8"); rescue RangeError; next; end
-  f = c.downcase(:fold)
-  next if f == c
+fold.keys.sort.each do |cp|
+  next if cp < 0x80
+  f = fold[cp]
   cased << cp
   to = nil
-  if f.length == 1
-    to = f.ord
+  if f.size == 1
+    to = f[0]
   else
     # A fold of several codepoints can still leave a single counterpart the
     # engine can use: U+1E9E lower cases to U+00DF, and the two fold alike, so
     # /ß/i reaching "ẞ" needs no more than the 1:1 machinery. Only a source
     # with no such counterpart at all (U+FB00 to "ff") is out of reach.
-    lo = c.downcase
-    to = lo.ord if lo.length == 1 && lo != c && lo.downcase(:fold) == f
+    lo = lower[cp]
+    to = lo[0] if lo && lo.size == 1 && lo != [cp] && (fold[lo[0]] || lo) == f
   end
   if to
     pairs << [cp, to]
@@ -89,8 +103,8 @@
     /*
     ** re_cased.h - codepoints /i cannot answer without Unicode case data
     **
-    ** Generated by tools/gen_cased.rb from Unicode #{RbConfig::CONFIG['UNICODE_VERSION'] || 'data'}
-    ** as carried by ruby #{RUBY_VERSION}. Do not edit by hand.
+    ** Generated by mrbgems/mruby-regexp/tools/gen_cased.rb from the Unicode
+    ** #{version} character database. Do not edit by hand.
     **
     ** A build without MRB_UNICODE_CASE refuses to compile an /i pattern
     ** holding one of these, rather than folding ASCII and answering wrongly.

@@ -1,8 +1,8 @@
 /*
 ** unicase.h - Unicode case mapping tables
 **
-** Generated by tools/gen_unicase.rb from Unicode 17.0.0
-** as carried by ruby 4.0.6. Do not edit by hand.
+** Generated by tools/gen_unicase.rb from the Unicode 17.0.0 character
+** database. Do not edit by hand.
 **
 ** Sources below 128 are not in the tables: the callers fold ASCII inline.
 ** A source that maps to several characters is in the multi table beside

@@ -0,0 +1,77 @@
+require 'digest'
+require 'rbconfig'
+require "#{MRUBY_ROOT}/tools/unicode/case_data"
+
+# The tables are committed, so a build never reaches any of this, which is why
+# what only one task needs is required inside it rather than up here. What runs
+# here is what a Unicode version bump comes to: change `VERSION` in
+# tools/unicode/case_data.rb, fetch the database it names, record the digests
+# the fetch prints in `CHECKSUMS` beside it, and regenerate every table at
+# once so that no build is left reading an older Unicode than its neighbour.
+UNICODE_DATA_DIR = Unicode::CaseData.dir
+UNICODE_FILES = Unicode::CaseData::FILES.map { |f| "#{UNICODE_DATA_DIR}/#{f}" }
+
+UNICODE_GENERATORS = {
+  'core' => ['tools/gen_unicase.rb', 'src'],
+  'gem:regexp' => ['mrbgems/mruby-regexp/tools/gen_cased.rb', 'mrbgems/mruby-regexp/src'],
+}
+
+# The database is not in the repository, so each file is a task that fetches
+# it. Asking for a table before anything was downloaded and asking for the
+# download reach the same rule that way.
+#
+# The digest of what came down is printed rather than checked, because a bump
+# fetches a release before anything can know its digests. What it prints, as
+# `sha256sum` spells it, is the digest `CHECKSUMS` is then to record.
+UNICODE_FILES.each do |path|
+  file path do
+    require 'open-uri'
+    url = "#{Unicode::CaseData::URL_BASE}/#{File.basename(path)}"
+    puts "downloading #{url}"
+    mkdir_p File.dirname(path)
+    File.binwrite("#{path}.tmp", URI.parse(url).open(&:read))
+    mv "#{path}.tmp", path
+    puts "  #{Digest::SHA256.file(path).hexdigest}  #{File.basename(path)}"
+  end
+end
+
+def unicode_generate(script, outdir)
+  sh RbConfig.ruby, "#{MRUBY_ROOT}/#{script}", "#{MRUBY_ROOT}/#{outdir}", UNICODE_DATA_DIR
+end
+
+namespace :unicode do
+  desc "download the Unicode #{Unicode::CaseData::VERSION} character database"
+  task :download => UNICODE_FILES
+
+  desc 'generate all Unicode tables'
+  task :generate => UNICODE_GENERATORS.keys.map { |name| "generate:#{name}" }
+
+  namespace :generate do
+    UNICODE_GENERATORS.each do |name, (script, outdir)|
+      desc "generate the Unicode tables in #{outdir}"
+      task name => UNICODE_FILES do
+        unicode_generate(script, outdir)
+      end
+    end
+  end
+
+  desc 'check the committed Unicode tables against the database'
+  task :verify => UNICODE_FILES do
+    require 'tmpdir'
+    stale = []
+    Dir.mktmpdir do |tmp|
+      UNICODE_GENERATORS.each_value do |script, outdir|
+        sh RbConfig.ruby, "#{MRUBY_ROOT}/#{script}", tmp, UNICODE_DATA_DIR
+        Dir.glob("#{tmp}/*.h").each do |built|
+          committed = "#{MRUBY_ROOT}/#{outdir}/#{File.basename(built)}"
+          stale << committed unless FileUtils.identical?(built, committed)
+        end
+        rm Dir.glob("#{tmp}/*.h")
+      end
+    end
+    stale.each { |path| puts "stale: #{path} is not what the database generates" }
+
+    fail 'the Unicode tables are out of date' unless stale.empty?
+    puts "the Unicode #{Unicode::CaseData::VERSION} tables are up to date"
+  end
+end
@@ -1,6 +1,12 @@
-# Generate the Unicode case mapping table from the data the host CRuby carries.
+# Generate the Unicode case mapping table from the Unicode Character Database.
 #
-#   ruby tools/gen_unicase.rb src
+#   ruby tools/gen_unicase.rb src [UCDDIR]
+#
+# UCDDIR holds UnicodeData.txt, SpecialCasing.txt and CaseFolding.txt as
+# published under https://www.unicode.org/Public/<version>/ucd/, and defaults
+# to the copy tools/unicode/case_data.rb names, which reads them for this
+# generator and for the one mruby-regexp carries. The files are not in the
+# repository: they are read when the table is regenerated and nowhere else.
 #
 # unicase.h holds what `String#downcase`, `#upcase`, `#capitalize`,
 # `#swapcase` and `#casecmp?` answer for a character above ASCII, and is
@@ -19,37 +25,31 @@
 #
 # Swap case is stored the same way, against the rule that a character with a
 # lower case swaps down and one without swaps up. What the rule misses is the
-# title case characters, which CRuby swaps to something neither case spells:
+# title case characters, which swap to something neither case spells:
 # `U+01C5` upcases to `U+01C4` and downcases to `U+01C6`, and swaps to "dŽ".
 
-require 'rbconfig'
+require_relative 'unicode/case_data'
 
-outdir = ARGV[0] or abort "usage: #{$0} OUTDIR"
+outdir = ARGV[0] or abort "usage: #{$0} OUTDIR [UCDDIR]"
+data = Unicode::CaseData.load(ARGV[1])
+version = data.version
 
 # ------------------------------------------------------------------- gather
 
-lower = {}
-upper = {}
-title = {}
-fold  = {}
-swap_diff = {}
-
-(0x80..0x10FFFF).each do |cp|
-  next if cp.between?(0xD800, 0xDFFF)
-  c = begin; cp.chr("UTF-8"); rescue RangeError; next; end
-  l = c.downcase
-  u = c.upcase
-  t = c.capitalize          # one character capitalized is its title case
-  f = c.downcase(:fold)
-  lower[cp] = l if l != c
-  upper[cp] = u if u != c
-  title[cp] = t if t != c
-  fold[cp] = f if f != c
-  # What the swap rule would answer, against what swapping actually answers.
-  s = c.swapcase
-  swap_diff[cp] = s if s != (l != c ? l : u)
+# One mapping as the UTF-8 each source answers with. ASCII is folded inline by
+# the callers, so nothing below it is carried.
+def table(map)
+  map.each_with_object({}) do |(cp, to), h|
+    h[cp] = to.pack("U*") if cp >= 0x80
+  end
 end
 
+lower = table(data.lower)
+upper = table(data.upper)
+title = table(data.title)
+fold  = table(data.fold)
+swap_diff = table(data.swap)
+
 # Title case rides on upper case, so only what the two disagree about is
 # stored. A source the two disagree about maps to itself under title case as
 # often as it maps to something, and both have to be said.
@@ -166,8 +166,8 @@ def hex(cp)
     /*
     ** unicase.h - Unicode case mapping tables
     **
-    ** Generated by tools/gen_unicase.rb from Unicode #{RbConfig::CONFIG['UNICODE_VERSION'] || 'data'}
-    ** as carried by ruby #{RUBY_VERSION}. Do not edit by hand.
+    ** Generated by tools/gen_unicase.rb from the Unicode #{version} character
+    ** database. Do not edit by hand.
     **
     ** Sources below 128 are not in the tables: the callers fold ASCII inline.
     ** A source that maps to several characters is in the multi table beside