Merge pull request dbpedia#94 from ninniuz/80_quote_scales_with_regex

jimkont · jimkont · commit fcbf365b6671 · 2013-11-07T23:08:01.000-08:00
dbpedia#80: Fix escaping of special chars in regex
diff --git a/core/src/main/scala/org/dbpedia/extraction/dataparser/ParserUtils.scala b/core/src/main/scala/org/dbpedia/extraction/dataparser/ParserUtils.scala
@@ -4,6 +4,7 @@ import org.dbpedia.extraction.config.dataparser.ParserUtilsConfig
 import java.text.{NumberFormat,DecimalFormatSymbols}
 import org.dbpedia.extraction.util.Language
 import java.util.Locale
+import java.util.regex.Pattern
 
 /**
  * Utility functions used by the data parsers.
@@ -30,7 +31,7 @@ class ParserUtils( context : { def language : Language } )
 
     // TODO: use "\s+" instead of "\s?" between number and scale?
     // TODO: in some Asian languages, digits are not separated by thousands but by ten thousands or so...
-    private val regex = ("""(?i)([\D]*)([0-9]+(?:\""" + groupingSeparator + """[0-9]{3})*)(""" + decimalSeparatorsRegex + """[0-9]+)?\s?\[?\[?(""" + scales.keySet.mkString("|") + """)\]?\]?(.*)""").r
+    private val regex = ("""(?i)([\D]*)([0-9]+(?:\""" + groupingSeparator + """[0-9]{3})*)(""" + decimalSeparatorsRegex + """[0-9]+)?\s?\[?\[?(""" + scales.keySet.map(Pattern.quote).mkString("|") + """)\]?\]?(.*)""").r
     
     def parse(str: String): Number = {
       // space is sometimes used as grouping separator
diff --git a/core/src/test/scala/org/dbpedia/extraction/dataparser/ParserUtilsTest.scala b/core/src/test/scala/org/dbpedia/extraction/dataparser/ParserUtilsTest.scala
@@ -1,31 +1,33 @@
 package org.dbpedia.extraction.dataparser
 
-//import junit.framework.TestCase
-//import junit.framework.Assert._
-//import org.dbpedia.extraction.util.Language
-//
-//class ParserUtilsTest extends TestCase
-//{
-//  def testConvertLargeNumbers() : Unit =
-//  {
-//    testConvert("en", "100.5 million", "100500000")
-//    testConvert("de", "100,5 million", "100500000")
-//
-//    testConvert("de", "1.234,5 mrd", "1234500000000")
-//    // FIXME: this should fail, mrd is not English
-//    testConvert("en", "1,234.5 mrd", "1234500000000")
-//
-//    testConvert("en", "1,234.5 billion", "1234500000000")
-//    // FIXME: this should work, billion is 10^12 in German
-//    // testConvert("de", "100,5 billion", "100500000000000")
-//
-//    testConvert("en", "1,234.5 trillion", "1234500000000000")
-//    // FIXME: this should work, trillion is 10^18 in German
-//    // testConvert("de", "1.234,5 trillion", "1234500000000000000000")
-//  }
-//
-//  private def testConvert( lang : String, value : String, expect : String ) : Unit =
-//  {
-//    assertEquals(expect, ParserUtils.convertLargeNumbers(value, Language(lang)))
-//  }
-//}
+import junit.framework.TestCase
+import junit.framework.Assert._
+import org.dbpedia.extraction.util.Language
+
+class ParserUtilsTest extends TestCase
+{
+  def testConvertLargeNumbers() : Unit =
+  {
+    testConvert("en", "100.5 million", "100500000")
+    testConvert("de", "100,5 million", "100500000")
+
+    // testConvert("de", "1.234,5 mrd", "1234500000000")
+    // FIXME: this should fail, mrd is not English
+    // testConvert("en", "1,234.5 mrd", "1234500000000")
+
+    // testConvert("en", "1,234.5 billion", "1234500000000")
+    testConvert("de", "100,5 billion", "100500000000000")
+
+    // testConvert("en", "1,234.5 trillion", "1234500000000000")
+    // FIXME: this should work, trillion is 10^18 in German
+    // testConvert("de", "1.234,5 trillion", "1234500000000000000000")
+    testConvert("nl", "123 milja", "123 milja")
+    testConvert("nl", "123 milj.", "123000000000")
+  }
+
+  private def testConvert( lang : String, value : String, expect : String ) : Unit =
+  {
+    val parser = new ParserUtils(new { def language = Language(lang) })
+    assertEquals(expect, parser.convertLargeNumbers(value))
+  }
+}
