Get rid of expected failures in tokenizer tests

mistotebe · web-flow · commit 7e084aec1ba9 · 2020-05-06T16:48:46.000+02:00
#283
diff --git a/Lib/ldap/schema/tokenizer.py b/Lib/ldap/schema/tokenizer.py
@@ -13,12 +13,16 @@
     r"|"              # or
     r"([^'$()\s]+)"   # string of length >= 1 without '$() or whitespace
     r"|"              # or
-    r"('.*?'(?!\w))"  # any string or empty string surrounded by single quotes
-                      # except if right quote is succeeded by alphanumeric char
+    r"('(?:[^'\\]|\\\\|\\.)*?'(?!\w))"
+                      # any string or empty string surrounded by unescaped
+                      # single quotes except if right quote is succeeded by
+                      # alphanumeric char
     r"|"              # or
     r"([^\s]+?)",     # residue, all non-whitespace strings
 ).findall
 
+UNESCAPE_PATTERN = re.compile(r"\\(.)")
+
 
 def split_tokens(s):
     """
@@ -30,7 +34,7 @@ def split_tokens(s):
         if unquoted:
             parts.append(unquoted)
         elif quoted:
-            parts.append(quoted[1:-1])
+            parts.append(UNESCAPE_PATTERN.sub(r'\1', quoted[1:-1]))
         elif opar:
             parens += 1
             parts.append(opar)
diff --git a/Tests/t_ldap_schema_tokenizer.py b/Tests/t_ldap_schema_tokenizer.py
@@ -44,8 +44,8 @@
 
 # broken schema of Oracle Internet Directory
 TESTCASES_BROKEN_OID = (
-    ("BLUBB DI 'BLU B B ER'MUST 'BLAH' ", ['BLUBB', 'DI', 'BLU B B ER', 'MUST', 'BLAH']),
-    ("BLUBBER DI 'BLU'BB ER' DA 'BLAH' ", ["BLUBBER", "DI", "BLU'BB ER", "DA", "BLAH"]),
+    "BLUBB DI 'BLU B B ER'MUST 'BLAH' ", #['BLUBB', 'DI', 'BLU B B ER', 'MUST', 'BLAH']
+    "BLUBBER DI 'BLU'BB ER' DA 'BLAH' ", #["BLUBBER", "DI", "BLU'BB ER", "DA", "BLAH"]
 )
 
 # for quoted single quotes inside string values
@@ -104,14 +104,12 @@ def test_utf8(self):
         """
         self._run_split_tokens_tests(TESTCASES_UTF8)
 
-    @unittest.expectedFailure
     def test_broken_oid(self):
         """
         run test cases specified in constant TESTCASES_BROKEN_OID
         """
         self._run_failure_tests(TESTCASES_BROKEN_OID)
 
-    @unittest.expectedFailure
     def test_escaped_quotes(self):
         """
         run test cases specified in constant TESTCASES_ESCAPED_QUOTES
