Skip to content

Commit eccbc5e

Browse files
ShaharNavehShaharNaveh
authored
Update xml to 3.14.6 (RustPython#8101)
* Update `xml` to 3.14.6 * Update tests * mark failing tests * Align patches
1 parent 12f8b77 commit eccbc5e

5 files changed

Lines changed: 358 additions & 56 deletions

File tree

‎Lib/test/test_pyexpat.py‎

Lines changed: 224 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,9 +228,9 @@ def _verify_parse_output(self, operations):
228228
"Character data: '\xb5'",
229229
"End element: 'root'",
230230
]
231-
for operation, expected_operation in zip(operations, expected_operations):
232-
self.assertEqual(operation, expected_operation)
231+
self.assertEqual(operations, expected_operations)
233232

233+
@unittest.expectedFailure # TODO: RUSTPYTHON
234234
def test_parse_bytes(self):
235235
out = self.Outputter()
236236
parser = expat.ParserCreate(namespace_separator='!')
@@ -243,6 +243,7 @@ def test_parse_bytes(self):
243243
# Issue #6697.
244244
self.assertRaises(AttributeError, getattr, parser, '\uD800')
245245

246+
@unittest.expectedFailure # TODO: RUSTPYTHON
246247
def test_parse_str(self):
247248
out = self.Outputter()
248249
parser = expat.ParserCreate(namespace_separator='!')
@@ -253,6 +254,7 @@ def test_parse_str(self):
253254
operations = out.out
254255
self._verify_parse_output(operations)
255256

257+
@unittest.expectedFailure # TODO: RUSTPYTHON
256258
def test_parse_file(self):
257259
# Try parsing a file
258260
out = self.Outputter()
@@ -278,6 +280,126 @@ def test_parse_again(self):
278280
self.assertEqual(expat.ErrorString(cm.exception.code),
279281
expat.errors.XML_ERROR_FINISHED)
280282

283+
@unittest.expectedFailure # TODO: RUSTPYTHON
284+
@support.subTests('encoding', [
285+
'utf-8', 'utf-16', 'utf-16be', 'utf-16le',
286+
'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5',
287+
'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
288+
'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16',
289+
'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852',
290+
'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862',
291+
'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125',
292+
'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
293+
'cp1256', 'cp1257', 'cp1258',
294+
'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2',
295+
'mac-roman', 'mac-turkish',
296+
'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154',
297+
])
298+
def test_supported_encodings(self, encoding):
299+
out = self.Outputter()
300+
parser = expat.ParserCreate()
301+
self._hookup_callbacks(parser, out)
302+
c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0]
303+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
304+
f'<root>{c}</root>').encode(encoding)
305+
parser.Parse(data, True)
306+
self.assertEqual(out.out, [
307+
('XML declaration', ('1.0', encoding, -1)),
308+
"Start element: 'root' {}",
309+
f'Character data: {c!r}',
310+
"End element: 'root'",
311+
])
312+
313+
@unittest.expectedFailure # TODO: RUSTPYTHON
314+
@support.subTests('encoding', [
315+
'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be',
316+
'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic',
317+
])
318+
def test_supported_encodings2(self, encoding):
319+
out = self.Outputter()
320+
parser = expat.ParserCreate()
321+
self._hookup_callbacks(parser, out)
322+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
323+
'<!-- коментар -->'
324+
'<корінь атрибут="значення">зміст</корінь>').encode(encoding)
325+
parser.Parse(data, True)
326+
self.assertEqual(out.out, [
327+
('XML declaration', ('1.0', encoding, -1)),
328+
"Comment: ' коментар '",
329+
"Start element: 'корінь' {'атрибут': 'значення'}",
330+
"Character data: 'зміст'",
331+
"End element: 'корінь'",
332+
])
333+
334+
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised
335+
@support.subTests('encoding', [
336+
'UTF-7',
337+
"Big5-HKSCS", "Big5",
338+
"cp932", "cp949", "cp950",
339+
"EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR",
340+
"GB18030", "GB2312", "GBK",
341+
"ISO-2022-KR",
342+
"johab",
343+
"Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213",
344+
])
345+
def test_unsupported_encodings(self, encoding):
346+
parser = expat.ParserCreate()
347+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
348+
'<root></root>').encode(encoding)
349+
with self.assertRaises(ValueError):
350+
parser.Parse(data, True)
351+
352+
parser = expat.ParserCreate()
353+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
354+
'<root></root>').encode()
355+
with self.assertRaises(ValueError):
356+
parser.Parse(data, True)
357+
358+
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ExpatError not raised
359+
@support.subTests('encoding', [
360+
'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875',
361+
'cp1026', 'cp1140',
362+
'mac_arabic', 'mac_farsi',
363+
])
364+
def test_incompatible_encodings(self, encoding):
365+
parser = expat.ParserCreate()
366+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
367+
'<root></root>').encode(encoding)
368+
with self.assertRaises(expat.ExpatError):
369+
parser.Parse(data, True)
370+
371+
parser = expat.ParserCreate()
372+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
373+
'<root></root>').encode()
374+
with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'):
375+
parser.Parse(data, True)
376+
377+
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: LookupError not raised
378+
@support.subTests('encoding', [
379+
'hex_codec', 'rot_13',
380+
])
381+
def test_non_text_encodings(self, encoding):
382+
parser = expat.ParserCreate()
383+
data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
384+
'<root></root>').encode()
385+
with self.assertRaises(LookupError):
386+
parser.Parse(data, True)
387+
388+
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UnicodeError not raised
389+
def test_undefined_encoding(self):
390+
parser = expat.ParserCreate()
391+
data = b'<?xml version="1.0" encoding="undefined"?>\n<root></root>'
392+
with self.assertRaises(UnicodeError):
393+
parser.Parse(data, True)
394+
395+
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: LookupError not raised
396+
def test_unknown_encoding(self):
397+
parser = expat.ParserCreate()
398+
data = b'<?xml version="1.0" encoding="xyz"?>\n<root></root>'
399+
with self.assertRaises(LookupError):
400+
parser.Parse(data, True)
401+
402+
281403
class NamespaceSeparatorTest(unittest.TestCase):
282404
def test_legal(self):
283405
# Tests that make sure we get errors when the namespace_separator value
@@ -688,6 +810,20 @@ def test_change_size_2(self):
688810
parser.Parse(xml2, True)
689811
self.assertEqual(self.n, 4)
690812

813+
@support.requires_resource('cpu')
814+
@support.requires_resource('walltime')
815+
@support.bigmemtest(size=2**31, memuse=4, dry_run=False)
816+
def test_large_character_data_does_not_crash(self):
817+
# See https://github.com/python/cpython/issues/148441
818+
parser = expat.ParserCreate()
819+
parser.buffer_text = True
820+
parser.buffer_size = 2**31 - 1 # INT_MAX
821+
N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX
822+
self.assertGreater(N, parser.buffer_size)
823+
parser.CharacterDataHandler = lambda text: None
824+
xml_data = b"<r>" + b"A" * N + b"</r>"
825+
self.assertEqual(parser.Parse(xml_data, True), 1)
826+
691827
class ElementDeclHandlerTest(unittest.TestCase):
692828
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by Parse
693829
def test_trigger_leak(self):
@@ -1087,6 +1223,92 @@ def test_set_maximum_amplification__fail_for_subparser(self):
10871223
self.assert_root_parser_failure(setter, 123.45)
10881224

10891225

1226+
@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0")
1227+
class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase):
1228+
1229+
def assert_rejected(self, func, /, *args, **kwargs):
1230+
"""Check that func(*args, **kwargs) hits the allocation limit."""
1231+
msg = (
1232+
r"limit on input amplification factor \(from DTD and entities\) "
1233+
r"breached: line \d+, column \d+"
1234+
)
1235+
self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
1236+
1237+
def set_activation_threshold(self, parser, threshold):
1238+
return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold)
1239+
1240+
def set_maximum_amplification(self, parser, max_factor):
1241+
return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor)
1242+
1243+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1244+
def test_set_activation_threshold__threshold_reached(self):
1245+
parser = expat.ParserCreate()
1246+
# Choose a threshold expected to be always reached.
1247+
self.set_activation_threshold(parser, 3)
1248+
# Check that the threshold is reached by choosing a small factor
1249+
# and a payload whose peak amplification factor exceeds it.
1250+
self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1251+
payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1252+
self.assert_rejected(parser.Parse, payload, True)
1253+
1254+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1255+
def test_set_activation_threshold__threshold_not_reached(self):
1256+
parser = expat.ParserCreate()
1257+
# Choose a threshold expected to be never reached.
1258+
self.set_activation_threshold(parser, pow(10, 5))
1259+
# Check that the threshold is reached by choosing a small factor
1260+
# and a payload whose peak amplification factor exceeds it.
1261+
self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1262+
payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1263+
self.assertIsNotNone(parser.Parse(payload, True))
1264+
1265+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1266+
def test_set_maximum_amplification__amplification_exceeded(self):
1267+
parser = expat.ParserCreate()
1268+
# Unconditionally enable maximum activation factor.
1269+
self.set_activation_threshold(parser, 0)
1270+
# Choose a max amplification factor expected to always be exceeded.
1271+
self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1272+
# Craft a payload for which the peak amplification factor is > 1.0.
1273+
payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1274+
self.assert_rejected(parser.Parse, payload, True)
1275+
1276+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1277+
def test_set_maximum_amplification__amplification_not_exceeded(self):
1278+
parser = expat.ParserCreate()
1279+
# Unconditionally enable maximum activation factor.
1280+
self.set_activation_threshold(parser, 0)
1281+
# Choose a max amplification factor expected to never be exceeded.
1282+
self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
1283+
# Craft a payload for which the peak amplification factor is < 1e4.
1284+
payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1285+
self.assertIsNotNone(parser.Parse(payload, True))
1286+
1287+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1288+
def test_set_activation_threshold__fail_for_subparser(self):
1289+
return super().test_set_activation_threshold__fail_for_subparser()
1290+
1291+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1292+
def test_set_activation_threshold__invalid_threshold_type(self):
1293+
return super().test_set_activation_threshold__invalid_threshold_type()
1294+
1295+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1296+
def test_set_maximum_amplification__fail_for_subparser(self):
1297+
return super().test_set_maximum_amplification__fail_for_subparser()
1298+
1299+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1300+
def test_set_maximum_amplification__infinity(self):
1301+
return super().test_set_maximum_amplification__infinity()
1302+
1303+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1304+
def test_set_maximum_amplification__invalid_max_factor_range(self):
1305+
return super().test_set_maximum_amplification__invalid_max_factor_range()
1306+
1307+
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification'
1308+
def test_set_maximum_amplification__invalid_max_factor_type(self):
1309+
return super().test_set_maximum_amplification__invalid_max_factor_type()
1310+
1311+
10901312
@unittest.skipIf(not hasattr(expat.XMLParserType,
10911313
"SetAllocTrackerMaximumAmplification"),
10921314
"requires Python compiled with Expat >= 2.7.2")

0 commit comments

Comments
 (0)