forked from mongodb/mongo-python-driver
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_utf8.py
More file actions
76 lines (61 loc) · 2.28 KB
/
test_utf8.py
File metadata and controls
76 lines (61 loc) · 2.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import sys
sys.path[0:0] = [""]
from bson import BSON
from bson.errors import InvalidStringData
from bson.py3compat import PY3
from test import unittest
class TestUTF8(unittest.TestCase):
# Verify that python and bson have the same understanding of
# legal utf-8 if the first byte is 0xf4 (244)
def _assert_same_utf8_validation(self, data):
try:
data.decode('utf-8')
py_is_legal = True
except UnicodeDecodeError:
py_is_legal = False
try:
BSON.encode({'x': data})
bson_is_legal = True
except InvalidStringData:
bson_is_legal = False
self.assertEqual(py_is_legal, bson_is_legal, data)
@unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
def test_legal_utf8_full_coverage(self):
# This test takes 400 seconds. Which is too long to run each time.
# However it is the only one which covers all possible bit combinations
# in the 244 space.
b1 = chr(0xf4)
for b2 in map(chr, range(255)):
m2 = b1 + b2
self._assert_same_utf8_validation(m2)
for b3 in map(chr, range(255)):
m3 = m2 + b3
self._assert_same_utf8_validation(m3)
for b4 in map(chr, range(255)):
m4 = m3 + b4
self._assert_same_utf8_validation(m4)
# In python3:
# - 'bytes' are not checked with isLegalutf
# - 'unicode' We cannot create unicode objects with invalid utf8, since it
# would result in non valid code-points.
@unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
def test_legal_utf8_few_samples(self):
good_samples = [
'\xf4\x80\x80\x80',
'\xf4\x8a\x80\x80',
'\xf4\x8e\x80\x80',
'\xf4\x81\x80\x80',
]
for data in good_samples:
self._assert_same_utf8_validation(data)
bad_samples = [
'\xf4\x00\x80\x80',
'\xf4\x3a\x80\x80',
'\xf4\x7f\x80\x80',
'\xf4\x90\x80\x80',
'\xf4\xff\x80\x80',
]
for data in bad_samples:
self._assert_same_utf8_validation(data)
if __name__ == "__main__":
unittest.main()