Skip to content

Commit d4a94d3

Browse files
author
Luke Lovett
committed
PYTHON-673 python 2/3 single-source for the bson module
1 parent e0b52ba commit d4a94d3

File tree

9 files changed

+108
-74
lines changed

9 files changed

+108
-74
lines changed

‎bson/__init__.py‎

Lines changed: 46 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,13 @@
3131
from bson.max_key import MaxKey
3232
from bson.min_key import MinKey
3333
from bson.objectid import ObjectId
34-
from bson.py3compat import b, binary_type
34+
from bson.py3compat import (b,
35+
PY3,
36+
binary_type,
37+
iteritems,
38+
text_type,
39+
string_type,
40+
reraise)
3541
from bson.regex import Regex
3642
from bson.son import SON, RE_TYPE
3743
from bson.timestamp import Timestamp
@@ -50,8 +56,8 @@
5056
except ImportError:
5157
_use_uuid = False
5258

53-
PY3 = sys.version_info[0] == 3
54-
59+
if PY3:
60+
long = int
5561

5662
MAX_INT32 = 2147483647
5763
MIN_INT32 = -2147483648
@@ -61,33 +67,30 @@
6167
EPOCH_AWARE = datetime.datetime.fromtimestamp(0, utc)
6268
EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0)
6369

64-
# Create constants compatible with all versions of
65-
# python from 2.4 forward. In 2.x b("foo") is just
66-
# "foo". In 3.x it becomes b"foo".
67-
EMPTY = b("")
68-
ZERO = b("\x00")
69-
ONE = b("\x01")
70-
71-
BSONNUM = b("\x01") # Floating point
72-
BSONSTR = b("\x02") # UTF-8 string
73-
BSONOBJ = b("\x03") # Embedded document
74-
BSONARR = b("\x04") # Array
75-
BSONBIN = b("\x05") # Binary
76-
BSONUND = b("\x06") # Undefined
77-
BSONOID = b("\x07") # ObjectId
78-
BSONBOO = b("\x08") # Boolean
79-
BSONDAT = b("\x09") # UTC Datetime
80-
BSONNUL = b("\x0A") # Null
81-
BSONRGX = b("\x0B") # Regex
82-
BSONREF = b("\x0C") # DBRef
83-
BSONCOD = b("\x0D") # Javascript code
84-
BSONSYM = b("\x0E") # Symbol
85-
BSONCWS = b("\x0F") # Javascript code with scope
86-
BSONINT = b("\x10") # 32bit int
87-
BSONTIM = b("\x11") # Timestamp
88-
BSONLON = b("\x12") # 64bit int
89-
BSONMIN = b("\xFF") # Min key
90-
BSONMAX = b("\x7F") # Max key
70+
EMPTY = b""
71+
ZERO = b"\x00"
72+
ONE = b"\x01"
73+
74+
BSONNUM = b"\x01" # Floating point
75+
BSONSTR = b"\x02" # UTF-8 string
76+
BSONOBJ = b"\x03" # Embedded document
77+
BSONARR = b"\x04" # Array
78+
BSONBIN = b"\x05" # Binary
79+
BSONUND = b"\x06" # Undefined
80+
BSONOID = b"\x07" # ObjectId
81+
BSONBOO = b"\x08" # Boolean
82+
BSONDAT = b"\x09" # UTC Datetime
83+
BSONNUL = b"\x0A" # Null
84+
BSONRGX = b"\x0B" # Regex
85+
BSONREF = b"\x0C" # DBRef
86+
BSONCOD = b"\x0D" # Javascript code
87+
BSONSYM = b"\x0E" # Symbol
88+
BSONCWS = b"\x0F" # Javascript code with scope
89+
BSONINT = b"\x10" # 32bit int
90+
BSONTIM = b"\x11" # Timestamp
91+
BSONLON = b"\x12" # 64bit int
92+
BSONMIN = b"\xFF" # Min key
93+
BSONMAX = b"\x7F" # Max key
9194

9295

9396
def _get_int(data, position, as_class=None,
@@ -117,12 +120,7 @@ def _get_c_string(data, position, length=None):
117120

118121

119122
def _make_c_string(string, check_null=False):
120-
if isinstance(string, unicode):
121-
if check_null and "\x00" in string:
122-
raise InvalidDocument("BSON keys / regex patterns must not "
123-
"contain a NULL character")
124-
return string.encode("utf-8") + ZERO
125-
else:
123+
if isinstance(string, bytes):
126124
if check_null and ZERO in string:
127125
raise InvalidDocument("BSON keys / regex patterns must not "
128126
"contain a NULL character")
@@ -132,6 +130,11 @@ def _make_c_string(string, check_null=False):
132130
except UnicodeError:
133131
raise InvalidStringData("strings in documents must be valid "
134132
"UTF-8: %r" % string)
133+
else:
134+
if check_null and "\x00" in string:
135+
raise InvalidDocument("BSON keys / regex patterns must not "
136+
"contain a NULL character")
137+
return string.encode("utf-8") + ZERO
135138

136139

137140
def _get_number(data, position, as_class, tz_aware, uuid_subtype, compile_re):
@@ -349,7 +352,7 @@ def _bson_to_dict(data, as_class, tz_aware, uuid_subtype, compile_re):
349352

350353

351354
def _element_to_bson(key, value, check_keys, uuid_subtype):
352-
if not isinstance(key, basestring):
355+
if not isinstance(key, string_type):
353356
raise InvalidDocument("documents must have only string keys, "
354357
"key was %r" % key)
355358

@@ -405,7 +408,7 @@ def _element_to_bson(key, value, check_keys, uuid_subtype):
405408
cstring = _make_c_string(value)
406409
length = struct.pack("<i", len(cstring))
407410
return BSONSTR + name + length + cstring
408-
if isinstance(value, unicode):
411+
if isinstance(value, text_type):
409412
cstring = _make_c_string(value)
410413
length = struct.pack("<i", len(cstring))
411414
return BSONSTR + name + length + cstring
@@ -429,7 +432,7 @@ def _element_to_bson(key, value, check_keys, uuid_subtype):
429432
return BSONINT + name + struct.pack("<i", value)
430433
# 2to3 will convert long to int here since there is no long in python3.
431434
# That's OK. The previous if block will match instead.
432-
if isinstance(value, long):
435+
if not PY3 and isinstance(value, long):
433436
if value > MAX_INT64 or value < MIN_INT64:
434437
raise OverflowError("BSON can only handle up to 8-byte ints")
435438
return BSONLON + name + struct.pack("<q", value)
@@ -479,7 +482,7 @@ def _dict_to_bson(dict, check_keys, uuid_subtype, top_level=True):
479482
if top_level and "_id" in dict:
480483
elements.append(_element_to_bson("_id", dict["_id"],
481484
check_keys, uuid_subtype))
482-
for (key, value) in dict.iteritems():
485+
for (key, value) in iteritems(dict):
483486
if not top_level or key != "_id":
484487
elements.append(_element_to_bson(key, value,
485488
check_keys, uuid_subtype))
@@ -537,7 +540,9 @@ def decode_all(data, as_class=dict,
537540
except Exception:
538541
# Change exception type to InvalidBSON but preserve traceback.
539542
exc_type, exc_value, exc_tb = sys.exc_info()
540-
raise InvalidBSON, str(exc_value), exc_tb
543+
reraise(InvalidBSON, exc_value, exc_tb)
544+
545+
541546
if _use_c:
542547
decode_all = _cbson.decode_all
543548

‎bson/code.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
"""Tools for representing JavaScript code in BSON.
1616
"""
17+
from bson.py3compat import string_type
18+
1719

1820
class Code(str):
1921
"""BSON's JavaScript code type.
@@ -42,9 +44,9 @@ class Code(str):
4244
_type_marker = 13
4345

4446
def __new__(cls, code, scope=None, **kwargs):
45-
if not isinstance(code, basestring):
47+
if not isinstance(code, string_type):
4648
raise TypeError("code must be an "
47-
"instance of %s" % (basestring.__name__,))
49+
"instance of %s" % (string_type.__name__))
4850

4951
self = str.__new__(cls, code)
5052

‎bson/dbref.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from copy import deepcopy
1818

19+
from bson.py3compat import iteritems, string_type
1920
from bson.son import SON
2021

2122

@@ -49,12 +50,12 @@ def __init__(self, collection, id, database=None, _extra={}, **kwargs):
4950
5051
.. mongodoc:: dbrefs
5152
"""
52-
if not isinstance(collection, basestring):
53+
if not isinstance(collection, string_type):
5354
raise TypeError("collection must be an "
54-
"instance of %s" % (basestring.__name__,))
55-
if database is not None and not isinstance(database, basestring):
55+
"instance of %s" % string_type.__name__)
56+
if database is not None and not isinstance(database, string_type):
5657
raise TypeError("database must be an "
57-
"instance of %s" % (basestring.__name__,))
58+
"instance of %s" % string_type.__name__)
5859

5960
self.__collection = collection
6061
self.__id = id
@@ -110,7 +111,7 @@ def as_doc(self):
110111

111112
def __repr__(self):
112113
extra = "".join([", %s=%r" % (k, v)
113-
for k, v in self.__kwargs.iteritems()])
114+
for k, v in iteritems(self.__kwargs)])
114115
if self.database is None:
115116
return "DBRef(%r, %r%s)" % (self.collection, self.id, extra)
116117
return "DBRef(%r, %r, %r%s)" % (self.collection, self.id,

‎bson/json_util.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@
7777
import datetime
7878
import re
7979

80+
from bson.py3compat import iteritems, text_type
81+
8082
json_lib = True
8183
try:
8284
import json
@@ -97,7 +99,7 @@
9799
from bson.regex import Regex
98100
from bson.timestamp import Timestamp
99101

100-
from bson.py3compat import PY3, binary_type, string_types
102+
from bson.py3compat import PY3, binary_type, string_type
101103

102104

103105
_RE_OPT_TABLE = {
@@ -151,8 +153,8 @@ def _json_convert(obj):
151153
converted into json.
152154
"""
153155
if hasattr(obj, 'iteritems') or hasattr(obj, 'items'): # PY3 support
154-
return SON(((k, _json_convert(v)) for k, v in obj.iteritems()))
155-
elif hasattr(obj, '__iter__') and not isinstance(obj, string_types):
156+
return SON(((k, _json_convert(v)) for k, v in iteritems(obj)))
157+
elif hasattr(obj, '__iter__') and not isinstance(obj, string_type):
156158
return list((_json_convert(v) for v in obj))
157159
try:
158160
return default(obj)
@@ -230,7 +232,7 @@ def default(obj):
230232
flags += "u"
231233
if obj.flags & re.VERBOSE:
232234
flags += "x"
233-
if isinstance(obj.pattern, unicode):
235+
if isinstance(obj.pattern, text_type):
234236
pattern = obj.pattern
235237
else:
236238
pattern = obj.pattern.decode('utf-8')

‎bson/objectid.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,11 @@
3333
import time
3434

3535
from bson.errors import InvalidId
36-
from bson.py3compat import (PY3, b, binary_type, text_type,
37-
bytes_from_hex, string_types)
36+
from bson.py3compat import PY3, binary_type, text_type, bytes_from_hex
3837
from bson.tz_util import utc
3938

40-
EMPTY = b("")
41-
ZERO = b("\x00")
39+
EMPTY = b""
40+
ZERO = b"\x00"
4241

4342
def _machine_bytes():
4443
"""Get the machine portion of an ObjectId.
@@ -181,7 +180,7 @@ def __validate(self, oid):
181180
"""
182181
if isinstance(oid, ObjectId):
183182
self.__id = oid.__id
184-
elif isinstance(oid, string_types):
183+
elif isinstance(oid, (text_type, binary_type)):
185184
if len(oid) == 12:
186185
if isinstance(oid, binary_type):
187186
self.__id = oid

‎bson/py3compat.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020

2121
if PY3:
2222
import codecs
23-
2423
from io import BytesIO as StringIO
2524

2625
def b(s):
@@ -33,12 +32,23 @@ def b(s):
3332
# See http://python3porting.com/problems.html#nicer-solutions
3433
return codecs.latin_1_encode(s)[0]
3534

35+
def u(s):
36+
# PY3 strings may already be treated as unicode literals
37+
return s
38+
3639
def bytes_from_hex(h):
3740
return bytes.fromhex(h)
3841

42+
def iteritems(d):
43+
return d.items()
44+
45+
def reraise(exctype, value, trace=None):
46+
raise exctype(str(value)).with_traceback(trace)
47+
3948
binary_type = bytes
40-
text_type = str
41-
next_item = "__next__"
49+
text_type = str
50+
string_type = str
51+
integer_types = int
4252

4353
else:
4454
try:
@@ -50,13 +60,24 @@ def b(s):
5060
# See comments above. In python 2.x b('foo') is just 'foo'.
5161
return s
5262

63+
import codecs
64+
65+
def u(s):
66+
"""Replacement for unicode literal prefix."""
67+
return unicode(s.replace('\\', '\\\\'), 'unicode_escape')
68+
5369
def bytes_from_hex(h):
5470
return h.decode('hex')
5571

56-
binary_type = str
57-
# 2to3 will convert this to "str". That's okay
58-
# since we won't ever get here under python3.
59-
text_type = unicode
60-
next_item = "next"
72+
def iteritems(d):
73+
return d.iteritems()
6174

62-
string_types = (binary_type, text_type)
75+
# "raise x, y, z" raises SyntaxError in Python 3
76+
exec("""def reraise(exctype, value, trace=None):
77+
raise exctype, str(value), trace
78+
""")
79+
80+
binary_type = str
81+
string_type = basestring
82+
text_type = unicode
83+
integer_types = (int, long)

‎bson/regex.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import re
1919

2020
from bson.son import RE_TYPE
21-
from bson.py3compat import string_types
21+
from bson.py3compat import text_type, binary_type, string_type
2222

2323

2424
def str_flags_to_int(str_flags):
@@ -86,11 +86,11 @@ def __init__(self, pattern, flags=0):
8686
- `flags`: (optional) an integer bitmask, or a string of flag
8787
characters like "im" for IGNORECASE and MULTILINE
8888
"""
89-
if not isinstance(pattern, string_types):
89+
if not isinstance(pattern, (text_type, binary_type)):
9090
raise TypeError("pattern must be a string, not %s" % type(pattern))
9191
self.pattern = pattern
9292

93-
if isinstance(flags, string_types):
93+
if isinstance(flags, string_type):
9494
self.flags = str_flags_to_int(flags)
9595
elif isinstance(flags, int):
9696
self.flags = flags

‎bson/son.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
import copy
2222
import re
2323

24+
from bson.py3compat import iteritems
25+
26+
2427
# This sort of sucks, but seems to be as good as it gets...
2528
# This is essentially the same as re._pattern_type
2629
RE_TYPE = type(re.compile(""))
@@ -174,7 +177,7 @@ def pop(self, key, *args):
174177

175178
def popitem(self):
176179
try:
177-
k, v = self.iteritems().next()
180+
k, v = next(self.iteritems())
178181
except StopIteration:
179182
raise KeyError('container is empty')
180183
del self[k]
@@ -214,7 +217,7 @@ def __ne__(self, other):
214217
return not self == other
215218

216219
def __len__(self):
217-
return len(self.keys())
220+
return len(self.__keys)
218221

219222
def to_dict(self):
220223
"""Convert a SON document to a normal Python dictionary instance.
@@ -229,7 +232,7 @@ def transform_value(value):
229232
if isinstance(value, SON):
230233
value = dict(value)
231234
if isinstance(value, dict):
232-
for k, v in value.iteritems():
235+
for k, v in iteritems(value):
233236
value[k] = transform_value(v)
234237
return value
235238

0 commit comments

Comments
 (0)