Skip to content

Commit 8945a04

Browse files
author
James William Pye
committed
Implement numeric <-> decimal binary typio.
1 parent aad60bf commit 8945a04

4 files changed

Lines changed: 269 additions & 11 deletions

File tree

‎postgresql/protocol/typio.py‎

Lines changed: 149 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,16 @@
3737
long-long based time I/O with noday-intervals.
3838
"""
3939
import codecs
40-
from operator import itemgetter
40+
from operator import itemgetter, __mul__
41+
get0 = itemgetter(0)
42+
get1 = itemgetter(1)
43+
44+
from itertools import chain, starmap, repeat, groupby, cycle, islice
45+
from ..python.itertools import interlace
46+
4147
from abc import ABCMeta, abstractmethod
4248

43-
from decimal import Decimal
49+
from decimal import Decimal, DecimalTuple
4450
import datetime
4551

4652
try:
@@ -92,7 +98,7 @@ class Row(tuple):
9298
def __new__(subtype, iter, attmap = {}):
9399
if isinstance(iter, dict):
94100
iter = [
95-
iter.get(k) for k,_ in sorted(attmap.items(), key = itemgetter(1))
101+
iter.get(k) for k,_ in sorted(attmap.items(), key = get1)
96102
]
97103
rob = tuple.__new__(subtype, iter)
98104
rob.attmap = attmap
@@ -310,9 +316,147 @@ def circle_pack(x):
310316
def circle_unpack(x):
311317
lambda x: circle_unpack(ts.circle_unpack(x))
312318

319+
##
320+
# numeric is represented using:
321+
# ndigits, the number of *numeric* digits.
322+
# weight, the *numeric* digits "left" of the decimal point
323+
# sign, negativity. see `numeric_signs` below
324+
# dscale, *display* precision. used to identify exponent.
325+
#
326+
# NOTE: A numeric digit is actually four digits in the representation.
327+
#
328+
# Python's Decimal consists of:
329+
# sign, negativity.
330+
# digits, sequence of int()'s
331+
# exponent, digits that fall to the right of the decimal point
332+
numeric_negative = 16384
333+
334+
def numeric_pack(x,
335+
numeric_digit_length : "number of decimal digits in a numeric digit" = 4
336+
):
337+
if not isinstance(x, Decimal):
338+
x = Decimal(x)
339+
x = x.as_tuple()
340+
341+
# normalize trailing zeros (truncate em')
342+
# this is important in order to get the weight and padding correct
343+
# and to avoid packing superfluous data which will make pg angry.
344+
trailing_zeros = 0
345+
weight = 0
346+
if x.exponent < 0:
347+
# only attempt to truncate if there are digits after the point,
348+
##
349+
for i in range(-1, max(-len(x.digits), x.exponent)-1, -1):
350+
if x.digits[i] != 0:
351+
break
352+
trailing_zeros += 1
353+
# truncate trailing zeros right of the decimal point
354+
# this *is* the case as exponent < 0.
355+
if trailing_zeros:
356+
digits = x.digits[:-trailing_zeros]
357+
else:
358+
digits = x.digits
359+
# the entire exponent is just trailing zeros(zero-weight).
360+
rdigits = -(x.exponent + trailing_zeros)
361+
ldigits = len(digits) - rdigits
362+
rpad = rdigits % numeric_digit_length
363+
if rpad:
364+
rpad = numeric_digit_length - rpad
365+
else:
366+
# Need the weight to be divisible by four,
367+
# so append zeros onto digits until it is.
368+
r = (x.exponent % numeric_digit_length)
369+
if x.exponent and r:
370+
digits = x.digits + ((0,) * r)
371+
weight = (x.exponent - r)
372+
else:
373+
digits = x.digits
374+
weight = x.exponent
375+
# The exponent is not evenly divisible by four, so
376+
# the weight can't simple be x.exponent as it doesn't
377+
# match the size of the numeric digit.
378+
ldigits = len(digits)
379+
# no fractional quantity.
380+
rdigits = 0
381+
rpad = 0
382+
383+
lpad = ldigits % numeric_digit_length
384+
if lpad:
385+
lpad = numeric_digit_length - lpad
386+
weight += (ldigits + lpad)
387+
388+
digit_groups = map(
389+
get1,
390+
groupby(
391+
zip(
392+
# group by numeric digit size
393+
# every four digits make up a numeric digit
394+
cycle((0,) * numeric_digit_length + (1,) * numeric_digit_length),
395+
396+
# multiply each digit appropriately
397+
# for the eventual sum() into a numeric digit
398+
starmap(
399+
__mul__,
400+
zip(
401+
# pad with leading zeros to make
402+
# the cardinality of the digit sequence
403+
# to be evenly divisible by four,
404+
# the numeric digit size.
405+
chain(
406+
repeat(0, lpad),
407+
digits,
408+
repeat(0, rpad),
409+
),
410+
cycle([10**x for x in range(numeric_digit_length-1, -1, -1)]),
411+
)
412+
),
413+
),
414+
get0,
415+
),
416+
)
417+
return ts.numeric_pack((
418+
(
419+
(ldigits + rdigits + lpad + rpad) // numeric_digit_length, # ndigits
420+
(weight // numeric_digit_length) - 1, # numeric weight
421+
numeric_negative if x.sign == 1 else x.sign, # sign
422+
- x.exponent if x.exponent < 0 else 0, # dscale
423+
),
424+
list(map(sum, ([get1(y) for y in x] for x in digit_groups))),
425+
))
426+
427+
def numeric_convert_digits(d):
428+
i = iter(d)
429+
for x in str(next(i)):
430+
# no leading zeros
431+
yield int(x)
432+
# leading digit should not include zeros
433+
for y in i:
434+
for x in str(y).rjust(4, '0'):
435+
yield int(x)
436+
437+
numeric_signs = {
438+
16384 : 1,
439+
}
440+
441+
def numeric_unpack(x):
442+
header, digits = ts.numeric_unpack(x)
443+
npad = (header[3] - ((header[0] - (header[1] + 1)) * 4))
444+
return Decimal(
445+
DecimalTuple(
446+
sign = numeric_signs.get(header[2], header[2]),
447+
digits = chain(
448+
numeric_convert_digits(digits),
449+
(0,) * npad
450+
) if npad >= 0 else list(
451+
numeric_convert_digits(digits)
452+
)[:npad],
453+
exponent = -header[3]
454+
)
455+
)
456+
313457
# Map type oids to a (pack, unpack) pair.
314458
oid_to_io = {
315-
pg_types.NUMERICOID : (None, Decimal),
459+
pg_types.NUMERICOID : (numeric_pack, numeric_unpack),
316460

317461
pg_types.DATEOID : (date_pack, date_unpack),
318462

@@ -426,7 +570,7 @@ def unpack_a_record(data):
426570
def pack_a_record(data):
427571
if isinstance(data, dict):
428572
data = [
429-
data.get(k) for k,_ in sorted(attmap.items(), key = itemgetter(1))
573+
data.get(k) for k,_ in sorted(attmap.items(), key = get1)
430574
]
431575
return ts.record_pack(
432576
tuple(zip(typids, transform_record(cio, data, 0)))

‎postgresql/protocol/typstruct.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ def mktime64(seconds_ms):
8585
dl_pack, dl_unpack = mk_pack("dl")
8686
ql_pack, ql_unpack = mk_pack("ql")
8787

88+
hhhh_pack, hhhh_unpack = mk_pack("hhhh")
8889

8990
int2_pack, int2_unpack = short_pack, short_unpack
9091
int4_pack, int4_unpack = long_pack, long_unpack
@@ -101,6 +102,14 @@ def mktime64(seconds_ms):
101102
lseg_pack = box_pack = dddd_pack
102103
lseg_unpack = box_unpack = dddd_unpack
103104

105+
def numeric_pack(data):
106+
(header, numbers) = data
107+
return hhhh_pack(header) + struct.pack("!%dh"%(len(numbers),), *numbers)
108+
109+
def numeric_unpack(data):
110+
header = hhhh_unpack(data[:8])
111+
return (header, struct.unpack("!8x%dh"%((len(data)-8) // 2,), data))
112+
104113
def path_pack(data):
105114
"""
106115
Given a sequence of point data, pack it into a path's serialized form.
@@ -493,7 +502,7 @@ def return_arg(arg):
493502
pg_types.INT2OID : (int2_pack, int2_unpack),
494503
pg_types.INT4OID : (int4_pack, int4_unpack),
495504
pg_types.INT8OID : (int8_pack, int8_unpack),
496-
pg_types.NUMERICOID : literal,
505+
pg_types.NUMERICOID : (numeric_pack, numeric_unpack),
497506

498507
pg_types.OIDOID : (oid_pack, oid_unpack),
499508
pg_types.XIDOID : (xid_pack, xid_unpack),

‎postgresql/test/test_driver.py‎

Lines changed: 100 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import threading
99
import time
1010
import datetime
11+
import decimal
1112
from itertools import chain
1213

1314
import postgresql.types as pg_types
@@ -32,6 +33,98 @@
3233
-1, 0, 1,
3334
),
3435
),
36+
('numeric', (
37+
-(2**64),
38+
2**64,
39+
-(2**128),
40+
2**128,
41+
-1, 0, 1,
42+
decimal.Decimal("0.00000000000000"),
43+
decimal.Decimal("1.00000000000000"),
44+
decimal.Decimal("-1.00000000000000"),
45+
decimal.Decimal("-2.00000000000000"),
46+
decimal.Decimal("1000000000000000.00000000000000"),
47+
decimal.Decimal("-0.00000000000000"),
48+
decimal.Decimal(1234),
49+
decimal.Decimal(-1234),
50+
decimal.Decimal("1234000000.00088883231"),
51+
decimal.Decimal(str(1234.00088883231)),
52+
decimal.Decimal("3123.23111"),
53+
decimal.Decimal("-3123000000.23111"),
54+
decimal.Decimal("3123.2311100000"),
55+
decimal.Decimal("-03123.0023111"),
56+
decimal.Decimal("3123.23111"),
57+
decimal.Decimal("3123.23111"),
58+
decimal.Decimal("10000.23111"),
59+
decimal.Decimal("100000.23111"),
60+
decimal.Decimal("1000000.23111"),
61+
decimal.Decimal("10000000.23111"),
62+
decimal.Decimal("100000000.23111"),
63+
decimal.Decimal("1000000000.23111"),
64+
decimal.Decimal("1000000000.3111"),
65+
decimal.Decimal("1000000000.111"),
66+
decimal.Decimal("1000000000.11"),
67+
decimal.Decimal("100000000.0"),
68+
decimal.Decimal("10000000.0"),
69+
decimal.Decimal("1000000.0"),
70+
decimal.Decimal("100000.0"),
71+
decimal.Decimal("10000.0"),
72+
decimal.Decimal("1000.0"),
73+
decimal.Decimal("100.0"),
74+
decimal.Decimal("100"),
75+
decimal.Decimal("100.1"),
76+
decimal.Decimal("100.12"),
77+
decimal.Decimal("100.123"),
78+
decimal.Decimal("100.1234"),
79+
decimal.Decimal("100.12345"),
80+
decimal.Decimal("100.123456"),
81+
decimal.Decimal("100.1234567"),
82+
decimal.Decimal("100.12345679"),
83+
decimal.Decimal("100.123456790"),
84+
decimal.Decimal("100.123456790000000000000000"),
85+
decimal.Decimal("1.0"),
86+
decimal.Decimal("0.0"),
87+
decimal.Decimal("-1.0"),
88+
decimal.Decimal("1.0E-1000"),
89+
decimal.Decimal("1.0E1000"),
90+
decimal.Decimal("1.0E10000"),
91+
decimal.Decimal("1.0E-10000"),
92+
decimal.Decimal("1.0E15000"),
93+
decimal.Decimal("1.0E-15000"),
94+
decimal.Decimal("1.0E-16382"),
95+
decimal.Decimal("1.0E32767"),
96+
decimal.Decimal("0.000000000000000000000000001"),
97+
decimal.Decimal("0.000000000000010000000000001"),
98+
decimal.Decimal("0.00000000000000000000000001"),
99+
decimal.Decimal("0.00000000100000000000000001"),
100+
decimal.Decimal("0.0000000000000000000000001"),
101+
decimal.Decimal("0.000000000000000000000001"),
102+
decimal.Decimal("0.00000000000000000000001"),
103+
decimal.Decimal("0.0000000000000000000001"),
104+
decimal.Decimal("0.000000000000000000001"),
105+
decimal.Decimal("0.00000000000000000001"),
106+
decimal.Decimal("0.0000000000000000001"),
107+
decimal.Decimal("0.000000000000000001"),
108+
decimal.Decimal("0.00000000000000001"),
109+
decimal.Decimal("0.0000000000000001"),
110+
decimal.Decimal("0.000000000000001"),
111+
decimal.Decimal("0.00000000000001"),
112+
decimal.Decimal("0.0000000000001"),
113+
decimal.Decimal("0.000000000001"),
114+
decimal.Decimal("0.00000000001"),
115+
decimal.Decimal("0.0000000001"),
116+
decimal.Decimal("0.000000001"),
117+
decimal.Decimal("0.00000001"),
118+
decimal.Decimal("0.0000001"),
119+
decimal.Decimal("0.000001"),
120+
decimal.Decimal("0.00001"),
121+
decimal.Decimal("0.0001"),
122+
decimal.Decimal("0.001"),
123+
decimal.Decimal("0.01"),
124+
decimal.Decimal("0.1"),
125+
# these require some weight transfer
126+
),
127+
),
35128
('bytea', (
36129
bytes(range(256)),
37130
bytes(range(255, -1, -1)),
@@ -96,7 +189,7 @@ def raise_exc(l):
96189
e, v, tb = rl[0]
97190
raise v
98191
self.failUnlessRaises(pg_exc.QueryCanceledError, raise_exc, rl)
99-
192+
100193
def testCopyToSTDOUT(self):
101194
with self.db.xact:
102195
self.db.execute("CREATE TABLE foo (i int)")
@@ -267,15 +360,17 @@ def testBatchDDL(self):
267360
def testTypes(self):
268361
'test basic object I/O--input must equal output'
269362
for (typname, sample_data) in type_samples:
270-
pb = self.db.prepare("SELECT $1::" + typname)
363+
pb = self.db.prepare(
364+
"SELECT $1::" + typname + ", $1::" + typname + "::text"
365+
)
271366
for sample in sample_data:
272-
rsample = pb.first(sample)
367+
rsample, tsample = pb.first(sample)
273368
if isinstance(rsample, pg_types.Array):
274369
rsample = rsample.nest()
275370
self.failUnless(
276371
rsample == sample,
277-
"failed to return %s object data as-is; gave %r, received %r" %(
278-
typname, sample, rsample
372+
"failed to return %s object data as-is; gave %r, received %r(%r::text)" %(
373+
typname, sample, rsample, tsample
279374
)
280375
)
281376

‎postgresql/test/test_protocol.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
##
55
import unittest
66
import struct
7+
import decimal
78
import postgresql.protocol.element3 as e3
89
import postgresql.protocol.client3 as c3
910
import postgresql.protocol.pbuffer as p_buffer_module
@@ -391,6 +392,15 @@ def testTransactionSamplesAll(self):
391392
(-3, b'\xff\xff\xff\xff\xff\xff\xff\xfd'),
392393
],
393394

395+
pg_types.NUMERICOID : [
396+
(((0,0,0,0),[]), b'\x00'*2*4),
397+
(((0,0,0,0),[1]), b'\x00'*2*4 + b'\x00\x01'),
398+
(((1,0,0,0),[1]), b'\x00\x01' + b'\x00'*2*3 + b'\x00\x01'),
399+
(((1,1,1,1),[1]), b'\x00\x01'*4 + b'\x00\x01'),
400+
(((1,1,1,1),[1,2]), b'\x00\x01'*4 + b'\x00\x01\x00\x02'),
401+
(((1,1,1,1),[1,2,3]), b'\x00\x01'*4 + b'\x00\x01\x00\x02\x00\x03'),
402+
],
403+
394404
pg_types.BITOID : [
395405
(False, b'\x00\x00\x00\x01\x00'),
396406
(True, b'\x00\x00\x00\x01\x01'),

0 commit comments

Comments
 (0)