Skip to content

Commit b51e9c7

Browse files
author
James William Pye
committed
Various cleanups.
Add performance tests, and migrate some of the greentrunk API tests into test_driver.py.
1 parent dc34358 commit b51e9c7

7 files changed

Lines changed: 530 additions & 20 deletions

File tree

‎postgresql/driver/python.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
# -*- encoding: utf-8 -*-
2-
# $Id$
31
##
42
# copyright 2007, pg/python project.
53
# http://python.projects.postgresql.org

‎postgresql/environ.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# http://python.projects.postgresql.org
44
##
55
"""
6-
PostgreSQL client environment configuration utilities
6+
PostgreSQL client environment variable extraction
77
88
This project provides a relatively simple way to translate an environment
99
mapping into a more normalized connection configuration dictionary.
@@ -44,8 +44,8 @@
4444
import sys
4545
import configparser
4646

47-
from postgresql.utility.config import instance as pg_config
48-
import postgresql.utility.client.iri as pg_iri
47+
from postgresql.pg_config import dictionary as pg_config
48+
import postgresql.iri as pg_iri
4949
import postgresql.strings as pg_str
5050

5151
# Environment variables that require no transformation.
@@ -99,8 +99,7 @@ def service_data(d, env):
9999
try:
100100
service_file = pg_config(
101101
env.get('PGCONFIG', 'pg_config'),
102-
validate = False
103-
).sysconfdir
102+
)["sysconfdir"]
104103
except:
105104
service_file = os.path.curdir
106105

‎postgresql/strings.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# -*- encoding: utf-8 -*-
21
##
32
# copyright 2008, pg/python project.
43
# http://python.projects.postgresql.org

‎postgresql/test/cursor_integrity.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# -*- encoding: utf-8 -*-
2+
# $Id: test_integrity.py,v 1.5 2008/04/01 03:33:43 jwp Exp $
3+
##
4+
# copyright 2006, pg/python project.
5+
# http://python.projects.postgresql.org
6+
##
7+
import os
8+
import unittest
9+
import random
10+
import itertools
11+
12+
iot = '_dst'
13+
if __name__ == '__main__':
14+
execute("CREATE TEMP TABLE _dst (i bigint)")
15+
copyin = query("COPY _dst FROM STDIN")
16+
loadin = query("INSERT INTO _dst VALUES ($1)")
17+
18+
getq = "SELECT i FROM generate_series(0, %d) AS g(i)"
19+
copy = "COPY (%s) TO STDOUT"
20+
21+
def random_read(curs, remaining_rows):
22+
"""
23+
Read from one of the three methods using a random amount if sized.
24+
- 50% chance of curs.read(random())
25+
- 40% chance of next()
26+
- 10% chance of read() # no count
27+
"""
28+
if random.random() > 0.5:
29+
rrows = random.randrange(0, remaining_rows)
30+
return curs.read(rrows), rrows
31+
elif random.random() < 0.1:
32+
return curs.read(), -1
33+
else:
34+
try:
35+
return [curs.next()], 1
36+
except StopIteration:
37+
return [], 1
38+
39+
def random_select_get(limit):
40+
return query(getq %(limit - 1,))
41+
42+
def random_copy_get(limit):
43+
return query(copy %(getq %(limit - 1,),))
44+
45+
class test_integrity(unittest.TestCase):
46+
"""
47+
test the integrity of the get and put interfaces on queries
48+
and result handles.
49+
"""
50+
def test_select(self):
51+
total = 0
52+
while total < 10000:
53+
limit = random.randrange(500000)
54+
read = 0
55+
total += limit
56+
p = random_select_get(limit)()
57+
last = ([(-1,)], 1)
58+
completed = [last[0]]
59+
while True:
60+
next = random_read(p, (limit - read) or 10)
61+
thisread = len(next[0])
62+
read += thisread
63+
completed.append(next[0])
64+
if thisread:
65+
self.failUnlessEqual(
66+
last[0][-1][0], next[0][0][0] - 1,
67+
"first row(-1) of next failed to match the last row of the previous"
68+
)
69+
last = next
70+
elif next[1] != 0:
71+
# done
72+
break
73+
self.failUnlessEqual(read, limit)
74+
self.failUnlessEqual(list(range(-1, limit)), [
75+
x[0] for x in itertools.chain(*completed)
76+
])
77+
78+
def test_insert(self):
79+
pass
80+
81+
if 'pg' in dir(__builtins__) and pg.version_info >= (8,2,0):
82+
def test_copy_out(self):
83+
total = 0
84+
while total < 10000000:
85+
limit = random.randrange(500000)
86+
read = 0
87+
total += limit
88+
p = random_copy_get(limit)()
89+
last = ([-1], 1)
90+
completed = [last[0]]
91+
while True:
92+
next = random_read(p, (limit - read) or 10)
93+
next = ([int(x) for x in next[0]], next[1])
94+
thisread = len(next[0])
95+
read += thisread
96+
completed.append(next[0])
97+
if thisread:
98+
self.failUnlessEqual(
99+
last[0][-1], next[0][0] - 1,
100+
"first row(-1) of next failed to match the last row of the previous"
101+
)
102+
last = next
103+
elif next[1] != 0:
104+
# done
105+
break
106+
self.failUnlessEqual(read, limit)
107+
self.failUnlessEqual(
108+
list(range(-1, limit)),
109+
list(itertools.chain(*completed))
110+
)
111+
112+
def test_copy_in(self):
113+
pass
114+
115+
if __name__ == '__main__':
116+
from types import ModuleType
117+
this = ModuleType("this")
118+
this.__dict__.update(globals())
119+
unittest.main(this)

‎postgresql/test/perf_copy_io.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
##
2+
# copyright 2008, pg/python project.
3+
# http://python.projects.postgresql.org
4+
##
5+
# Copy I/O: To and From performance
6+
##
7+
import os, sys, gc, random, time
8+
9+
if __name__ == '__main__':
10+
Words = open('/usr/share/dict/words').readlines()
11+
else:
12+
Words = ['/usr/share/dict/words', 'is', 'read', 'in', '__main__']
13+
wordcount = len(Words)
14+
random.seed()
15+
16+
def getWord():
17+
"extract a random word from ``Words``"
18+
return Words[random.randrange(0, wordcount)].strip()
19+
20+
def testSpeed(tuples = 50000 * 3):
21+
execute("CREATE TEMP TABLE _copy "
22+
"(i int, t text, mt text, ts text, ty text, tx text);")
23+
try:
24+
Q = query("COPY _copy FROM STDIN")
25+
size = [0]
26+
def incsize(data):
27+
size[0] += len(data)
28+
return data
29+
sys.stderr.write("preparing data(%d tuples)...\n" %(tuples,))
30+
31+
# Use an LC to avoid the Python overhead involved with a GE
32+
data = [incsize('\t'.join((
33+
str(x), getWord(), getWord(),
34+
getWord(), getWord(), getWord()
35+
)))+'\n' for x in xrange(tuples)]
36+
37+
sys.stderr.write("starting copy...\n")
38+
start = time.time()
39+
copied_in = Q(data)
40+
duration = time.time() - start
41+
sys.stderr.write(
42+
"COPY FROM STDIN Summary,\n " \
43+
"copied tuples: %d\n " \
44+
"copied bytes: %d\n " \
45+
"duration: %f\n " \
46+
"average tuple size(bytes): %f\n " \
47+
"average KB per second: %f\n " \
48+
"average tuples per second: %f\n" %(
49+
tuples, size[0], duration,
50+
size[0] / tuples,
51+
size[0] / 1024 / duration,
52+
tuples / duration,
53+
)
54+
)
55+
Q = query("COPY _copy TO STDOUT")
56+
start = time.time()
57+
c = 0
58+
for x in Q():
59+
c += 1
60+
duration = time.time() - start
61+
sys.stderr.write(
62+
"COPY TO STDOUT Summary,\n " \
63+
"copied tuples: %d\n " \
64+
"duration: %f\n " \
65+
"average KB per second: %f\n " \
66+
"average tuples per second: %f\n " %(
67+
c, duration,
68+
size[0] / 1024 / duration,
69+
tuples / duration,
70+
)
71+
)
72+
finally:
73+
execute("DROP TABLE _copy")
74+
75+
if __name__ == '__main__':
76+
testSpeed()

‎postgresql/test/perf_query_io.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python
2+
##
3+
# copyright 2008, pg/python project.
4+
# http://python.projects.postgresql.org
5+
##
6+
# Query I/O: Mass insert and select performance
7+
##
8+
import os
9+
import time
10+
import sys
11+
12+
def insertSamples(count, insert_records):
13+
recs = [
14+
(-3, 123, 0xfffffea023, u'some_óäæ_thing', 'varying', u'æ')
15+
for x in xrange(count)
16+
]
17+
18+
gen = time.time()
19+
insert_records << recs
20+
fin = time.time()
21+
xacttime = fin - gen
22+
ats = count / xacttime
23+
print >>sys.stderr, \
24+
"INSERT Summary,\n " \
25+
"inserted tuples: %d\n " \
26+
"total time: %f\n " \
27+
"average tuples per second: %f\n " %(
28+
count, xacttime, ats,
29+
)
30+
31+
def timeTupleRead(portal):
32+
loops = 0
33+
genesis = time.time()
34+
for x in portal:
35+
loops += 1
36+
finalis = time.time()
37+
looptime = finalis - genesis
38+
ats = loops / looptime
39+
print >>sys.stderr, \
40+
"SELECT Summary,\n " \
41+
"looped/tuples: %d\n " \
42+
"looptime: %f\n " \
43+
"average tuples per second: %f\n " %(loops, looptime, ats,)
44+
45+
def main(count):
46+
execute('CREATE TEMP TABLE samples '
47+
'(i2 int2, i4 int4, i8 int8, t text, v varchar, c char)')
48+
insert_records = query(
49+
"INSERT INTO samples VALUES ($1, $2, $3, $4, $5, $6)"
50+
)
51+
select_records = query("SELECT * FROM samples")
52+
try:
53+
insertSamples(count, insert_records)
54+
timeTupleRead(select_records())
55+
finally:
56+
execute("DROP TABLE samples")
57+
58+
def command(args):
59+
main(int((args + [15000])[1]))
60+
61+
if __name__ == '__main__':
62+
command(sys.argv)

0 commit comments

Comments
 (0)