Mercurial > p > roundup > code
view test/benchmark.py @ 7668:5b41018617f2
fix: out of memory error when importing under postgresql
If you try importing more than 20k items under postgresql you can run
out of memory:
psycopg2.errors.OutOfMemory: out of shared memory
HINT: You might need to increase max_locks_per_transaction.
Tuning memory may help, it's unknown at this point.
This checkin forces a commit to the postgres database after 10,000
rows have been added. This clears out the savepoints for each row and
starts a new transaction.
back_postgresql.py:
Implement commit mechanism in checkpoint_data(). Add two class level
attributes for tracking the number of savepoints and the limit when
the commit should happen.
roundup_admin.py:
implement pragma and dynamically create the config item
RDBMS_SAVEPOINT_LIMIT used by checkpoint_data.
Also fixed formatting of descriptions when using pragma list in
verbose mode.
admin_guide.txt, upgrading.txt:
Document change and use of pragma savepoint_limit in roundup-admin
for changing the default of 10,000.
test/db_test_base.py:
add some more asserts. In existing testAdminImportExport, set the
savepoint limit to 5 to test setting method and so that the commit
code will be run by existing tests. This provides coverage, but
does not actually test that the commit is done every 5 savepoints
8-(. The verification of every 5 savepoints was done manually
using a pdb breakpoint just before the commit.
acknowledgements.txt:
Added 2.4.0 section mentioning Norbert as he has done a ton of
testing with much larger datasets than I can test with.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Thu, 19 Oct 2023 16:11:25 -0400 |
| parents | 2120f77554d5 |
| children | 30fcdf60da44 |
line wrap: on
line source
from __future__ import print_function import sys, os, time from roundup.hyperdb import String, Password, Link, Multilink, Date, \ Interval, DatabaseError, Boolean, Number from roundup import date, password from .db_test_base import config def setupSchema(db, module): status = module.Class(db, "status", name=String()) status.setkey("name") user = module.Class(db, "user", username=String(), password=Password(), assignable=Boolean(), age=Number(), roles=String()) user.setkey("username") file = module.FileClass(db, "file", name=String(), type=String(), comment=String(indexme="yes")) issue = module.IssueClass(db, "issue", title=String(indexme="yes"), status=Link("status"), nosy=Multilink("user"), deadline=Date(), foo=Interval(), files=Multilink("file"), assignedto=Link('user')) session = module.Class(db, 'session', title=String()) session.disableJournalling() db.post_init() db.commit() def main(backendname, time=time.time, numissues=10): try: exec('from roundup.backends import %s as backend'%backendname) except ImportError: return times = [] config.DATABASE = os.path.join('_benchmark', '%s-%s'%(backendname, numissues)) if not os.path.exists(config.DATABASE): db = backend.Database(config, 'admin') setupSchema(db, backend) # create a whole bunch of stuff db.user.create(**{'username': 'admin'}) db.status.create(name="unread") db.status.create(name="in-progress") db.status.create(name="testing") db.status.create(name="resolved") pc = -1 for i in range(numissues): db.user.create(**{'username': 'user %s'%i}) for j in range(10): db.user.set(str(i+1), assignable=1) db.user.set(str(i+1), assignable=0) db.issue.create(**{'title': 'issue %s'%i}) for j in range(10): db.issue.set(str(i+1), status='2', assignedto='2', nosy=[]) db.issue.set(str(i+1), status='1', assignedto='1', nosy=['1','2']) if (i*100//numissues) != pc: pc = (i*100//numissues) sys.stdout.write("%d%%\r"%pc) sys.stdout.flush() db.commit() else: db = backend.Database(config, 'admin') setupSchema(db, backend) sys.stdout.write('%7s: %-6d'%(backendname, numissues)) sys.stdout.flush() times.append(('start', time())) # fetch db.clearCache() for i in db.issue.list(): db.issue.get(i, 'title') times.append(('fetch', time())) # journals db.clearCache() for i in db.issue.list(): db.issue.history(i) times.append(('journal', time())) # "calculated" props db.clearCache() for i in db.issue.list(): db.issue.get(i, 'activity') db.issue.get(i, 'creator') db.issue.get(i, 'creation') times.append(('jprops', time())) # lookup db.clearCache() for i in range(numissues): db.user.lookup('user %s'%i) times.append(('lookup', time())) # filter db.clearCache() for i in range(100): db.issue.filter(None, {'assignedto': '1', 'title':'issue'}, ('+', 'activity'), ('+', 'status')) times.append(('filter', time())) # filter with multilink db.clearCache() for i in range(100): db.issue.filter(None, {'nosy': ['1'], 'assignedto': '1', 'title':'issue'}, ('+', 'activity'), ('+', 'status')) times.append(('filtml', time())) # results last = None for event, stamp in times: if last is None: first = stamp else: sys.stdout.write(' %-6.2f'%(stamp-last)) last = stamp print(' %-6.2f'%(last-first)) sys.stdout.flush() if __name__ == '__main__': # 0 1 2 3 4 5 6 # 01234567890123456789012345678901234567890123456789012345678901234 print('Test name fetch journl jprops lookup filter filtml TOTAL ') for name in 'anydbm metakit sqlite'.split(): main(name) for name in 'anydbm metakit sqlite'.split(): main(name, numissues=20) for name in 'anydbm metakit sqlite'.split(): main(name, numissues=100) # don't even bother benchmarking the dbm backends > 100! for name in 'metakit sqlite'.split(): main(name, numissues=1000) # vim: set et sts=4 sw=4 :
