Mercurial > p > roundup > code

comparison roundup/backends/rdbms_common.py @ 1174:8e318dfaf479

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Verify contents of tracker module when the tracker is opened Performance improvements in *dbm and sq backends New benchmark module. To use: PYTHONPATH=. python2 test/benchmark.py (yes, it's a little basic at present ;)
author Richard Jones <richard@users.sourceforge.net>
date Fri, 20 Sep 2002 01:20:32 +0000
parents 58f1a2c174ed
children 762f48bfbc0b
comparison
equal deleted inserted replaced
1173:58f1a2c174ed 1174:8e318dfaf479
1 # $Id: rdbms_common.py,v 1.6 2002-09-19 05:30:25 richard Exp $ 1 # $Id: rdbms_common.py,v 1.7 2002-09-20 01:20:32 richard Exp $
2 2
3 # standard python modules 3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy 4 import sys, os, time, re, errno, weakref, copy
5 5
6 # roundup modules 6 # roundup modules
21 21
22 - some functionality is specific to the actual SQL database, hence 22 - some functionality is specific to the actual SQL database, hence
23 the sql_* methods that are NotImplemented 23 the sql_* methods that are NotImplemented
24 - we keep a cache of the latest ROW_CACHE_SIZE row fetches. 24 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
25 ''' 25 '''
26 # flag to set on retired entries
27 RETIRED_FLAG = '__hyperdb_retired'
28
29 def __init__(self, config, journaltag=None): 26 def __init__(self, config, journaltag=None):
30 ''' Open the database and load the schema from it. 27 ''' Open the database and load the schema from it.
31 ''' 28 '''
32 self.config, self.journaltag = config, journaltag 29 self.config, self.journaltag = config, journaltag
33 self.dir = config.DATABASE 30 self.dir = config.DATABASE
128 ''' Figure the column names and multilink properties from the spec 125 ''' Figure the column names and multilink properties from the spec
129 126
130 "properties" is a list of (name, prop) where prop may be an 127 "properties" is a list of (name, prop) where prop may be an
131 instance of a hyperdb "type" _or_ a string repr of that type. 128 instance of a hyperdb "type" _or_ a string repr of that type.
132 ''' 129 '''
133 cols = [] 130 cols = ['_activity', '_creator', '_creation']
134 mls = [] 131 mls = []
135 # add the multilinks separately 132 # add the multilinks separately
136 for col, prop in properties: 133 for col, prop in properties:
137 if isinstance(prop, Multilink): 134 if isinstance(prop, Multilink):
138 mls.append(col) 135 mls.append(col)
459 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) 456 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
460 # gadfly requires values for all non-multilink columns 457 # gadfly requires values for all non-multilink columns
461 cl = self.classes[classname] 458 cl = self.classes[classname]
462 cols, mls = self.determine_columns(cl.properties.items()) 459 cols, mls = self.determine_columns(cl.properties.items())
463 460
461 # add the special props
462 node = node.copy()
463 node['creation'] = node['activity'] = date.Date()
464 node['creator'] = self.journaltag
465
464 # default the non-multilink columns 466 # default the non-multilink columns
465 for col, prop in cl.properties.items(): 467 for col, prop in cl.properties.items():
466 if not isinstance(col, Multilink): 468 if not isinstance(col, Multilink):
467 if not node.has_key(col): 469 if not node.has_key(col):
468 node[col] = None 470 node[col] = None
497 self.sql(cursor, sql, (entry, nodeid)) 499 self.sql(cursor, sql, (entry, nodeid))
498 500
499 # make sure we do the commit-time extra stuff for this node 501 # make sure we do the commit-time extra stuff for this node
500 self.transactions.append((self.doSaveNode, (classname, nodeid, node))) 502 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
501 503
502 def setnode(self, classname, nodeid, node, multilink_changes): 504 def setnode(self, classname, nodeid, values, multilink_changes):
503 ''' Change the specified node. 505 ''' Change the specified node.
504 ''' 506 '''
505 if __debug__: 507 if __debug__:
506 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) 508 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
507 509
508 # clear this node out of the cache if it's in there 510 # clear this node out of the cache if it's in there
509 key = (classname, nodeid) 511 key = (classname, nodeid)
510 if self.cache.has_key(key): 512 if self.cache.has_key(key):
511 del self.cache[key] 513 del self.cache[key]
512 self.cache_lru.remove(key) 514 self.cache_lru.remove(key)
513 515
514 node = self.serialise(classname, node) 516 # add the special props
517 values = values.copy()
518 values['activity'] = date.Date()
519
520 # make db-friendly
521 values = self.serialise(classname, values)
515 522
516 cl = self.classes[classname] 523 cl = self.classes[classname]
517 cols = [] 524 cols = []
518 mls = [] 525 mls = []
519 # add the multilinks separately 526 # add the multilinks separately
520 for col in node.keys(): 527 props = cl.getprops()
521 prop = cl.properties[col] 528 for col in values.keys():
529 prop = props[col]
522 if isinstance(prop, Multilink): 530 if isinstance(prop, Multilink):
523 mls.append(col) 531 mls.append(col)
524 else: 532 else:
525 cols.append('_'+col) 533 cols.append('_'+col)
526 cols.sort() 534 cols.sort()
527 535
528 # make sure the ordering is correct for column name -> column value
529 vals = tuple([node[col[1:]] for col in cols])
530 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
531 cols = ','.join(cols)
532
533 # perform the update
534 cursor = self.conn.cursor() 536 cursor = self.conn.cursor()
535 sql = 'update _%s set %s'%(classname, s) 537
536 if __debug__: 538 # if there's any updates to regular columns, do them
537 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals) 539 if cols:
538 cursor.execute(sql, vals) 540 # make sure the ordering is correct for column name -> column value
541 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
542 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
543 cols = ','.join(cols)
544
545 # perform the update
546 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
547 if __debug__:
548 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
549 cursor.execute(sql, sqlvals)
539 550
540 # now the fun bit, updating the multilinks ;) 551 # now the fun bit, updating the multilinks ;)
541 for col, (add, remove) in multilink_changes.items(): 552 for col, (add, remove) in multilink_changes.items():
542 tn = '%s_%s'%(classname, col) 553 tn = '%s_%s'%(classname, col)
543 if add: 554 if add:
550 self.arg, self.arg) 561 self.arg, self.arg)
551 for removeid in remove: 562 for removeid in remove:
552 self.sql(cursor, sql, (nodeid, removeid)) 563 self.sql(cursor, sql, (nodeid, removeid))
553 564
554 # make sure we do the commit-time extra stuff for this node 565 # make sure we do the commit-time extra stuff for this node
555 self.transactions.append((self.doSaveNode, (classname, nodeid, node))) 566 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
556 567
557 def getnode(self, classname, nodeid): 568 def getnode(self, classname, nodeid):
558 ''' Get a node from the database. 569 ''' Get a node from the database.
559 ''' 570 '''
560 if __debug__: 571 if __debug__:
601 node = self.unserialise(classname, node) 612 node = self.unserialise(classname, node)
602 613
603 # save off in the cache 614 # save off in the cache
604 key = (classname, nodeid) 615 key = (classname, nodeid)
605 self.cache[key] = node 616 self.cache[key] = node
606 # update the LRU 617 # update the LRU
607 self.cache_lru.insert(0, key) 618 self.cache_lru.insert(0, key)
608 del self.cache[self.cache_lru.pop()] 619 del self.cache[self.cache_lru.pop()]
609 620
610 return node 621 return node
611 622
612 def destroynode(self, classname, nodeid): 623 def destroynode(self, classname, nodeid):
613 '''Remove a node from the database. Called exclusively by the 624 '''Remove a node from the database. Called exclusively by the
1169 set cache=0. 1180 set cache=0.
1170 ''' 1181 '''
1171 if propname == 'id': 1182 if propname == 'id':
1172 return nodeid 1183 return nodeid
1173 1184
1185 # get the node's dict
1186 d = self.db.getnode(self.classname, nodeid)
1187
1174 if propname == 'creation': 1188 if propname == 'creation':
1175 if not self.do_journal: 1189 if d.has_key('creation'):
1176 raise ValueError, 'Journalling is disabled for this class' 1190 return d['creation']
1177 journal = self.db.getjournal(self.classname, nodeid)
1178 if journal:
1179 return self.db.getjournal(self.classname, nodeid)[0][1]
1180 else: 1191 else:
1181 # on the strange chance that there's no journal
1182 return date.Date() 1192 return date.Date()
1183 if propname == 'activity': 1193 if propname == 'activity':
1184 if not self.do_journal: 1194 if d.has_key('activity'):
1185 raise ValueError, 'Journalling is disabled for this class' 1195 return d['activity']
1186 journal = self.db.getjournal(self.classname, nodeid)
1187 if journal:
1188 return self.db.getjournal(self.classname, nodeid)[-1][1]
1189 else: 1196 else:
1190 # on the strange chance that there's no journal
1191 return date.Date() 1197 return date.Date()
1192 if propname == 'creator': 1198 if propname == 'creator':
1193 if not self.do_journal: 1199 if d.has_key('creator'):
1194 raise ValueError, 'Journalling is disabled for this class' 1200 return d['creator']
1195 journal = self.db.getjournal(self.classname, nodeid)
1196 if journal:
1197 name = self.db.getjournal(self.classname, nodeid)[0][2]
1198 else: 1201 else:
1199 return None 1202 return self.db.journaltag
1200 try:
1201 return self.db.user.lookup(name)
1202 except KeyError:
1203 # the journaltag user doesn't exist any more
1204 return None
1205 1203
1206 # get the property (raises KeyErorr if invalid) 1204 # get the property (raises KeyErorr if invalid)
1207 prop = self.properties[propname] 1205 prop = self.properties[propname]
1208
1209 # get the node's dict
1210 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1211 1206
1212 if not d.has_key(propname): 1207 if not d.has_key(propname):
1213 if default is self._marker: 1208 if default is self._marker:
1214 if isinstance(prop, Multilink): 1209 if isinstance(prop, Multilink):
1215 return [] 1210 return []
1296 raise ValueError, 'node with key "%s" exists'%value 1291 raise ValueError, 'node with key "%s" exists'%value
1297 1292
1298 # this will raise the KeyError if the property isn't valid 1293 # this will raise the KeyError if the property isn't valid
1299 # ... we don't use getprops() here because we only care about 1294 # ... we don't use getprops() here because we only care about
1300 # the writeable properties. 1295 # the writeable properties.
1301 prop = self.properties[propname] 1296 try:
1297 prop = self.properties[propname]
1298 except KeyError:
1299 raise KeyError, '"%s" has no property named "%s"'%(
1300 self.classname, propname)
1302 1301
1303 # if the value's the same as the existing value, no sense in 1302 # if the value's the same as the existing value, no sense in
1304 # doing anything 1303 # doing anything
1305 if node.has_key(propname) and value == node[propname]: 1304 if node.has_key(propname) and value == node[propname]:
1306 del propvalues[propname] 1305 del propvalues[propname]
1429 try: 1428 try:
1430 int(value) 1429 int(value)
1431 except ValueError: 1430 except ValueError:
1432 raise TypeError, 'new property "%s" not boolean'%propname 1431 raise TypeError, 'new property "%s" not boolean'%propname
1433 1432
1434 node[propname] = value
1435
1436 # nothing to do? 1433 # nothing to do?
1437 if not propvalues: 1434 if not propvalues:
1438 return propvalues 1435 return propvalues
1439 1436
1440 # do the set, and journal it 1437 # do the set, and journal it
1441 self.db.setnode(self.classname, nodeid, node, multilink_changes) 1438 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1442 1439
1443 if self.do_journal: 1440 if self.do_journal:
1444 propvalues.update(journalvalues) 1441 propvalues.update(journalvalues)
1445 self.db.addjournal(self.classname, nodeid, 'set', propvalues) 1442 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1446 1443
1573 ''' 1570 '''
1574 if not self.key: 1571 if not self.key:
1575 raise TypeError, 'No key property set for class %s'%self.classname 1572 raise TypeError, 'No key property set for class %s'%self.classname
1576 1573
1577 cursor = self.db.conn.cursor() 1574 cursor = self.db.conn.cursor()
1578 sql = 'select id from _%s where _%s=%s'%(self.classname, self.key, 1575 sql = 'select id,__retired__ from _%s where _%s=%s'%(self.classname,
1579 self.db.arg) 1576 self.key, self.db.arg)
1580 if __debug__: 1577 self.db.sql(cursor, sql, (keyvalue,))
1581 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue) 1578
1582 cursor.execute(sql, (keyvalue,)) 1579 # see if there was a result that's not retired
1583
1584 # see if there was a result
1585 l = cursor.fetchall() 1580 l = cursor.fetchall()
1586 if not l: 1581 if not l or int(l[0][1]):
1587 raise KeyError, keyvalue 1582 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1583 keyvalue, self.classname)
1588 1584
1589 # return the id 1585 # return the id
1590 return l[0][0] 1586 return l[0][0]
1591 1587
1592 def find(self, **propspec): 1588 def find(self, **propspec):
1663 # now do other where clause stuff 1659 # now do other where clause stuff
1664 if isinstance(propclass, Multilink): 1660 if isinstance(propclass, Multilink):
1665 tn = '%s_%s'%(cn, k) 1661 tn = '%s_%s'%(cn, k)
1666 frum.append(tn) 1662 frum.append(tn)
1667 if isinstance(v, type([])): 1663 if isinstance(v, type([])):
1668 s = ','.join([self.arg for x in v]) 1664 s = ','.join([a for x in v])
1669 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s)) 1665 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1670 args = args + v 1666 args = args + v
1671 else: 1667 else:
1672 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a)) 1668 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1673 args.append(v) 1669 args.append(v)
1731 # now add in the sorting 1727 # now add in the sorting
1732 group = '' 1728 group = ''
1733 if sort[0] is not None and sort[1] is not None: 1729 if sort[0] is not None and sort[1] is not None:
1734 direction, colname = sort 1730 direction, colname = sort
1735 if direction != '-': 1731 if direction != '-':
1736 if colname == 'activity': 1732 if colname == 'id':
1737 orderby.append('activity')
1738 ordercols.append('max(%s__journal.date) as activity'%cn)
1739 frum.append('%s__journal'%cn)
1740 where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
1741 # we need to group by id
1742 group = ' group by id'
1743 elif colname == 'id':
1744 orderby.append(colname) 1733 orderby.append(colname)
1745 else: 1734 else:
1746 orderby.append('_'+colname) 1735 orderby.append('_'+colname)
1747 ordercols.append('_'+colname) 1736 ordercols.append('_'+colname)
1748 else: 1737 else:
1749 if colname == 'activity': 1738 if colname == 'id':
1750 orderby.append('activity desc')
1751 ordercols.append('max(%s__journal.date) as activity'%cn)
1752 frum.append('%s__journal'%cn)
1753 where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
1754 # we need to group by id
1755 group = ' group by id'
1756 elif colname == 'id':
1757 orderby.append(colname+' desc') 1739 orderby.append(colname+' desc')
1758 ordercols.append(colname) 1740 ordercols.append(colname)
1759 else: 1741 else:
1760 orderby.append('_'+colname+' desc') 1742 orderby.append('_'+colname+' desc')
1761 ordercols.append('_'+colname) 1743 ordercols.append('_'+colname)
Roundup Issue Tracker: http://roundup-tracker.org/