code-c3e391d9c4e9/roundup/backends/.cvsignore0000644000000000000000000000002407535063756017227 0ustar 00000000000000*.pyc *.pyo *.cover code-c3e391d9c4e9/roundup/backends/__init__.py0000644000000000000000000001114207535063756017343 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # # $Id: __init__.py,v 1.15 2002-08-23 04:48:10 richard Exp $ __all__ = [] try: import sys, anydbm if not hasattr(sys, 'version_info') or sys.version_info < (2,1,2): import dumbdbm # dumbdbm only works in python 2.1.2+ assert anydbm._defaultmod != dumbdbm del anydbm del dumbdbm except AssertionError: print "WARNING: you should upgrade to python 2.1.3" except ImportError, message: if str(message) != 'No module named anydbm': raise else: import back_anydbm anydbm = back_anydbm __all__.append('anydbm') try: import gadfly except ImportError, message: if str(message) != 'No module named gadfly': raise else: import back_gadfly gadfly = back_gadfly __all__.append('gadfly') try: import bsddb except ImportError, message: if str(message) != 'No module named bsddb': raise else: import back_bsddb bsddb = back_bsddb __all__.append('bsddb') try: import bsddb3 except ImportError, message: if str(message) != 'No module named bsddb3': raise else: import back_bsddb3 bsddb3 = back_bsddb3 __all__.append('bsddb3') try: import metakit except ImportError, message: if str(message) != 'No module named metakit': raise else: import back_metakit metakit = back_metakit __all__.append('metakit') # # $Log: not supported by cvs2svn $ # Revision 1.14 2002/08/22 07:56:51 richard # Whee! It's not finished yet, but I can create a new instance and play with # it a little bit :) # # Revision 1.13 2002/07/11 01:11:03 richard # Added metakit backend to the db tests and fixed the more easily fixable test # failures. # # Revision 1.12 2002/05/22 00:32:33 richard # . changed the default message list in issues to display the message body # . made backends.__init__ be more specific about which ImportErrors it really # wants to ignore # . fixed the example addresses in the templates to use correct example domains # . cleaned out the template stylesheets, removing a bunch of junk that really # wasn't necessary (font specs, styles never used) and added a style for # message content # # Revision 1.11 2002/02/16 08:39:42 richard # . #516854 ] "My Issues" and redisplay # # Revision 1.10 2002/01/22 07:08:50 richard # I was certain I'd already done this (there's even a change note in # CHANGES)... # # Revision 1.9 2001/12/12 02:30:51 richard # I fixed the problems with people whose anydbm was using the dbm module at the # backend. It turns out the dbm module modifies the file name to append ".db" # and my check to determine if we're opening an existing or new db just # tested os.path.exists() on the filename. Well, no longer! We now perform a # much better check _and_ cope with the anydbm implementation module changing # too! # I also fixed the backends __init__ so only ImportError is squashed. # # Revision 1.8 2001/12/10 22:20:01 richard # Enabled transaction support in the bsddb backend. It uses the anydbm code # where possible, only replacing methods where the db is opened (it uses the # btree opener specifically.) # Also cleaned up some change note generation. # Made the backends package work with pydoc too. # # Revision 1.7 2001/12/10 00:57:38 richard # From CHANGES: # . Added the "display" command to the admin tool - displays a node's values # . #489760 ] [issue] only subject # . fixed the doc/index.html to include the quoting in the mail alias. # # Also: # . fixed roundup-admin so it works with transactions # . disabled the back_anydbm module if anydbm tries to use dumbdbm # # Revision 1.6 2001/08/07 00:24:42 richard # stupid typo # # Revision 1.5 2001/08/07 00:15:51 richard # Added the copyright/license notice to (nearly) all files at request of # Bizar Software. # # # # vim: set filetype=python ts=4 sw=4 et si code-c3e391d9c4e9/roundup/backends/back_anydbm.py0000644000000000000000000025441407535063756020051 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_anydbm.py,v 1.69 2002-09-03 07:33:01 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several serious bugs, and is not available) ''' import whichdb, anydbm, os, marshal, re, weakref, string, copy from roundup import hyperdb, date, password, roundupdb, security from blobfiles import FileStorage from sessions import Sessions from roundup.indexer import Indexer from locking import acquire_lock, release_lock from roundup.hyperdb import String, Password, Date, Interval, Link, \ Multilink, DatabaseError, Boolean, Number # # Now the database # class Database(FileStorage, hyperdb.Database, roundupdb.Database): '''A database for storing records containing flexible data types. Transaction stuff TODO: . check the timestamp of the class file and nuke the cache if it's modified. Do some sort of conflict checking on the dirty stuff. . perhaps detect write collisions (related to above)? ''' def __init__(self, config, journaltag=None): '''Open a hyperdatabase given a specifier to some storage. The 'storagelocator' is obtained from config.DATABASE. The meaning of 'storagelocator' depends on the particular implementation of the hyperdatabase. It could be a file name, a directory path, a socket descriptor for a connection to a database over the network, etc. The 'journaltag' is a token that will be attached to the journal entries for any edits done on the database. If 'journaltag' is None, the database is opened in read-only mode: the Class.create(), Class.set(), and Class.retire() methods are disabled. ''' self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.cache = {} # cache of nodes loaded or created self.dirtynodes = {} # keep track of the dirty nodes by class self.newnodes = {} # keep track of the new nodes by class self.destroyednodes = {}# keep track of the destroyed nodes by class self.transactions = [] self.indexer = Indexer(self.dir) self.sessions = Sessions(self.config) self.security = security.Security(self) # ensure files are group readable and writable os.umask(0002) def post_init(self): '''Called once the schema initialisation has finished.''' # reindex the db if necessary if self.indexer.should_reindex(): self.reindex() def reindex(self): for klass in self.classes.values(): for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() def __repr__(self): return ''%id(self) # # Classes # def __getattr__(self, classname): '''A convenient way of calling self.getclass(classname).''' if self.classes.has_key(classname): if __debug__: print >>hyperdb.DEBUG, '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): if __debug__: print >>hyperdb.DEBUG, 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl def getclasses(self): '''Return a list of the names of all existing classes.''' if __debug__: print >>hyperdb.DEBUG, 'getclasses', (self,) l = self.classes.keys() l.sort() return l def getclass(self, classname): '''Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. ''' if __debug__: print >>hyperdb.DEBUG, 'getclass', (self, classname) return self.classes[classname] # # Class DBs # def clear(self): '''Delete all database contents ''' if __debug__: print >>hyperdb.DEBUG, 'clear', (self,) for cn in self.classes.keys(): for dummy in 'nodes', 'journals': path = os.path.join(self.dir, 'journals.%s'%cn) if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' if __debug__: print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode) return self.opendb('nodes.%s'%classname, mode) def determine_db_type(self, path): ''' determine which DB wrote the class file ''' db_type = '' if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! db_type = 'dbm' return db_type def opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if __debug__: print >>hyperdb.DEBUG, 'opendb', (self, name, mode) # figure the class db type path = os.path.join(os.getcwd(), self.dir, name) db_type = self.determine_db_type(path) # new database? let anydbm pick the best dbm if not db_type: if __debug__: print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path return anydbm.open(path, 'c') # open the database with the correct module try: dbm = __import__(db_type) except ImportError: raise hyperdb.DatabaseError, \ "Couldn't open database - the required module '%s'"\ " is not available"%db_type if __debug__: print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path, mode) return dbm.open(path, mode) def lockdb(self, name): ''' Lock a database file ''' path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name) return acquire_lock(path) # # Node IDs # def newid(self, classname): ''' Generate a new id for the given class ''' # open the ids DB - create if if doesn't exist lock = self.lockdb('_ids') db = self.opendb('_ids', 'c') if db.has_key(classname): newid = db[classname] = str(int(db[classname]) + 1) else: # the count() bit is transitional - older dbs won't start at 1 newid = str(self.getclass(classname).count()+1) db[classname] = newid db.close() release_lock(lock) return newid def setid(self, classname, setid): ''' Set the id counter: used during import of database ''' # open the ids DB - create if if doesn't exist lock = self.lockdb('_ids') db = self.opendb('_ids', 'c') db[classname] = str(setid) db.close() release_lock(lock) # # Nodes # def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node self.savenode(classname, nodeid, node) def setnode(self, classname, nodeid, node): ''' change the specified node ''' if __debug__: print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node self.cache[classname][nodeid] = node self.savenode(classname, nodeid, node) def savenode(self, classname, nodeid, node): ''' perform the saving of data specified by the set/addnode ''' if __debug__: print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node) self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database ''' if __debug__: print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db) if cache: # try the cache cache_dict = self.cache.setdefault(classname, {}) if cache_dict.has_key(nodeid): if __debug__: print >>hyperdb.TRACE, 'get %s %s cached'%(classname, nodeid) return cache_dict[nodeid] if __debug__: print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid) # get from the database and save in the cache if db is None: db = self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, "no such %s %s"%(classname, nodeid) # check the uncommitted, destroyed nodes if (self.destroyednodes.has_key(classname) and self.destroyednodes[classname].has_key(nodeid)): raise IndexError, "no such %s %s"%(classname, nodeid) # decode res = marshal.loads(db[nodeid]) # reverse the serialisation res = self.unserialise(classname, res) # store off in the cache dict if cache: cache_dict[nodeid] = res return res def destroynode(self, classname, nodeid): '''Remove a node from the database. Called exclusively by the destroy() method on Class. ''' if __debug__: print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid) # remove from cache and newnodes if it's there if (self.cache.has_key(classname) and self.cache[classname].has_key(nodeid)): del self.cache[classname][nodeid] if (self.newnodes.has_key(classname) and self.newnodes[classname].has_key(nodeid)): del self.newnodes[classname][nodeid] # see if there's any obvious commit actions that we should get rid of for entry in self.transactions[:]: if entry[1][:2] == (classname, nodeid): self.transactions.remove(entry) # add to the destroyednodes map self.destroyednodes.setdefault(classname, {})[nodeid] = 1 # add the destroy commit action self.transactions.append((self.doDestroyNode, (classname, nodeid))) def serialise(self, classname, node): '''Copy the node contents, converting non-marshallable data into marshallable data. ''' if __debug__: print >>hyperdb.DEBUG, 'serialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): # if the property doesn't exist, or is the "retired" flag then # it won't be in the properties dict if not properties.has_key(k): d[k] = v continue # get the property spec prop = properties[k] if isinstance(prop, Password): d[k] = str(v) elif isinstance(prop, Date) and v is not None: d[k] = v.serialise() elif isinstance(prop, Interval) and v is not None: d[k] = v.serialise() else: d[k] = v return d def unserialise(self, classname, node): '''Decode the marshalled node data ''' if __debug__: print >>hyperdb.DEBUG, 'unserialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): # if the property doesn't exist, or is the "retired" flag then # it won't be in the properties dict if not properties.has_key(k): d[k] = v continue # get the property spec prop = properties[k] if isinstance(prop, Date) and v is not None: d[k] = date.Date(v) elif isinstance(prop, Interval) and v is not None: d[k] = date.Interval(v) elif isinstance(prop, Password): p = password.Password() p.unpack(v) d[k] = p else: d[k] = v return d def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' if __debug__: print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db) # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): if __debug__: print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid) return 1 if __debug__: print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid) # not in the cache - check the database if db is None: db = self.getclassdb(classname) res = db.has_key(nodeid) return res def countnodes(self, classname, db=None): if __debug__: print >>hyperdb.DEBUG, 'countnodes', (self, classname, db) count = 0 # include the uncommitted nodes if self.newnodes.has_key(classname): count += len(self.newnodes[classname]) if self.destroyednodes.has_key(classname): count -= len(self.destroyednodes[classname]) # and count those in the DB if db is None: db = self.getclassdb(classname) count = count + len(db.keys()) return count def getnodeids(self, classname, db=None): if __debug__: print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db) res = [] # start off with the new nodes if self.newnodes.has_key(classname): res += self.newnodes[classname].keys() if db is None: db = self.getclassdb(classname) res = res + db.keys() # remove the uncommitted, destroyed nodes if self.destroyednodes.has_key(classname): for nodeid in self.destroyednodes[classname].keys(): if db.has_key(nodeid): res.remove(nodeid) return res # # Files - special node properties # inherited from FileStorage # # Journal # def addjournal(self, classname, nodeid, action, params): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' if __debug__: print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid, action, params) self.transactions.append((self.doSaveJournal, (classname, nodeid, action, params))) def getjournal(self, classname, nodeid): ''' get the journal for id Raise IndexError if the node doesn't exist (as per history()'s API) ''' if __debug__: print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid) # attempt to open the journal - in some rare cases, the journal may # not exist try: db = self.opendb('journals.%s'%classname, 'r') except anydbm.error, error: if str(error) == "need 'c' or 'n' flag to open new db": raise IndexError, 'no such %s %s'%(classname, nodeid) elif error.args[0] != 2: raise raise IndexError, 'no such %s %s'%(classname, nodeid) try: journal = marshal.loads(db[nodeid]) except KeyError: db.close() raise IndexError, 'no such %s %s'%(classname, nodeid) db.close() res = [] for nodeid, date_stamp, user, action, params in journal: res.append((nodeid, date.Date(date_stamp), user, action, params)) return res def pack(self, pack_before): ''' Delete all journal entries except "create" before 'pack_before'. ''' if __debug__: print >>hyperdb.DEBUG, 'packjournal', (self, pack_before) for classname in self.getclasses(): # get the journal db db_name = 'journals.%s'%classname path = os.path.join(os.getcwd(), self.dir, classname) db_type = self.determine_db_type(path) db = self.opendb(db_name, 'w') for key in db.keys(): # get the journal for this db entry journal = marshal.loads(db[key]) l = [] last_set_entry = None for entry in journal: # unpack the entry (nodeid, date_stamp, self.journaltag, action, params) = entry date_stamp = date.Date(date_stamp) # if the entry is after the pack date, _or_ the initial # create entry, then it stays if date_stamp > pack_before or action == 'create': l.append(entry) elif action == 'set': # grab the last set entry to keep information on # activity last_set_entry = entry if last_set_entry: date_stamp = last_set_entry[1] # if the last set entry was made after the pack date # then it is already in the list if date_stamp < pack_before: l.append(last_set_entry) db[key] = marshal.dumps(l) if db_type == 'gdbm': db.reorganize() db.close() # # Basic transaction support # def commit(self): ''' Commit the current transactions. ''' if __debug__: print >>hyperdb.DEBUG, 'commit', (self,) # TODO: lock the DB # keep a handle to all the database files opened self.databases = {} # now, do all the transactions reindex = {} for method, args in self.transactions: reindex[method(*args)] = 1 # now close all the database files for db in self.databases.values(): db.close() del self.databases # TODO: unlock the DB # reindex the nodes that request it for classname, nodeid in filter(None, reindex.keys()): print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid) self.getclass(classname).index(nodeid) # save the indexer state self.indexer.save_index() # all transactions committed, back to normal self.cache = {} self.dirtynodes = {} self.newnodes = {} self.destroyednodes = {} self.transactions = [] def getCachedClassDB(self, classname): ''' get the class db, looking in our cache of databases for commit ''' # get the database handle db_name = 'nodes.%s'%classname if not self.databases.has_key(db_name): self.databases[db_name] = self.getclassdb(classname, 'c') return self.databases[db_name] def doSaveNode(self, classname, nodeid, node): if __debug__: print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid, node) db = self.getCachedClassDB(classname) # now save the marshalled data db[nodeid] = marshal.dumps(self.serialise(classname, node)) # return the classname, nodeid so we reindex this content return (classname, nodeid) def getCachedJournalDB(self, classname): ''' get the journal db, looking in our cache of databases for commit ''' # get the database handle db_name = 'journals.%s'%classname if not self.databases.has_key(db_name): self.databases[db_name] = self.opendb(db_name, 'c') return self.databases[db_name] def doSaveJournal(self, classname, nodeid, action, params): # handle supply of the special journalling parameters (usually # supplied on importing an existing database) if isinstance(params, type({})): if params.has_key('creator'): journaltag = self.user.get(params['creator'], 'username') del params['creator'] else: journaltag = self.journaltag if params.has_key('created'): journaldate = params['created'].serialise() del params['created'] else: journaldate = date.Date().serialise() if params.has_key('activity'): del params['activity'] # serialise the parameters now if action in ('set', 'create'): params = self.serialise(classname, params) else: journaltag = self.journaltag journaldate = date.Date().serialise() # create the journal entry entry = (nodeid, journaldate, journaltag, action, params) if __debug__: print >>hyperdb.DEBUG, 'doSaveJournal', entry db = self.getCachedJournalDB(classname) # now insert the journal entry if db.has_key(nodeid): # append to existing s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) def doDestroyNode(self, classname, nodeid): if __debug__: print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid) # delete from the class database db = self.getCachedClassDB(classname) if db.has_key(nodeid): del db[nodeid] # delete from the database db = self.getCachedJournalDB(classname) if db.has_key(nodeid): del db[nodeid] # return the classname, nodeid so we reindex this content return (classname, nodeid) def rollback(self): ''' Reverse all actions from the current transaction. ''' if __debug__: print >>hyperdb.DEBUG, 'rollback', (self, ) for method, args in self.transactions: # delete temporary files if method == self.doStoreFile: self.rollbackStoreFile(*args) self.cache = {} self.dirtynodes = {} self.newnodes = {} self.destroyednodes = {} self.transactions = [] _marker = [] class Class(hyperdb.Class): '''The handle to a particular class of nodes in a hyperdatabase.''' def __init__(self, db, classname, **properties): '''Create a new class with a given name and property specification. 'classname' must not collide with the name of an existing class, or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. ''' if (properties.has_key('creation') or properties.has_key('activity') or properties.has_key('creator')): raise ValueError, '"creation", "activity" and "creator" are '\ 'reserved' self.classname = classname self.properties = properties self.db = weakref.proxy(db) # use a weak ref to avoid circularity self.key = '' # should we journal changes (default yes) self.do_journal = 1 # do the db-related init stuff db.addclass(self) self.auditors = {'create': [], 'set': [], 'retire': []} self.reactors = {'create': [], 'set': [], 'retire': []} def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # Editing nodes: def create(self, **propvalues): '''Create a new node of this class and return its id. The keyword arguments in 'propvalues' map property names to values. The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' self.fireAuditors('create', None, propvalues) # new node's id newid = self.db.newid(self.classname) # validate propvalues num_re = re.compile('^\d+$') for key, value in propvalues.items(): if key == self.key: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # try to handle this property try: prop = self.properties[key] except KeyError: raise KeyError, '"%s" has no property "%s"'%(self.classname, key) if value is not None and isinstance(prop, Link): if type(value) != type(''): raise ValueError, 'link value must be String' link_class = self.properties[key].classname # if it isn't a number, it's a key if not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, link_class) elif not self.db.getclass(link_class).hasnode(value): raise IndexError, '%s has no node %s'%(link_class, value) # save off the value propvalues[key] = value # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, value, 'link', (self.classname, newid, key)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of ids'%key # clean up and validate the list of links link_class = self.properties[key].classname l = [] for entry in value: if type(entry) != type(''): raise ValueError, '"%s" link value (%s) must be '\ 'String'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, self.properties[key].classname) l.append(entry) value = l propvalues[key] = value # handle additions for nodeid in value: if not self.db.getclass(link_class).hasnode(nodeid): raise IndexError, '%s has no node %s'%(link_class, nodeid) # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) elif isinstance(prop, String): if type(value) != type(''): raise TypeError, 'new property "%s" not a string'%key elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%key elif isinstance(prop, Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'%key elif isinstance(prop, Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'%key elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%key elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%key # make sure there's data where there needs to be for key, prop in self.properties.items(): if propvalues.has_key(key): continue if key == self.key: raise ValueError, 'key property "%s" is required'%key if isinstance(prop, Multilink): propvalues[key] = [] else: propvalues[key] = None # done self.db.addnode(self.classname, newid, propvalues) if self.do_journal: self.db.addjournal(self.classname, newid, 'create', propvalues) self.fireReactors('create', newid, None) return newid def export_list(self, propnames, nodeid): ''' Export a node - generate a list of CSV-able data in the order specified by propnames for the given node. ''' properties = self.getprops() l = [] for prop in propnames: proptype = properties[prop] value = self.get(nodeid, prop) # "marshal" data where needed if isinstance(proptype, hyperdb.Date): value = value.get_tuple() elif isinstance(proptype, hyperdb.Interval): value = value.get_tuple() elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) return l def import_list(self, propnames, proplist): ''' Import a node - all information including "id" is present and should not be sanity checked. Triggers are not triggered. The journal should be initialised using the "creator" and "created" information. Return the nodeid of the node imported. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' properties = self.getprops() # make the new node's property map d = {} for i in range(len(propnames)): # Use eval to reverse the repr() used to output the CSV value = eval(proplist[i]) # Figure the property for this column propname = propnames[i] prop = properties[propname] # "unmarshal" where necessary if propname == 'id': newid = value continue elif isinstance(prop, hyperdb.Date): value = date.Date(value) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value) elif isinstance(prop, hyperdb.Password): pwd = password.Password() pwd.unpack(value) value = pwd if value is not None: d[propname] = value # add self.db.addnode(self.classname, newid, d) self.db.addjournal(self.classname, newid, 'create', d) return newid def get(self, nodeid, propname, default=_marker, cache=1): '''Get the value of a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. 'cache' indicates whether the transaction cache should be queried for the node. If the node has been modified and you need to determine what its values prior to modification are, you need to set cache=0. Attempts to get the "creation" or "activity" properties should do the right thing. ''' if propname == 'id': return nodeid if propname == 'creation': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[0][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'activity': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[-1][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'creator': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: name = self.db.getjournal(self.classname, nodeid)[0][2] else: return None try: return self.db.user.lookup(name) except KeyError: # the journaltag user doesn't exist any more return None # get the property (raises KeyErorr if invalid) prop = self.properties[propname] # get the node's dict d = self.db.getnode(self.classname, nodeid, cache=cache) if not d.has_key(propname): if default is _marker: if isinstance(prop, Multilink): return [] else: return None else: return default # return a dupe of the list so code doesn't get confused if isinstance(prop, Multilink): return d[propname][:] return d[propname] # XXX not in spec def getnode(self, nodeid, cache=1): ''' Return a convenience wrapper for the node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'cache' indicates whether the transaction cache should be queried for the node. If the node has been modified and you need to determine what its values prior to modification are, you need to set cache=0. ''' return Node(self, nodeid, cache=cache) def set(self, nodeid, **propvalues): '''Modify a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. Each key in 'propvalues' must be the name of a property of this class or a KeyError is raised. All values in 'propvalues' must be acceptable types for their corresponding properties or a TypeError is raised. If the value of the key property is set, it must not collide with other key strings or a ValueError is raised. If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' if not propvalues: return propvalues if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.fireAuditors('set', nodeid, propvalues) # Take a copy of the node dict so that the subsequent set # operation doesn't modify the oldvalues structure. try: # try not using the cache initially oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid, cache=0)) except IndexError: # this will be needed if somone does a create() and set() # with no intervening commit() oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) node = self.db.getnode(self.classname, nodeid) if node.has_key(self.db.RETIRED_FLAG): raise IndexError num_re = re.compile('^\d+$') # if the journal value is to be different, store it in here journalvalues = {} for propname, value in propvalues.items(): # check to make sure we're not duplicating an existing key if propname == self.key and node[propname] != value: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. prop = self.properties[propname] # if the value's the same as the existing value, no sense in # doing anything if node.has_key(propname) and value == node[propname]: del propvalues[propname] continue # do stuff based on the prop type if isinstance(prop, Link): link_class = prop.classname # if it isn't a number, it's a key if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( propname) if isinstance(value, type('')) and not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) if self.do_journal and prop.do_journal: # register the unlink with the old linked node if node[propname] is not None: self.db.addjournal(link_class, node[propname], 'unlink', (self.classname, nodeid, propname)) # register the link with the newly linked node if value is not None: self.db.addjournal(link_class, value, 'link', (self.classname, nodeid, propname)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of'\ ' ids'%propname link_class = self.properties[propname].classname l = [] for entry in value: # if it isn't a number, it's a key if type(entry) != type(''): raise ValueError, 'new property "%s" link value ' \ 'must be a string'%propname if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, entry, self.properties[propname].classname) l.append(entry) value = l propvalues[propname] = value # figure the journal entry for this property add = [] remove = [] # handle removals if node.has_key(propname): l = node[propname] else: l = [] for id in l[:]: if id in value: continue # register the unlink with the old linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'unlink', (self.classname, nodeid, propname)) l.remove(id) remove.append(id) # handle additions for id in value: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%(link_class, id) if id in l: continue # register the link with the newly linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'link', (self.classname, nodeid, propname)) l.append(id) add.append(id) # figure the journal entry l = [] if add: l.append(('+', add)) if remove: l.append(('-', remove)) if l: journalvalues[propname] = tuple(l) elif isinstance(prop, String): if value is not None and type(value) != type(''): raise TypeError, 'new property "%s" not a string'%propname elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% propname propvalues[propname] = value elif value is not None and isinstance(prop, Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an '\ 'Interval'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%propname elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%propname node[propname] = value # nothing to do? if not propvalues: return propvalues # do the set, and journal it self.db.setnode(self.classname, nodeid, node) if self.do_journal: propvalues.update(journalvalues) self.db.addjournal(self.classname, nodeid, 'set', propvalues) self.fireReactors('set', nodeid, oldvalues) return propvalues def retire(self, nodeid): '''Retire a node. The properties on the node remain available from the get() method, and the node's id is never reused. Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.fireAuditors('retire', nodeid, None) node = self.db.getnode(self.classname, nodeid) node[self.db.RETIRED_FLAG] = 1 self.db.setnode(self.classname, nodeid, node) if self.do_journal: self.db.addjournal(self.classname, nodeid, 'retired', None) self.fireReactors('retire', nodeid, None) def is_retired(self, nodeid): '''Return true if the node is retired. ''' node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): return 1 return 0 def destroy(self, nodeid): '''Destroy a node. WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted WARNING: use retire() instead WARNING: the properties of this node will not be available ever again WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.db.destroynode(self.classname, nodeid) def history(self, nodeid): '''Retrieve the journal of edits on a particular node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. The returned list contains tuples of the form (date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.getjournal(self.classname, nodeid) # Locating nodes: def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' return self.db.hasnode(self.classname, nodeid) def setkey(self, propname): '''Select a String property of this class to be the key property. 'propname' must be the name of a String property of this class or None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. If the property doesn't exist, KeyError is raised. ''' prop = self.getprops()[propname] if not isinstance(prop, String): raise TypeError, 'key properties must be String' self.key = propname def getkey(self): '''Return the name of the key property for this class or None.''' return self.key def labelprop(self, default_to_id=0): ''' Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] # TODO: set up a separate index db file for this? profile? def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. If this class has no key property, a TypeError is raised. If the 'keyvalue' matches one of the values for the key property among the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. ''' if not self.key: raise TypeError, 'No key property set' cldb = self.db.getclassdb(self.classname) try: for nodeid in self.db.getnodeids(self.classname, cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue if node[self.key] == keyvalue: cldb.close() return nodeid finally: cldb.close() raise KeyError, keyvalue # XXX: change from spec - allows multiple props to match def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname={nodeid:1,} 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Used by the full text indexing, which knows that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' propspec = propspec.items() for propname, nodeids in propspec: # check the prop is OK prop = self.properties[propname] if not isinstance(prop, Link) and not isinstance(prop, Multilink): raise TypeError, "'%s' not a Link/Multilink property"%propname # ok, now do the find cldb = self.db.getclassdb(self.classname) l = [] try: for id in self.db.getnodeids(self.classname, db=cldb): node = self.db.getnode(self.classname, id, db=cldb) if node.has_key(self.db.RETIRED_FLAG): continue for propname, nodeids in propspec: # can't test if the node doesn't have this property if not node.has_key(propname): continue if type(nodeids) is type(''): nodeids = {nodeids:1} prop = self.properties[propname] value = node[propname] if isinstance(prop, Link) and nodeids.has_key(value): l.append(id) break elif isinstance(prop, Multilink): hit = 0 for v in value: if nodeids.has_key(v): l.append(id) hit = 1 break if hit: break finally: cldb.close() return l def stringFind(self, **requirements): '''Locate a particular node by matching a set of its String properties in a caseless search. If the property is not a String property, a TypeError is raised. The return is a list of the id of all nodes that match. ''' for propname in requirements.keys(): prop = self.properties[propname] if isinstance(not prop, String): raise TypeError, "'%s' not a String property"%propname requirements[propname] = requirements[propname].lower() l = [] cldb = self.db.getclassdb(self.classname) try: for nodeid in self.db.getnodeids(self.classname, cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue for key, value in requirements.items(): if node[key] is None or node[key].lower() != value: break else: l.append(nodeid) finally: cldb.close() return l def list(self): ''' Return a list of the ids of the active nodes in this class. ''' l = [] cn = self.classname cldb = self.db.getclassdb(cn) try: for nodeid in self.db.getnodeids(cn, cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue l.append(nodeid) finally: cldb.close() l.sort() return l def filter(self, search_matches, filterspec, sort, group, num_re = re.compile('^\d+$')): ''' Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec. "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} ''' cn = self.classname # optimise filterspec l = [] props = self.getprops() LINK = 0 MULTILINK = 1 STRING = 2 OTHER = 6 for k, v in filterspec.items(): propclass = props[k] if isinstance(propclass, Link): if type(v) is not type([]): v = [v] # replace key values with node ids u = [] link_class = self.db.classes[propclass.classname] for entry in v: if entry == '-1': entry = None elif not num_re.match(entry): try: entry = link_class.lookup(entry) except (TypeError,KeyError): raise ValueError, 'property "%s": %s not a %s'%( k, entry, self.properties[k].classname) u.append(entry) l.append((LINK, k, u)) elif isinstance(propclass, Multilink): if type(v) is not type([]): v = [v] # replace key values with node ids u = [] link_class = self.db.classes[propclass.classname] for entry in v: if not num_re.match(entry): try: entry = link_class.lookup(entry) except (TypeError,KeyError): raise ValueError, 'new property "%s": %s not a %s'%( k, entry, self.properties[k].classname) u.append(entry) l.append((MULTILINK, k, u)) elif isinstance(propclass, String): # simple glob searching v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) v = v.replace('?', '.') v = v.replace('*', '.*?') l.append((STRING, k, re.compile(v, re.I))) elif isinstance(propclass, Boolean): if type(v) is type(''): bv = v.lower() in ('yes', 'true', 'on', '1') else: bv = v l.append((OTHER, k, bv)) elif isinstance(propclass, Number): l.append((OTHER, k, int(v))) else: l.append((OTHER, k, v)) filterspec = l # now, find all the nodes that are active and pass filtering l = [] cldb = self.db.getclassdb(cn) try: # TODO: only full-scan once (use items()) for nodeid in self.db.getnodeids(cn, cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue # apply filter for t, k, v in filterspec: # make sure the node has the property if not node.has_key(k): # this node doesn't have this property, so reject it break # now apply the property filter if t == LINK: # link - if this node's property doesn't appear in the # filterspec's nodeid list, skip it if node[k] not in v: break elif t == MULTILINK: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip # it have = node[k] for want in v: if want not in have: break else: continue break elif t == STRING: # RE search if node[k] is None or not v.search(node[k]): break elif t == OTHER: # straight value comparison for the other types if node[k] != v: break else: l.append((nodeid, node)) finally: cldb.close() l.sort() # filter based on full text search if search_matches is not None: k = [] for v in l: if search_matches.has_key(v[0]): k.append(v) l = k # now, sort the result def sortfun(a, b, sort=sort, group=group, properties=self.getprops(), db = self.db, cl=self): a_id, an = a b_id, bn = b # sort by group and then sort for dir, prop in group, sort: if dir is None: continue # sorting is class-specific propclass = properties[prop] # handle the properties that might be "faked" # also, handle possible missing properties try: if not an.has_key(prop): an[prop] = cl.get(a_id, prop) av = an[prop] except KeyError: # the node doesn't have a value for this property if isinstance(propclass, Multilink): av = [] else: av = '' try: if not bn.has_key(prop): bn[prop] = cl.get(b_id, prop) bv = bn[prop] except KeyError: # the node doesn't have a value for this property if isinstance(propclass, Multilink): bv = [] else: bv = '' # String and Date values are sorted in the natural way if isinstance(propclass, String): # clean up the strings if av and av[0] in string.uppercase: av = an[prop] = av.lower() if bv and bv[0] in string.uppercase: bv = bn[prop] = bv.lower() if (isinstance(propclass, String) or isinstance(propclass, Date)): # it might be a string that's really an integer try: av = int(av) bv = int(bv) except: pass if dir == '+': r = cmp(av, bv) if r != 0: return r elif dir == '-': r = cmp(bv, av) if r != 0: return r # Link properties are sorted according to the value of # the "order" property on the linked nodes if it is # present; or otherwise on the key string of the linked # nodes; or finally on the node ids. elif isinstance(propclass, Link): link = db.classes[propclass.classname] if av is None and bv is not None: return -1 if av is not None and bv is None: return 1 if av is None and bv is None: continue if link.getprops().has_key('order'): if dir == '+': r = cmp(link.get(av, 'order'), link.get(bv, 'order')) if r != 0: return r elif dir == '-': r = cmp(link.get(bv, 'order'), link.get(av, 'order')) if r != 0: return r elif link.getkey(): key = link.getkey() if dir == '+': r = cmp(link.get(av, key), link.get(bv, key)) if r != 0: return r elif dir == '-': r = cmp(link.get(bv, key), link.get(av, key)) if r != 0: return r else: if dir == '+': r = cmp(av, bv) if r != 0: return r elif dir == '-': r = cmp(bv, av) if r != 0: return r # Multilink properties are sorted according to how many # links are present. elif isinstance(propclass, Multilink): if dir == '+': r = cmp(len(av), len(bv)) if r != 0: return r elif dir == '-': r = cmp(len(bv), len(av)) if r != 0: return r elif isinstance(propclass, Number) or isinstance(propclass, Boolean): if dir == '+': r = cmp(av, bv) elif dir == '-': r = cmp(bv, av) # end for dir, prop in sort, group: # if all else fails, compare the ids return cmp(a[0], b[0]) l.sort(sortfun) return [i[0] for i in l] def count(self): '''Get the number of nodes in this class. If the returned integer is 'numnodes', the ids of all the nodes in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. ''' return self.db.countnodes(self.classname) # Manipulating properties: def getprops(self, protected=1): '''Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - those which may not be modified. In addition to the actual properties on the node, these methods provide the "creation" and "activity" properties. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = self.properties.copy() if protected: d['id'] = String() d['creation'] = hyperdb.Date() d['activity'] = hyperdb.Date() d['creator'] = hyperdb.Link("user") return d def addprop(self, **properties): '''Add properties to this class. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. None of the keys in 'properties' may collide with the names of existing properties, or a ValueError is raised before any properties have been added. ''' for key in properties.keys(): if self.properties.has_key(key): raise ValueError, key self.properties.update(properties) def index(self, nodeid): '''Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, String) and propclass.indexme: try: value = str(self.get(nodeid, prop)) except IndexError: # node no longer exists - entry should be removed self.db.indexer.purge_entry((self.classname, nodeid, prop)) else: # and index them under (classname, nodeid, property) self.db.indexer.add_text((self.classname, nodeid, prop), value) # # Detector interface # def audit(self, event, detector): '''Register a detector ''' l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): '''Fire all registered auditors. ''' for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def react(self, event, detector): '''Register a detector ''' l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) def fireReactors(self, action, nodeid, oldvalues): '''Fire all registered reactors. ''' for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) class FileClass(Class): '''This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. The default MIME type of this data is defined by the "default_mime_type" class attribute, which may be overridden by each node if the class defines a "type" String property. ''' default_mime_type = 'text/plain' def create(self, **propvalues): ''' snaffle the file propvalue and store in a file ''' content = propvalues['content'] del propvalues['content'] newid = Class.create(self, **propvalues) self.db.storefile(self.classname, newid, None, content) return newid def import_list(self, propnames, proplist): ''' Trap the "content" property... ''' # dupe this list so we don't affect others propnames = propnames[:] # extract the "content" property from the proplist i = propnames.index('content') content = proplist[i] del propnames[i] del proplist[i] # do the normal import newid = Class.import_list(self, propnames, proplist) # save off the "content" file self.db.storefile(self.classname, newid, None, content) return newid def get(self, nodeid, propname, default=_marker, cache=1): ''' trap the content propname and get it from the file ''' poss_msg = 'Possibly a access right configuration problem.' if propname == 'content': try: return self.db.getfile(self.classname, nodeid, None) except IOError, (strerror): # BUG: by catching this we donot see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) if default is not _marker: return Class.get(self, nodeid, propname, default, cache=cache) else: return Class.get(self, nodeid, propname, cache=cache) def getprops(self, protected=1): ''' In addition to the actual properties on the node, these methods provide the "content" property. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = Class.getprops(self, protected=protected).copy() if protected: d['content'] = hyperdb.String() return d def index(self, nodeid): ''' Index the node in the search index. We want to index the content in addition to the normal String property indexing. ''' # perform normal indexing Class.index(self, nodeid) # get the content to index content = self.get(nodeid, 'content') # figure the mime type if self.properties.has_key('type'): mime_type = self.get(nodeid, 'type') else: mime_type = self.default_mime_type # and index! self.db.indexer.add_text((self.classname, nodeid, 'content'), content, mime_type) # XXX deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): '''The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation" or "activity" property, a ValueError is raised. ''' if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) # #$Log: not supported by cvs2svn $ #Revision 1.68 2002/09/03 05:46:20 richard #handle disappearing users for journaltags # #Revision 1.67 2002/09/03 02:53:53 richard #Fixed nasty bug that was preventing changes to multilinks going through. # #Revision 1.66 2002/09/01 04:32:30 richard #. Lots of cleanup in the classic html (stylesheet, search page, index page, ...) #. Reinstated searching, but not query saving yet #. Filtering only allows sorting and grouping by one property - all backends # now implement this behaviour. #. Nosy list journalling turned off by default, everything else is on. #. Added some convenience methods (reverse, propchanged, [item] accesses, ...) #. Did I mention the stylesheet is much cleaner now? :) # #Revision 1.65 2002/08/30 08:35:45 richard #minor edits # #Revision 1.64 2002/08/22 07:57:11 richard #Consistent quoting # #Revision 1.63 2002/08/22 04:42:28 richard #use more robust date stamp comparisons in pack(), make journal smaller too # #Revision 1.62 2002/08/21 07:07:27 richard #In preparing to turn back on link/unlink journal events (by default these #are turned off) I've: #- fixed back_anydbm so it can journal those events again (had broken it # with recent changes) #- changed the serialisation format for dates and intervals to use a # numbers-only (and sign for Intervals) string instead of tuple-of-ints. # Much smaller. # #Revision 1.61 2002/08/19 02:53:27 richard #full database export and import is done # #Revision 1.60 2002/08/19 00:23:19 richard #handle "unset" initial Link values (!) # #Revision 1.59 2002/08/16 04:28:13 richard #added is_retired query to Class # #Revision 1.58 2002/08/01 15:06:24 gmcm #Use same regex to split search terms as used to index text. #Fix to back_metakit for not changing journaltag on reopen. #Fix htmltemplate's do_link so [No ] strings are href'd. #Fix bogus "nosy edited ok" msg - the **d syntax does NOT share d between caller and callee. # #Revision 1.57 2002/07/31 23:57:36 richard # . web forms may now unset Link values (like assignedto) # #Revision 1.56 2002/07/31 22:04:33 richard #cleanup # #Revision 1.55 2002/07/30 08:22:38 richard #Session storage in the hyperdb was horribly, horribly inefficient. We use #a simple anydbm wrapper now - which could be overridden by the metakit #backend or RDB backend if necessary. #Much, much better. # #Revision 1.54 2002/07/26 08:26:59 richard #Very close now. The cgi and mailgw now use the new security API. The two #templates have been migrated to that setup. Lots of unit tests. Still some #issue in the web form for editing Roles assigned to users. # #Revision 1.53 2002/07/25 07:14:06 richard #Bugger it. Here's the current shape of the new security implementation. #Still to do: # . call the security funcs from cgi and mailgw # . change shipped templates to include correct initialisation and remove # the old config vars #... that seems like a lot. The bulk of the work has been done though. Honest :) # #Revision 1.52 2002/07/19 03:36:34 richard #Implemented the destroy() method needed by the session database (and possibly #others). At the same time, I removed the leading underscores from the hyperdb #methods that Really Didn't Need Them. #The journal also raises IndexError now for all situations where there is a #request for the journal of a node that doesn't have one. It used to return #[] in _some_ situations, but not all. This _may_ break code, but the tests #pass... # #Revision 1.51 2002/07/18 23:07:08 richard #Unit tests and a few fixes. # #Revision 1.50 2002/07/18 11:50:58 richard #added tests for number type too # #Revision 1.49 2002/07/18 11:41:10 richard #added tests for boolean type, and fixes to anydbm backend # #Revision 1.48 2002/07/18 11:17:31 gmcm #Add Number and Boolean types to hyperdb. #Add conversion cases to web, mail & admin interfaces. #Add storage/serialization cases to back_anydbm & back_metakit. # #Revision 1.47 2002/07/14 23:18:20 richard #. fixed the journal bloat from multilink changes - we just log the add or # remove operations, not the whole list # #Revision 1.46 2002/07/14 06:06:34 richard #Did some old TODOs # #Revision 1.45 2002/07/14 04:03:14 richard #Implemented a switch to disable journalling for a Class. CGI session #database now uses it. # #Revision 1.44 2002/07/14 02:05:53 richard #. all storage-specific code (ie. backend) is now implemented by the backends # #Revision 1.43 2002/07/10 06:30:30 richard #...except of course it's nice to use valid Python syntax # #Revision 1.42 2002/07/10 06:21:38 richard #Be extra safe # #Revision 1.41 2002/07/10 00:21:45 richard #explicit database closing # #Revision 1.40 2002/07/09 04:19:09 richard #Added reindex command to roundup-admin. #Fixed reindex on first access. #Also fixed reindexing of entries that change. # #Revision 1.39 2002/07/09 03:02:52 richard #More indexer work: #- all String properties may now be indexed too. Currently there's a bit of # "issue" specific code in the actual searching which needs to be # addressed. In a nutshell: # + pass 'indexme="yes"' as a String() property initialisation arg, eg: # file = FileClass(db, "file", name=String(), type=String(), # comment=String(indexme="yes")) # + the comment will then be indexed and be searchable, with the results # related back to the issue that the file is linked to #- as a result of this work, the FileClass has a default MIME type that may # be overridden in a subclass, or by the use of a "type" property as is # done in the default templates. #- the regeneration of the indexes (if necessary) is done once the schema is # set up in the dbinit. # #Revision 1.38 2002/07/08 06:58:15 richard #cleaned up the indexer code: # - it splits more words out (much simpler, faster splitter) # - removed code we'll never use (roundup.roundup_indexer has the full # implementation, and replaces roundup.indexer) # - only index text/plain and rfc822/message (ideas for other text formats to # index are welcome) # - added simple unit test for indexer. Needs more tests for regression. # #Revision 1.37 2002/06/20 23:52:35 richard #More informative error message # #Revision 1.36 2002/06/19 03:07:19 richard #Moved the file storage commit into blobfiles where it belongs. # #Revision 1.35 2002/05/25 07:16:24 rochecompaan #Merged search_indexing-branch with HEAD # #Revision 1.34 2002/05/15 06:21:21 richard # . node caching now works, and gives a small boost in performance # #As a part of this, I cleaned up the DEBUG output and implemented TRACE #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff #(using if __debug__ which is compiled out with -O) # #Revision 1.33 2002/04/24 10:38:26 rochecompaan #All database files are now created group readable and writable. # #Revision 1.32 2002/04/15 23:25:15 richard #. node ids are now generated from a lockable store - no more race conditions # #We're using the portalocker code by Jonathan Feinberg that was contributed #to the ASPN Python cookbook. This gives us locking across Unix and Windows. # #Revision 1.31 2002/04/03 05:54:31 richard #Fixed serialisation problem by moving the serialisation step out of the #hyperdb.Class (get, set) into the hyperdb.Database. # #Also fixed htmltemplate after the showid changes I made yesterday. # #Unit tests for all of the above written. # #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan # . Added feature #526730 - search for messages capability # #Revision 1.30 2002/02/27 03:40:59 richard #Ran it through pychecker, made fixes # #Revision 1.29 2002/02/25 14:34:31 grubert # . use blobfiles in back_anydbm which is used in back_bsddb. # change test_db as dirlist does not work for subdirectories. # ATTENTION: blobfiles now creates subdirectories for files. # #Revision 1.28 2002/02/16 09:14:17 richard # . #514854 ] History: "User" is always ticket creator # #Revision 1.27 2002/01/22 07:21:13 richard #. fixed back_bsddb so it passed the journal tests # #... it didn't seem happy using the back_anydbm _open method, which is odd. #Yet another occurrance of whichdb not being able to recognise older bsddb #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the #process. # #Revision 1.26 2002/01/22 05:18:38 rochecompaan #last_set_entry was referenced before assignment # #Revision 1.25 2002/01/22 05:06:08 rochecompaan #We need to keep the last 'set' entry in the journal to preserve #information on 'activity' for nodes. # #Revision 1.24 2002/01/21 16:33:20 rochecompaan #You can now use the roundup-admin tool to pack the database # #Revision 1.23 2002/01/18 04:32:04 richard #Rollback was breaking because a message hadn't actually been written to the file. Needs #more investigation. # #Revision 1.22 2002/01/14 02:20:15 richard # . changed all config accesses so they access either the instance or the # config attriubute on the db. This means that all config is obtained from # instance_config instead of the mish-mash of classes. This will make # switching to a ConfigParser setup easier too, I hope. # #At a minimum, this makes migration a _little_ easier (a lot easier in the #0.5.0 switch, I hope!) # #Revision 1.21 2002/01/02 02:31:38 richard #Sorry for the huge checkin message - I was only intending to implement #496356 #but I found a number of places where things had been broken by transactions: # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename # for _all_ roundup-generated smtp messages to be sent to. # . the transaction cache had broken the roundupdb.Class set() reactors # . newly-created author users in the mailgw weren't being committed to the db # #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working #on when I found that stuff :): # . #496356 ] Use threading in messages # . detectors were being registered multiple times # . added tests for mailgw # . much better attaching of erroneous messages in the mail gateway # #Revision 1.20 2001/12/18 15:30:34 rochecompaan #Fixed bugs: # . Fixed file creation and retrieval in same transaction in anydbm # backend # . Cgi interface now renders new issue after issue creation # . Could not set issue status to resolved through cgi interface # . Mail gateway was changing status back to 'chatting' if status was # omitted as an argument # #Revision 1.19 2001/12/17 03:52:48 richard #Implemented file store rollback. As a bonus, the hyperdb is now capable of #storing more than one file per node - if a property name is supplied, #the file is called designator.property. #I decided not to migrate the existing files stored over to the new naming #scheme - the FileClass just doesn't specify the property name. # #Revision 1.18 2001/12/16 10:53:38 richard #take a copy of the node dict so that the subsequent set #operation doesn't modify the oldvalues structure # #Revision 1.17 2001/12/14 23:42:57 richard #yuck, a gdbm instance tests false :( #I've left the debugging code in - it should be removed one day if we're ever #_really_ anal about performace :) # #Revision 1.16 2001/12/12 03:23:14 richard #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb #incorrectly identifies a dbm file as a dbhash file on my system. This has #been submitted to the python bug tracker as issue #491888: #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470 # #Revision 1.15 2001/12/12 02:30:51 richard #I fixed the problems with people whose anydbm was using the dbm module at the #backend. It turns out the dbm module modifies the file name to append ".db" #and my check to determine if we're opening an existing or new db just #tested os.path.exists() on the filename. Well, no longer! We now perform a #much better check _and_ cope with the anydbm implementation module changing #too! #I also fixed the backends __init__ so only ImportError is squashed. # #Revision 1.14 2001/12/10 22:20:01 richard #Enabled transaction support in the bsddb backend. It uses the anydbm code #where possible, only replacing methods where the db is opened (it uses the #btree opener specifically.) #Also cleaned up some change note generation. #Made the backends package work with pydoc too. # #Revision 1.13 2001/12/02 05:06:16 richard #. We now use weakrefs in the Classes to keep the database reference, so # the close() method on the database is no longer needed. # I bumped the minimum python requirement up to 2.1 accordingly. #. #487480 ] roundup-server #. #487476 ] INSTALL.txt # #I also cleaned up the change message / post-edit stuff in the cgi client. #There's now a clearly marked "TODO: append the change note" where I believe #the change note should be added there. The "changes" list will obviously #have to be modified to be a dict of the changes, or somesuch. # #More testing needed. # #Revision 1.12 2001/12/01 07:17:50 richard #. We now have basic transaction support! Information is only written to # the database when the commit() method is called. Only the anydbm # backend is modified in this way - neither of the bsddb backends have been. # The mail, admin and cgi interfaces all use commit (except the admin tool # doesn't have a commit command, so interactive users can't commit...) #. Fixed login/registration forwarding the user to the right page (or not, # on a failure) # #Revision 1.11 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.10 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.9 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.8 2001/09/29 13:27:00 richard #CGI interfaces now spit up a top-level index of all the instances they can #serve. # #Revision 1.7 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.6 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.5 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.4 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.3 2001/07/25 01:23:07 richard #Added the Roundup spec to the new documentation directory. # #Revision 1.2 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # # code-c3e391d9c4e9/roundup/backends/back_bsddb.py0000644000000000000000000002121007535063756017637 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_bsddb.py,v 1.21 2002-09-03 07:33:01 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' import bsddb, os, marshal from roundup import hyperdb, date # these classes are so similar, we just use the anydbm methods from back_anydbm import Database, Class, FileClass, IssueClass # # Now the database # class Database(Database): """A database for storing records containing flexible data types.""" # # Class DBs # def clear(self): for cn in self.classes.keys(): db = os.path.join(self.dir, 'nodes.%s'%cn) bsddb.btopen(db, 'n') db = os.path.join(self.dir, 'journals.%s'%cn) bsddb.btopen(db, 'n') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname) if os.path.exists(path): return bsddb.btopen(path, mode) else: return bsddb.btopen(path, 'n') def opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if __debug__: print >>hyperdb.DEBUG, self, 'opendb', (self, name, mode) # determine which DB wrote the class file path = os.path.join(os.getcwd(), self.dir, name) if not os.path.exists(path): if __debug__: print >>hyperdb.DEBUG, "opendb bsddb.open(%r, 'c')"%path return bsddb.btopen(path, 'c') # open the database with the correct module if __debug__: print >>hyperdb.DEBUG, "opendb bsddb.open(%r, %r)"%(path, mode) return bsddb.btopen(path, mode) # # Journal # def getjournal(self, classname, nodeid): ''' get the journal for id ''' # attempt to open the journal - in some rare cases, the journal may # not exist try: db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb.error, error: if error.args[0] != 2: raise raise IndexError, 'no such %s %s'%(classname, nodeid) # more handling of bad journals if not db.has_key(nodeid): raise IndexError, 'no such %s %s'%(classname, nodeid) journal = marshal.loads(db[nodeid]) res = [] for entry in journal: (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, user, action, params)) db.close() return res def getCachedJournalDB(self, classname): ''' get the journal db, looking in our cache of databases for commit ''' # get the database handle db_name = 'journals.%s'%classname if self.databases.has_key(db_name): return self.databases[db_name] else: db = bsddb.btopen(os.path.join(self.dir, db_name), 'c') self.databases[db_name] = db return db def doSaveJournal(self, classname, nodeid, action, params): # serialise first if action in ('set', 'create'): params = self.serialise(classname, params) entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) if __debug__: print >>hyperdb.DEBUG, 'doSaveJournal', entry db = self.getCachedJournalDB(classname) if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) # #$Log: not supported by cvs2svn $ #Revision 1.20 2002/07/19 03:36:34 richard #Implemented the destroy() method needed by the session database (and possibly #others). At the same time, I removed the leading underscores from the hyperdb #methods that Really Didn't Need Them. #The journal also raises IndexError now for all situations where there is a #request for the journal of a node that doesn't have one. It used to return #[] in _some_ situations, but not all. This _may_ break code, but the tests #pass... # #Revision 1.19 2002/07/14 02:05:53 richard #. all storage-specific code (ie. backend) is now implemented by the backends # #Revision 1.18 2002/05/15 06:21:21 richard # . node caching now works, and gives a small boost in performance # #As a part of this, I cleaned up the DEBUG output and implemented TRACE #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff #(using if __debug__ which is compiled out with -O) # #Revision 1.17 2002/04/03 05:54:31 richard #Fixed serialisation problem by moving the serialisation step out of the #hyperdb.Class (get, set) into the hyperdb.Database. # #Also fixed htmltemplate after the showid changes I made yesterday. # #Unit tests for all of the above written. # #Revision 1.16 2002/02/27 03:40:59 richard #Ran it through pychecker, made fixes # #Revision 1.15 2002/02/16 09:15:33 richard #forgot to patch bsddb backend too # #Revision 1.14 2002/01/22 07:21:13 richard #. fixed back_bsddb so it passed the journal tests # #... it didn't seem happy using the back_anydbm _open method, which is odd. #Yet another occurrance of whichdb not being able to recognise older bsddb #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the #process. # #Revision 1.13 2001/12/10 22:20:01 richard #Enabled transaction support in the bsddb backend. It uses the anydbm code #where possible, only replacing methods where the db is opened (it uses the #btree opener specifically.) #Also cleaned up some change note generation. #Made the backends package work with pydoc too. # #Revision 1.12 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.11 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.10 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.9 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.8 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.7 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.6 2001/07/30 02:36:23 richard #Handle non-existence of db files in the other backends (code from anydbm). # #Revision 1.5 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.4 2001/07/23 08:25:33 richard #more handling of bad journals # #Revision 1.3 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # #Revision 1.2 2001/07/23 07:56:05 richard #Storing only marshallable data in the db - no nasty pickled class references. # #Revision 1.1 2001/07/23 07:22:13 richard #*sigh* some databases have _foo.so as their underlying implementation. #This time for sure, Rocky. # #Revision 1.1 2001/07/23 07:15:57 richard #Moved the backends into the backends package. Anydbm hasn't been tested at all. # #Revision 1.1 2001/07/23 06:23:41 richard #moved hyper_bsddb.py to the new backends package as bsddb.py # #Revision 1.2 2001/07/22 12:09:32 richard #Final commit of Grande Splite # #Revision 1.1 2001/07/22 11:58:35 richard #More Grande Splite # code-c3e391d9c4e9/roundup/backends/back_bsddb3.py0000644000000000000000000001710207535063756017727 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_bsddb3.py,v 1.15 2002-07-19 03:36:34 richard Exp $ import bsddb3, os, marshal from roundup import hyperdb, date # these classes are so similar, we just use the anydbm methods from back_anydbm import Database, Class, FileClass, IssueClass # # Now the database # class Database(Database): """A database for storing records containing flexible data types.""" # # Class DBs # def clear(self): for cn in self.classes.keys(): db = os.path.join(self.dir, 'nodes.%s'%cn) bsddb3.btopen(db, 'n') db = os.path.join(self.dir, 'journals.%s'%cn) bsddb3.btopen(db, 'n') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname) if os.path.exists(path): return bsddb3.btopen(path, mode) else: return bsddb3.btopen(path, 'c') def opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if __debug__: print >>hyperdb.DEBUG, self, 'opendb', (self, name, mode) # determine which DB wrote the class file path = os.path.join(os.getcwd(), self.dir, name) if not os.path.exists(path): if __debug__: print >>hyperdb.DEBUG, "opendb bsddb3.open(%r, 'c')"%path return bsddb3.btopen(path, 'c') # open the database with the correct module if __debug__: print >>hyperdb.DEBUG, "opendb bsddb3.open(%r, %r)"%(path, mode) return bsddb3.btopen(path, mode) # # Journal # def getjournal(self, classname, nodeid): ''' get the journal for id ''' # attempt to open the journal - in some rare cases, the journal may # not exist try: db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb3._db.DBNoSuchFileError: raise IndexError, 'no such %s %s'%(classname, nodeid) # more handling of bad journals if not db.has_key(nodeid): raise IndexError, 'no such %s %s'%(classname, nodeid) journal = marshal.loads(db[nodeid]) res = [] for entry in journal: (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, user, action, params)) db.close() return res def getCachedJournalDB(self, classname): ''' get the journal db, looking in our cache of databases for commit ''' # get the database handle db_name = 'journals.%s'%classname if self.databases.has_key(db_name): return self.databases[db_name] else: db = bsddb3.btopen(os.path.join(self.dir, db_name), 'c') self.databases[db_name] = db return db def doSaveJournal(self, classname, nodeid, action, params): # serialise first if action in ('set', 'create'): params = self.serialise(classname, params) entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) if __debug__: print >>hyperdb.DEBUG, 'doSaveJournal', entry db = self.getCachedJournalDB(classname) if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) # #$Log: not supported by cvs2svn $ #Revision 1.14 2002/07/14 02:05:54 richard #. all storage-specific code (ie. backend) is now implemented by the backends # #Revision 1.13 2002/07/08 06:41:03 richard #Was reopening the database with 'n'. # #Revision 1.12 2002/05/21 05:52:11 richard #Well whadya know, bsddb3 works again. #The backend is implemented _exactly_ the same as bsddb - so there's no #using its transaction or locking support. It'd be nice to use those some #day I suppose. # #Revision 1.11 2002/01/14 02:20:15 richard # . changed all config accesses so they access either the instance or the # config attriubute on the db. This means that all config is obtained from # instance_config instead of the mish-mash of classes. This will make # switching to a ConfigParser setup easier too, I hope. # #At a minimum, this makes migration a _little_ easier (a lot easier in the #0.5.0 switch, I hope!) # #Revision 1.10 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.9 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.8 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.7 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.6 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.5 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.4 2001/08/03 02:45:47 anthonybaxter #'n' -> 'c' for create. # #Revision 1.3 2001/07/30 02:36:23 richard #Handle non-existence of db files in the other backends (code from anydbm). # #Revision 1.2 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.1 2001/07/24 04:26:03 anthonybaxter #bsddb3 implementation. For now, it's the bsddb implementation with a "3" #added in crayon. # #Revision 1.4 2001/07/23 08:25:33 richard #more handling of bad journals # #Revision 1.3 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # #Revision 1.2 2001/07/23 07:56:05 richard #Storing only marshallable data in the db - no nasty pickled class references. # #Revision 1.1 2001/07/23 07:22:13 richard #*sigh* some databases have _foo.so as their underlying implementation. #This time for sure, Rocky. # #Revision 1.1 2001/07/23 07:15:57 richard #Moved the backends into the backends package. Anydbm hasn't been tested at all. # #Revision 1.1 2001/07/23 06:23:41 richard #moved hyper_bsddb.py to the new backends package as bsddb.py # #Revision 1.2 2001/07/22 12:09:32 richard #Final commit of Grande Splite # #Revision 1.1 2001/07/22 11:58:35 richard #More Grande Splite # code-c3e391d9c4e9/roundup/backends/back_gadfly.py0000644000000000000000000024271707535063756020050 0ustar 00000000000000# $Id: back_gadfly.py,v 1.10 2002-09-03 06:08:50 richard Exp $ __doc__ = ''' About Gadfly ============ Gadfly is a collection of python modules that provides relational database functionality entirely implemented in Python. It supports a subset of the intergalactic standard RDBMS Structured Query Language SQL. Basic Structure =============== We map roundup classes to relational tables. Automatically detect schema changes and modify the gadfly table schemas appropriately. Multilinks (which represent a many-to-many relationship) are handled through intermediate tables. Journals are stored adjunct to the per-class tables. Table names and columns have "_" prepended so the names can't clash with restricted names (like "order"). Retirement is determined by the __retired__ column being true. All columns are defined as VARCHAR, since it really doesn't matter what type they're defined as. We stuff all kinds of data in there ;) [as long as it's marshallable, gadfly doesn't care] Additional Instance Requirements ================================ The instance configuration must specify where the database is. It does this with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly() method: Using an on-disk database directly (not a good idea): GADFLY_DATABASE = (database name, directory) Using a network database (much better idea): GADFLY_DATABASE = (policy, password, address, port) Because multiple accesses directly to a gadfly database aren't handled, but multiple network accesses are, it's strongly advised that the latter setup be used. ''' # standard python modules import sys, os, time, re, errno, weakref, copy # roundup modules from roundup import hyperdb, date, password, roundupdb, security from roundup.hyperdb import String, Password, Date, Interval, Link, \ Multilink, DatabaseError, Boolean, Number # the all-important gadfly :) import gadfly import gadfly.client import gadfly.database # support from blobfiles import FileStorage from roundup.indexer import Indexer from sessions import Sessions class Database(FileStorage, hyperdb.Database, roundupdb.Database): # flag to set on retired entries RETIRED_FLAG = '__hyperdb_retired' def __init__(self, config, journaltag=None): ''' Open the database and load the schema from it. ''' self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.indexer = Indexer(self.dir) self.sessions = Sessions(self.config) self.security = security.Security(self) # additional transaction support for external files and the like self.transactions = [] db = config.GADFLY_DATABASE if len(db) == 2: # ensure files are group readable and writable os.umask(0002) try: self.conn = gadfly.gadfly(*db) except IOError, error: if error.errno != errno.ENOENT: raise self.database_schema = {} self.conn = gadfly.gadfly() self.conn.startup(*db) cursor = self.conn.cursor() cursor.execute('create table schema (schema varchar)') cursor.execute('create table ids (name varchar, num integer)') else: cursor = self.conn.cursor() cursor.execute('select schema from schema') self.database_schema = cursor.fetchone()[0] else: self.conn = gadfly.client.gfclient(*db) cursor = self.conn.cursor() cursor.execute('select schema from schema') self.database_schema = cursor.fetchone()[0] def __repr__(self): return ''%id(self) def post_init(self): ''' Called once the schema initialisation has finished. We should now confirm that the schema defined by our "classes" attribute actually matches the schema in the database. ''' # now detect changes in the schema for classname, spec in self.classes.items(): if self.database_schema.has_key(classname): dbspec = self.database_schema[classname] self.update_class(spec, dbspec) self.database_schema[classname] = spec.schema() else: self.create_class(spec) self.database_schema[classname] = spec.schema() for classname in self.database_schema.keys(): if not self.classes.has_key(classname): self.drop_class(classname) # update the database version of the schema cursor = self.conn.cursor() cursor.execute('delete from schema') cursor.execute('insert into schema values (?)', (self.database_schema,)) # reindex the db if necessary if self.indexer.should_reindex(): self.reindex() # commit self.conn.commit() def reindex(self): for klass in self.classes.values(): for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() def determine_columns(self, spec): ''' Figure the column names and multilink properties from the spec ''' cols = [] mls = [] # add the multilinks separately for col, prop in spec.properties.items(): if isinstance(prop, Multilink): mls.append(col) else: cols.append('_'+col) cols.sort() return cols, mls def update_class(self, spec, dbspec): ''' Determine the differences between the current spec and the database version of the spec, and update where necessary NOTE that this doesn't work for adding/deleting properties! ... until gadfly grows an ALTER TABLE command, it's not going to! ''' spec_schema = spec.schema() if spec_schema == dbspec: return if __debug__: print >>hyperdb.DEBUG, 'update_class FIRING' # key property changed? if dbspec[0] != spec_schema[0]: if __debug__: print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]` # XXX turn on indexing for the key property # dict 'em up spec_propnames,spec_props = [],{} for propname,prop in spec_schema[1]: spec_propnames.append(propname) spec_props[propname] = prop dbspec_propnames,dbspec_props = [],{} for propname,prop in dbspec[1]: dbspec_propnames.append(propname) dbspec_props[propname] = prop # we're going to need one of these cursor = self.conn.cursor() # now compare for propname in spec_propnames: prop = spec_props[propname] if __debug__: print >>hyperdb.DEBUG, 'update_class ...', `prop` if dbspec_props.has_key(propname) and prop==dbspec_props[propname]: continue if __debug__: print >>hyperdb.DEBUG, 'update_class', `prop` if not dbspec_props.has_key(propname): # add the property if isinstance(prop, Multilink): sql = 'create table %s_%s (linkid varchar, nodeid '\ 'varchar)'%(spec.classname, prop) if __debug__: print >>hyperdb.DEBUG, 'update_class', (self, sql) cursor.execute(sql) else: # XXX gadfly doesn't have an ALTER TABLE command raise NotImplementedError sql = 'alter table _%s add column (_%s varchar)'%( spec.classname, propname) if __debug__: print >>hyperdb.DEBUG, 'update_class', (self, sql) cursor.execute(sql) else: # modify the property if __debug__: print >>hyperdb.DEBUG, 'update_class NOOP' pass # NOOP in gadfly # and the other way - only worry about deletions here for propname in dbspec_propnames: prop = dbspec_props[propname] if spec_props.has_key(propname): continue if __debug__: print >>hyperdb.DEBUG, 'update_class', `prop` # delete the property if isinstance(prop, Multilink): sql = 'drop table %s_%s'%(spec.classname, prop) if __debug__: print >>hyperdb.DEBUG, 'update_class', (self, sql) cursor.execute(sql) else: # XXX gadfly doesn't have an ALTER TABLE command raise NotImplementedError sql = 'alter table _%s delete column _%s'%(spec.classname, propname) if __debug__: print >>hyperdb.DEBUG, 'update_class', (self, sql) cursor.execute(sql) def create_class(self, spec): ''' Create a database table according to the given spec. ''' cols, mls = self.determine_columns(spec) # add on our special columns cols.append('id') cols.append('__retired__') cursor = self.conn.cursor() # create the base table cols = ','.join(['%s varchar'%x for x in cols]) sql = 'create table _%s (%s)'%(spec.classname, cols) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql) cursor.execute(sql) # journal table cols = ','.join(['%s varchar'%x for x in 'nodeid date tag action params'.split()]) sql = 'create table %s__journal (%s)'%(spec.classname, cols) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql) cursor.execute(sql) # now create the multilink tables for ml in mls: sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%( spec.classname, ml) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql) cursor.execute(sql) # ID counter sql = 'insert into ids (name, num) values (?,?)' vals = (spec.classname, 1) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql, vals) cursor.execute(sql, vals) def drop_class(self, spec): ''' Drop the given table from the database. Drop the journal and multilink tables too. ''' # figure the multilinks mls = [] for col, prop in spec.properties.items(): if isinstance(prop, Multilink): mls.append(col) cursor = self.conn.cursor() sql = 'drop table _%s'%spec.classname if __debug__: print >>hyperdb.DEBUG, 'drop_class', (self, sql) cursor.execute(sql) sql = 'drop table %s__journal'%spec.classname if __debug__: print >>hyperdb.DEBUG, 'drop_class', (self, sql) cursor.execute(sql) for ml in mls: sql = 'drop table %s_%s'%(spec.classname, ml) if __debug__: print >>hyperdb.DEBUG, 'drop_class', (self, sql) cursor.execute(sql) # # Classes # def __getattr__(self, classname): ''' A convenient way of calling self.getclass(classname). ''' if self.classes.has_key(classname): if __debug__: print >>hyperdb.DEBUG, '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): ''' Add a Class to the hyperdatabase. ''' if __debug__: print >>hyperdb.DEBUG, 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl def getclasses(self): ''' Return a list of the names of all existing classes. ''' if __debug__: print >>hyperdb.DEBUG, 'getclasses', (self,) l = self.classes.keys() l.sort() return l def getclass(self, classname): '''Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. ''' if __debug__: print >>hyperdb.DEBUG, 'getclass', (self, classname) return self.classes[classname] def clear(self): ''' Delete all database contents. Note: I don't commit here, which is different behaviour to the "nuke from orbit" behaviour in the *dbms. ''' if __debug__: print >>hyperdb.DEBUG, 'clear', (self,) cursor = self.conn.cursor() for cn in self.classes.keys(): sql = 'delete from _%s'%cn if __debug__: print >>hyperdb.DEBUG, 'clear', (self, sql) cursor.execute(sql) # # Node IDs # def newid(self, classname): ''' Generate a new id for the given class ''' # get the next ID cursor = self.conn.cursor() sql = 'select num from ids where name=?' if __debug__: print >>hyperdb.DEBUG, 'newid', (self, sql, classname) cursor.execute(sql, (classname, )) newid = cursor.fetchone()[0] # update the counter sql = 'update ids set num=? where name=?' vals = (newid+1, classname) if __debug__: print >>hyperdb.DEBUG, 'newid', (self, sql, vals) cursor.execute(sql, vals) # return as string return str(newid) def setid(self, classname, setid): ''' Set the id counter: used during import of database ''' cursor = self.conn.cursor() sql = 'update ids set num=? where name=?' vals = (setid, spec.classname) if __debug__: print >>hyperdb.DEBUG, 'setid', (self, sql, vals) cursor.execute(sql, vals) # # Nodes # def addnode(self, classname, nodeid, node): ''' Add the specified node to its class's db. ''' if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) # gadfly requires values for all non-multilink columns cl = self.classes[classname] cols, mls = self.determine_columns(cl) # default the non-multilink columns for col, prop in cl.properties.items(): if not isinstance(col, Multilink): if not node.has_key(col): node[col] = None node = self.serialise(classname, node) # make sure the ordering is correct for column name -> column value vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0) s = ','.join(['?' for x in cols]) + ',?,?' cols = ','.join(cols) + ',id,__retired__' # perform the inserts cursor = self.conn.cursor() sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s) if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, sql, vals) cursor.execute(sql, vals) # insert the multilink rows for col in mls: t = '%s_%s'%(classname, col) for entry in node[col]: sql = 'insert into %s (linkid, nodeid) values (?,?)'%t vals = (entry, nodeid) if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, sql, vals) cursor.execute(sql, vals) # make sure we do the commit-time extra stuff for this node self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def setnode(self, classname, nodeid, node, multilink_changes): ''' Change the specified node. ''' if __debug__: print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) node = self.serialise(classname, node) cl = self.classes[classname] cols = [] mls = [] # add the multilinks separately for col in node.keys(): prop = cl.properties[col] if isinstance(prop, Multilink): mls.append(col) else: cols.append('_'+col) cols.sort() # make sure the ordering is correct for column name -> column value vals = tuple([node[col[1:]] for col in cols]) s = ','.join(['%s=?'%x for x in cols]) cols = ','.join(cols) # perform the update cursor = self.conn.cursor() sql = 'update _%s set %s'%(classname, s) if __debug__: print >>hyperdb.DEBUG, 'setnode', (self, sql, vals) cursor.execute(sql, vals) # now the fun bit, updating the multilinks ;) for col, (add, remove) in multilink_changes.items(): tn = '%s_%s'%(classname, col) if add: sql = 'insert into %s (nodeid, linkid) values (?,?)'%tn vals = [(nodeid, addid) for addid in add] if __debug__: print >>hyperdb.DEBUG, 'setnode (add)', (self, sql, vals) cursor.execute(sql, vals) if remove: sql = 'delete from %s where nodeid=? and linkid=?'%tn vals = [(nodeid, removeid) for removeid in remove] if __debug__: print >>hyperdb.DEBUG, 'setnode (rem)', (self, sql, vals) cursor.execute(sql, vals) # make sure we do the commit-time extra stuff for this node self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid): ''' Get a node from the database. ''' if __debug__: print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid) # figure the columns we're fetching cl = self.classes[classname] cols, mls = self.determine_columns(cl) scols = ','.join(cols) # perform the basic property fetch cursor = self.conn.cursor() sql = 'select %s from _%s where id=?'%(scols, classname) if __debug__: print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) try: values = cursor.fetchone() except gadfly.database.error, message: if message == 'no more results': raise IndexError, 'no such %s node %s'%(classname, nodeid) raise # make up the node node = {} for col in range(len(cols)): node[cols[col][1:]] = values[col] # now the multilinks for col in mls: # get the link ids sql = 'select linkid from %s_%s where nodeid=?'%(classname, col) if __debug__: print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) # extract the first column from the result node[col] = [x[0] for x in cursor.fetchall()] return self.unserialise(classname, node) def destroynode(self, classname, nodeid): '''Remove a node from the database. Called exclusively by the destroy() method on Class. ''' if __debug__: print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid) # make sure the node exists if not self.hasnode(classname, nodeid): raise IndexError, '%s has no node %s'%(classname, nodeid) # see if there's any obvious commit actions that we should get rid of for entry in self.transactions[:]: if entry[1][:2] == (classname, nodeid): self.transactions.remove(entry) # now do the SQL cursor = self.conn.cursor() sql = 'delete from _%s where id=?'%(classname) if __debug__: print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) def serialise(self, classname, node): '''Copy the node contents, converting non-marshallable data into marshallable data. ''' if __debug__: print >>hyperdb.DEBUG, 'serialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): # if the property doesn't exist, or is the "retired" flag then # it won't be in the properties dict if not properties.has_key(k): d[k] = v continue # get the property spec prop = properties[k] if isinstance(prop, Password): d[k] = str(v) elif isinstance(prop, Date) and v is not None: d[k] = v.serialise() elif isinstance(prop, Interval) and v is not None: d[k] = v.serialise() else: d[k] = v return d def unserialise(self, classname, node): '''Decode the marshalled node data ''' if __debug__: print >>hyperdb.DEBUG, 'unserialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): # if the property doesn't exist, or is the "retired" flag then # it won't be in the properties dict if not properties.has_key(k): d[k] = v continue # get the property spec prop = properties[k] if isinstance(prop, Date) and v is not None: d[k] = date.Date(v) elif isinstance(prop, Interval) and v is not None: d[k] = date.Interval(v) elif isinstance(prop, Password): p = password.Password() p.unpack(v) d[k] = p else: d[k] = v return d def hasnode(self, classname, nodeid): ''' Determine if the database has a given node. ''' cursor = self.conn.cursor() sql = 'select count(*) from _%s where id=?'%classname if __debug__: print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) return cursor.fetchone()[0] def countnodes(self, classname): ''' Count the number of nodes that exist for a particular Class. ''' cursor = self.conn.cursor() sql = 'select count(*) from _%s'%classname if __debug__: print >>hyperdb.DEBUG, 'countnodes', (self, sql) cursor.execute(sql) return cursor.fetchone()[0] def getnodeids(self, classname, retired=0): ''' Retrieve all the ids of the nodes for a particular Class. Set retired=None to get all nodes. Otherwise it'll get all the retired or non-retired nodes, depending on the flag. ''' cursor = self.conn.cursor() # flip the sense of the flag if we don't want all of them if retired is not None: retired = not retired sql = 'select id from _%s where __retired__ <> ?'%classname if __debug__: print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired) cursor.execute(sql, (retired,)) return [x[0] for x in cursor.fetchall()] def addjournal(self, classname, nodeid, action, params): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' if isinstance(params, type({})): if params.has_key('creator'): journaltag = self.user.get(params['creator'], 'username') del params['creator'] else: journaltag = self.journaltag if params.has_key('created'): journaldate = params['created'].serialise() del params['created'] else: journaldate = date.Date().serialise() if params.has_key('activity'): del params['activity'] # serialise the parameters now if action in ('set', 'create'): params = self.serialise(classname, params) else: journaltag = self.journaltag journaldate = date.Date().serialise() # create the journal entry cols = ','.join('nodeid date tag action params'.split()) entry = (nodeid, journaldate, journaltag, action, params) if __debug__: print >>hyperdb.DEBUG, 'doSaveJournal', entry # do the insert cursor = self.conn.cursor() sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname, cols) if __debug__: print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry) cursor.execute(sql, entry) def getjournal(self, classname, nodeid): ''' get the journal for id ''' # make sure the node exists if not self.hasnode(classname, nodeid): raise IndexError, '%s has no node %s'%(classname, nodeid) # now get the journal entries cols = ','.join('nodeid date tag action params'.split()) cursor = self.conn.cursor() sql = 'select %s from %s__journal where nodeid=?'%(cols, classname) if __debug__: print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) res = [] for nodeid, date_stamp, user, action, params in cursor.fetchall(): res.append((nodeid, date.Date(date_stamp), user, action, params)) return res def pack(self, pack_before): ''' Delete all journal entries except "create" before 'pack_before'. ''' # get a 'yyyymmddhhmmss' version of the date date_stamp = pack_before.serialise() # do the delete cursor = self.conn.cursor() for classname in self.classes.keys(): sql = "delete from %s__journal where date'create'"%classname if __debug__: print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp) cursor.execute(sql, (date_stamp,)) def commit(self): ''' Commit the current transactions. Save all data changed since the database was opened or since the last commit() or rollback(). ''' if __debug__: print >>hyperdb.DEBUG, 'commit', (self,) # commit gadfly self.conn.commit() # now, do all the other transaction stuff reindex = {} for method, args in self.transactions: reindex[method(*args)] = 1 # reindex the nodes that request it for classname, nodeid in filter(None, reindex.keys()): print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid) self.getclass(classname).index(nodeid) # save the indexer state self.indexer.save_index() # clear out the transactions self.transactions = [] def rollback(self): ''' Reverse all actions from the current transaction. Undo all the changes made since the database was opened or the last commit() or rollback() was performed. ''' if __debug__: print >>hyperdb.DEBUG, 'rollback', (self,) # roll back gadfly self.conn.rollback() # roll back "other" transaction stuff for method, args in self.transactions: # delete temporary files if method == self.doStoreFile: self.rollbackStoreFile(*args) self.transactions = [] def doSaveNode(self, classname, nodeid, node): ''' dummy that just generates a reindex event ''' # return the classname, nodeid so we reindex this content return (classname, nodeid) # # The base Class class # class Class(hyperdb.Class): ''' The handle to a particular class of nodes in a hyperdatabase. All methods except __repr__ and getnode must be implemented by a concrete backend Class. ''' def __init__(self, db, classname, **properties): '''Create a new class with a given name and property specification. 'classname' must not collide with the name of an existing class, or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. ''' if (properties.has_key('creation') or properties.has_key('activity') or properties.has_key('creator')): raise ValueError, '"creation", "activity" and "creator" are '\ 'reserved' self.classname = classname self.properties = properties self.db = weakref.proxy(db) # use a weak ref to avoid circularity self.key = '' # should we journal changes (default yes) self.do_journal = 1 # do the db-related init stuff db.addclass(self) self.auditors = {'create': [], 'set': [], 'retire': []} self.reactors = {'create': [], 'set': [], 'retire': []} def schema(self): ''' A dumpable version of the schema that we can store in the database ''' return (self.key, [(x, repr(y)) for x,y in self.properties.items()]) def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # Editing nodes: def create(self, **propvalues): ''' Create a new node of this class and return its id. The keyword arguments in 'propvalues' map property names to values. The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. ''' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' self.fireAuditors('create', None, propvalues) # new node's id newid = self.db.newid(self.classname) # validate propvalues num_re = re.compile('^\d+$') for key, value in propvalues.items(): if key == self.key: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # try to handle this property try: prop = self.properties[key] except KeyError: raise KeyError, '"%s" has no property "%s"'%(self.classname, key) if value is not None and isinstance(prop, Link): if type(value) != type(''): raise ValueError, 'link value must be String' link_class = self.properties[key].classname # if it isn't a number, it's a key if not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, link_class) elif not self.db.getclass(link_class).hasnode(value): raise IndexError, '%s has no node %s'%(link_class, value) # save off the value propvalues[key] = value # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, value, 'link', (self.classname, newid, key)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of ids'%key # clean up and validate the list of links link_class = self.properties[key].classname l = [] for entry in value: if type(entry) != type(''): raise ValueError, '"%s" link value (%s) must be '\ 'String'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, self.properties[key].classname) l.append(entry) value = l propvalues[key] = value # handle additions for nodeid in value: if not self.db.getclass(link_class).hasnode(nodeid): raise IndexError, '%s has no node %s'%(link_class, nodeid) # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) elif isinstance(prop, String): if type(value) != type(''): raise TypeError, 'new property "%s" not a string'%key elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%key elif isinstance(prop, Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'%key elif isinstance(prop, Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'%key elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%key elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%key # make sure there's data where there needs to be for key, prop in self.properties.items(): if propvalues.has_key(key): continue if key == self.key: raise ValueError, 'key property "%s" is required'%key if isinstance(prop, Multilink): propvalues[key] = [] else: propvalues[key] = None # done self.db.addnode(self.classname, newid, propvalues) if self.do_journal: self.db.addjournal(self.classname, newid, 'create', propvalues) self.fireReactors('create', newid, None) return newid _marker = [] def get(self, nodeid, propname, default=_marker, cache=1): '''Get the value of a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. 'cache' indicates whether the transaction cache should be queried for the node. If the node has been modified and you need to determine what its values prior to modification are, you need to set cache=0. ''' if propname == 'id': return nodeid if propname == 'creation': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[0][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'activity': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[-1][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'creator': if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: name = self.db.getjournal(self.classname, nodeid)[0][2] else: return None try: return self.db.user.lookup(name) except KeyError: # the journaltag user doesn't exist any more return None # get the property (raises KeyErorr if invalid) prop = self.properties[propname] # get the node's dict d = self.db.getnode(self.classname, nodeid) #, cache=cache) if not d.has_key(propname): if default is self._marker: if isinstance(prop, Multilink): return [] else: return None else: return default # don't pass our list to other code if isinstance(prop, Multilink): return d[propname][:] return d[propname] def getnode(self, nodeid, cache=1): ''' Return a convenience wrapper for the node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'cache' indicates whether the transaction cache should be queried for the node. If the node has been modified and you need to determine what its values prior to modification are, you need to set cache=0. ''' return Node(self, nodeid, cache=cache) def set(self, nodeid, **propvalues): '''Modify a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. Each key in 'propvalues' must be the name of a property of this class or a KeyError is raised. All values in 'propvalues' must be acceptable types for their corresponding properties or a TypeError is raised. If the value of the key property is set, it must not collide with other key strings or a ValueError is raised. If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. ''' if not propvalues: return propvalues if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.fireAuditors('set', nodeid, propvalues) # Take a copy of the node dict so that the subsequent set # operation doesn't modify the oldvalues structure. # XXX used to try the cache here first oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) node = self.db.getnode(self.classname, nodeid) if self.is_retired(nodeid): raise IndexError num_re = re.compile('^\d+$') # if the journal value is to be different, store it in here journalvalues = {} # remember the add/remove stuff for multilinks, making it easier # for the Database layer to do its stuff multilink_changes = {} for propname, value in propvalues.items(): # check to make sure we're not duplicating an existing key if propname == self.key and node[propname] != value: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. prop = self.properties[propname] # if the value's the same as the existing value, no sense in # doing anything if node.has_key(propname) and value == node[propname]: del propvalues[propname] continue # do stuff based on the prop type if isinstance(prop, Link): link_class = prop.classname # if it isn't a number, it's a key if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( propname) if isinstance(value, type('')) and not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) if self.do_journal and prop.do_journal: # register the unlink with the old linked node if node[propname] is not None: self.db.addjournal(link_class, node[propname], 'unlink', (self.classname, nodeid, propname)) # register the link with the newly linked node if value is not None: self.db.addjournal(link_class, value, 'link', (self.classname, nodeid, propname)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of'\ ' ids'%propname link_class = self.properties[propname].classname l = [] for entry in value: # if it isn't a number, it's a key if type(entry) != type(''): raise ValueError, 'new property "%s" link value ' \ 'must be a string'%propname if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, entry, self.properties[propname].classname) l.append(entry) value = l propvalues[propname] = value # figure the journal entry for this property add = [] remove = [] # handle removals if node.has_key(propname): l = node[propname] else: l = [] for id in l[:]: if id in value: continue # register the unlink with the old linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'unlink', (self.classname, nodeid, propname)) l.remove(id) remove.append(id) # handle additions for id in value: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%(link_class, id) if id in l: continue # register the link with the newly linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'link', (self.classname, nodeid, propname)) l.append(id) add.append(id) # figure the journal entry l = [] if add: l.append(('+', add)) if remove: l.append(('-', remove)) multilink_changes[propname] = (add, remove) if l: journalvalues[propname] = tuple(l) elif isinstance(prop, String): if value is not None and type(value) != type(''): raise TypeError, 'new property "%s" not a string'%propname elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% propname propvalues[propname] = value elif value is not None and isinstance(prop, Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an '\ 'Interval'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%propname elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%propname node[propname] = value # nothing to do? if not propvalues: return propvalues # do the set, and journal it self.db.setnode(self.classname, nodeid, node, multilink_changes) if self.do_journal: propvalues.update(journalvalues) self.db.addjournal(self.classname, nodeid, 'set', propvalues) self.fireReactors('set', nodeid, oldvalues) return propvalues def retire(self, nodeid): '''Retire a node. The properties on the node remain available from the get() method, and the node's id is never reused. Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' cursor = self.db.conn.cursor() sql = 'update _%s set __retired__=1 where id=?'%self.classname if __debug__: print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) def is_retired(self, nodeid): '''Return true if the node is rerired ''' cursor = self.db.conn.cursor() sql = 'select __retired__ from _%s where id=?'%self.classname if __debug__: print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) return cursor.fetchone()[0] def destroy(self, nodeid): '''Destroy a node. WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted WARNING: use retire() instead WARNING: the properties of this node will not be available ever again WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. The node is completely removed from the hyperdb, including all journal entries. It will no longer be available, and will generally break code if there are any references to the node. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.db.destroynode(self.classname, nodeid) def history(self, nodeid): '''Retrieve the journal of edits on a particular node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. The returned list contains tuples of the form (date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.getjournal(self.classname, nodeid) # Locating nodes: def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' return self.db.hasnode(self.classname, nodeid) def setkey(self, propname): '''Select a String property of this class to be the key property. 'propname' must be the name of a String property of this class or None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. ''' # XXX create an index on the key prop column prop = self.getprops()[propname] if not isinstance(prop, String): raise TypeError, 'key properties must be String' self.key = propname def getkey(self): '''Return the name of the key property for this class or None.''' return self.key def labelprop(self, default_to_id=0): ''' Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. If this class has no key property, a TypeError is raised. If the 'keyvalue' matches one of the values for the key property among the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. ''' if not self.key: raise TypeError, 'No key property set' cursor = self.db.conn.cursor() sql = 'select id from _%s where _%s=?'%(self.classname, self.key) if __debug__: print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue) cursor.execute(sql, (keyvalue,)) # see if there was a result l = cursor.fetchall() if not l: raise KeyError, keyvalue # return the id return l[0][0] def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname={nodeid:1,} 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Used by the full text indexing, which knows that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' if __debug__: print >>hyperdb.DEBUG, 'find', (self, propspec) if not propspec: return [] queries = [] tables = [] allvalues = () for prop, values in propspec.items(): allvalues += tuple(values.keys()) tables.append('select nodeid from %s_%s where linkid in (%s)'%( self.classname, prop, ','.join(['?' for x in values.keys()]))) sql = '\nintersect\n'.join(tables) if __debug__: print >>hyperdb.DEBUG, 'find', (self, sql, allvalues) cursor = self.db.conn.cursor() cursor.execute(sql, allvalues) try: l = [x[0] for x in cursor.fetchall()] except gadfly.database.error, message: if message == 'no more results': l = [] raise if __debug__: print >>hyperdb.DEBUG, 'find ... ', l return l def list(self): ''' Return a list of the ids of the active nodes in this class. ''' return self.db.getnodeids(self.classname, retired=0) def filter(self, search_matches, filterspec, sort, group): ''' Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} ''' cn = self.classname # figure the WHERE clause from the filterspec props = self.getprops() frum = ['_'+cn] where = [] args = [] for k, v in filterspec.items(): propclass = props[k] if isinstance(propclass, Multilink): tn = '%s_%s'%(cn, k) frum.append(tn) if isinstance(v, type([])): s = ','.join(['?' for x in v]) where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s)) args = args + v else: where.append('id=%s.nodeid and %s.linkid = ?'%(tn, tn)) args.append(v) else: if isinstance(v, type([])): s = ','.join(['?' for x in v]) where.append('_%s in (%s)'%(k, s)) args = args + v else: where.append('_%s=?'%k) args.append(v) # add results of full text search if search_matches is not None: v = search_matches.keys() s = ','.join(['?' for x in v]) where.append('id in (%s)'%s) args = args + v # figure the order by clause orderby = [] ordercols = [] if sort is not None: if sort[0] != '-': orderby.append('_'+sort[1]) ordercols.append(sort[1]) else: orderby.append('_'+sort[1]+' desc') ordercols.append(sort[1]) # figure the group by clause groupby = [] groupcols = [] if group is not None: if group[0] != '-': groupby.append('_'+group[1]) groupcols.append(group[1]) else: groupby.append('_'+group[1]+' desc') groupcols.append(group[1]) # construct the SQL frum = ','.join(frum) where = ' and '.join(where) cols = ['id'] if orderby: cols = cols + ordercols order = ' order by %s'%(','.join(orderby)) else: order = '' if groupby: cols = cols + groupcols group = ' group by %s'%(','.join(groupby)) else: group = '' cols = ','.join(cols) sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order, group) args = tuple(args) if __debug__: print >>hyperdb.DEBUG, 'find', (self, sql, args) cursor = self.db.conn.cursor() cursor.execute(sql, args) def count(self): '''Get the number of nodes in this class. If the returned integer is 'numnodes', the ids of all the nodes in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. ''' return self.db.countnodes(self.classname) # Manipulating properties: def getprops(self, protected=1): '''Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = self.properties.copy() if protected: d['id'] = String() d['creation'] = hyperdb.Date() d['activity'] = hyperdb.Date() d['creator'] = hyperdb.Link("user") return d def addprop(self, **properties): '''Add properties to this class. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. None of the keys in 'properties' may collide with the names of existing properties, or a ValueError is raised before any properties have been added. ''' for key in properties.keys(): if self.properties.has_key(key): raise ValueError, key self.properties.update(properties) def index(self, nodeid): '''Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, String) and propclass.indexme: try: value = str(self.get(nodeid, prop)) except IndexError: # node no longer exists - entry should be removed self.db.indexer.purge_entry((self.classname, nodeid, prop)) else: # and index them under (classname, nodeid, property) self.db.indexer.add_text((self.classname, nodeid, prop), value) # # Detector interface # def audit(self, event, detector): '''Register a detector ''' l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): '''Fire all registered auditors. ''' for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def react(self, event, detector): '''Register a detector ''' l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) def fireReactors(self, action, nodeid, oldvalues): '''Fire all registered reactors. ''' for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) class FileClass(Class): '''This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. The default MIME type of this data is defined by the "default_mime_type" class attribute, which may be overridden by each node if the class defines a "type" String property. ''' default_mime_type = 'text/plain' def create(self, **propvalues): ''' snaffle the file propvalue and store in a file ''' content = propvalues['content'] del propvalues['content'] newid = Class.create(self, **propvalues) self.db.storefile(self.classname, newid, None, content) return newid def import_list(self, propnames, proplist): ''' Trap the "content" property... ''' # dupe this list so we don't affect others propnames = propnames[:] # extract the "content" property from the proplist i = propnames.index('content') content = proplist[i] del propnames[i] del proplist[i] # do the normal import newid = Class.import_list(self, propnames, proplist) # save off the "content" file self.db.storefile(self.classname, newid, None, content) return newid _marker = [] def get(self, nodeid, propname, default=_marker, cache=1): ''' trap the content propname and get it from the file ''' poss_msg = 'Possibly a access right configuration problem.' if propname == 'content': try: return self.db.getfile(self.classname, nodeid, None) except IOError, (strerror): # BUG: by catching this we donot see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) if default is not self._marker: return Class.get(self, nodeid, propname, default, cache=cache) else: return Class.get(self, nodeid, propname, cache=cache) def getprops(self, protected=1): ''' In addition to the actual properties on the node, these methods provide the "content" property. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = Class.getprops(self, protected=protected).copy() if protected: d['content'] = hyperdb.String() return d def index(self, nodeid): ''' Index the node in the search index. We want to index the content in addition to the normal String property indexing. ''' # perform normal indexing Class.index(self, nodeid) # get the content to index content = self.get(nodeid, 'content') # figure the mime type if self.properties.has_key('type'): mime_type = self.get(nodeid, 'type') else: mime_type = self.default_mime_type # and index! self.db.indexer.add_text((self.classname, nodeid, 'content'), content, mime_type) # XXX deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): '''The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation" or "activity" property, a ValueError is raised. ''' if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) # # $Log: not supported by cvs2svn $ # Revision 1.9 2002/09/03 05:46:21 richard # handle disappearing users for journaltags # # Revision 1.8 2002/09/03 02:53:53 richard # Fixed nasty bug that was preventing changes to multilinks going through. # # Revision 1.7 2002/09/01 04:32:30 richard # . Lots of cleanup in the classic html (stylesheet, search page, index page, ...) # . Reinstated searching, but not query saving yet # . Filtering only allows sorting and grouping by one property - all backends # now implement this behaviour. # . Nosy list journalling turned off by default, everything else is on. # . Added some convenience methods (reverse, propchanged, [item] accesses, ...) # . Did I mention the stylesheet is much cleaner now? :) # # Revision 1.6 2002/08/30 08:35:16 richard # very basic filter support # # Revision 1.5 2002/08/23 05:33:32 richard # implemented multilink changes (and a unit test) # # Revision 1.4 2002/08/23 05:00:38 richard # fixed read-only gadfly retire() # # Revision 1.3 2002/08/23 04:58:00 richard # ahhh, I understand now # # Revision 1.2 2002/08/23 04:48:10 richard # That's gadfly done, mostly. Things left: # - Class.filter (I'm a wuss ;) # - schema changes adding new non-multilink properties are not implemented. # gadfly doesn't have an ALTER TABLE command, making that quite difficult :) # # I had to mangle two unit tests to get this all working: # - gadfly also can't handle two handles open on the one database, so # testIDGeneration doesn't try that. # - testNewProperty is disabled as per the second comment above. # # I noticed test_pack was incorrect, and the *dbm tests fail there now. # Looking into it... # # Revision 1.1 2002/08/22 07:56:51 richard # Whee! It's not finished yet, but I can create a new instance and play with # it a little bit :) # # Revision 1.80 2002/08/16 04:28:13 richard # added is_retired query to Class # # Revision 1.79 2002/07/29 23:30:14 richard # documentation reorg post-new-security # # Revision 1.78 2002/07/21 03:26:37 richard # Gordon, does this help? # # Revision 1.77 2002/07/18 11:27:47 richard # ws # # Revision 1.76 2002/07/18 11:17:30 gmcm # Add Number and Boolean types to hyperdb. # Add conversion cases to web, mail & admin interfaces. # Add storage/serialization cases to back_anydbm & back_metakit. # # Revision 1.75 2002/07/14 02:05:53 richard # . all storage-specific code (ie. backend) is now implemented by the backends # # Revision 1.74 2002/07/10 00:24:10 richard # braino # # Revision 1.73 2002/07/10 00:19:48 richard # Added explicit closing of backend database handles. # # Revision 1.72 2002/07/09 21:53:38 gmcm # Optimize Class.find so that the propspec can contain a set of ids to match. # This is used by indexer.search so it can do just one find for all the index matches. # This was already confusing code, but for common terms (lots of index matches), # it is enormously faster. # # Revision 1.71 2002/07/09 03:02:52 richard # More indexer work: # - all String properties may now be indexed too. Currently there's a bit of # "issue" specific code in the actual searching which needs to be # addressed. In a nutshell: # + pass 'indexme="yes"' as a String() property initialisation arg, eg: # file = FileClass(db, "file", name=String(), type=String(), # comment=String(indexme="yes")) # + the comment will then be indexed and be searchable, with the results # related back to the issue that the file is linked to # - as a result of this work, the FileClass has a default MIME type that may # be overridden in a subclass, or by the use of a "type" property as is # done in the default templates. # - the regeneration of the indexes (if necessary) is done once the schema is # set up in the dbinit. # # Revision 1.70 2002/06/27 12:06:20 gmcm # Improve an error message. # # Revision 1.69 2002/06/17 23:15:29 richard # Can debug to stdout now # # Revision 1.68 2002/06/11 06:52:03 richard # . #564271 ] find() and new properties # # Revision 1.67 2002/06/11 05:02:37 richard # . #565979 ] code error in hyperdb.Class.find # # Revision 1.66 2002/05/25 07:16:24 rochecompaan # Merged search_indexing-branch with HEAD # # Revision 1.65 2002/05/22 04:12:05 richard # . applied patch #558876 ] cgi client customization # ... with significant additions and modifications ;) # - extended handling of ML assignedto to all places it's handled # - added more NotFound info # # Revision 1.64 2002/05/15 06:21:21 richard # . node caching now works, and gives a small boost in performance # # As a part of this, I cleaned up the DEBUG output and implemented TRACE # output (HYPERDBTRACE='file to trace to') with checkpoints at the start of # CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff # (using if __debug__ which is compiled out with -O) # # Revision 1.63 2002/04/15 23:25:15 richard # . node ids are now generated from a lockable store - no more race conditions # # We're using the portalocker code by Jonathan Feinberg that was contributed # to the ASPN Python cookbook. This gives us locking across Unix and Windows. # # Revision 1.62 2002/04/03 07:05:50 richard # d'oh! killed retirement of nodes :( # all better now... # # Revision 1.61 2002/04/03 06:11:51 richard # Fix for old databases that contain properties that don't exist any more. # # Revision 1.60 2002/04/03 05:54:31 richard # Fixed serialisation problem by moving the serialisation step out of the # hyperdb.Class (get, set) into the hyperdb.Database. # # Also fixed htmltemplate after the showid changes I made yesterday. # # Unit tests for all of the above written. # # Revision 1.59.2.2 2002/04/20 13:23:33 rochecompaan # We now have a separate search page for nodes. Search links for # different classes can be customized in instance_config similar to # index links. # # Revision 1.59.2.1 2002/04/19 19:54:42 rochecompaan # cgi_client.py # removed search link for the time being # moved rendering of matches to htmltemplate # hyperdb.py # filtering of nodes on full text search incorporated in filter method # roundupdb.py # added paramater to call of filter method # roundup_indexer.py # added search method to RoundupIndexer class # # Revision 1.59 2002/03/12 22:52:26 richard # more pychecker warnings removed # # Revision 1.58 2002/02/27 03:23:16 richard # Ran it through pychecker, made fixes # # Revision 1.57 2002/02/20 05:23:24 richard # Didn't accomodate new values for new properties # # Revision 1.56 2002/02/20 05:05:28 richard # . Added simple editing for classes that don't define a templated interface. # - access using the admin "class list" interface # - limited to admin-only # - requires the csv module from object-craft (url given if it's missing) # # Revision 1.55 2002/02/15 07:27:12 richard # Oops, precedences around the way w0rng. # # Revision 1.54 2002/02/15 07:08:44 richard # . Alternate email addresses are now available for users. See the MIGRATION # file for info on how to activate the feature. # # Revision 1.53 2002/01/22 07:21:13 richard # . fixed back_bsddb so it passed the journal tests # # ... it didn't seem happy using the back_anydbm _open method, which is odd. # Yet another occurrance of whichdb not being able to recognise older bsddb # databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the # process. # # Revision 1.52 2002/01/21 16:33:19 rochecompaan # You can now use the roundup-admin tool to pack the database # # Revision 1.51 2002/01/21 03:01:29 richard # brief docco on the do_journal argument # # Revision 1.50 2002/01/19 13:16:04 rochecompaan # Journal entries for link and multilink properties can now be switched on # or off. # # Revision 1.49 2002/01/16 07:02:57 richard # . lots of date/interval related changes: # - more relaxed date format for input # # Revision 1.48 2002/01/14 06:32:34 richard # . #502951 ] adding new properties to old database # # Revision 1.47 2002/01/14 02:20:15 richard # . changed all config accesses so they access either the instance or the # config attriubute on the db. This means that all config is obtained from # instance_config instead of the mish-mash of classes. This will make # switching to a ConfigParser setup easier too, I hope. # # At a minimum, this makes migration a _little_ easier (a lot easier in the # 0.5.0 switch, I hope!) # # Revision 1.46 2002/01/07 10:42:23 richard # oops # # Revision 1.45 2002/01/02 04:18:17 richard # hyperdb docstrings # # Revision 1.44 2002/01/02 02:31:38 richard # Sorry for the huge checkin message - I was only intending to implement #496356 # but I found a number of places where things had been broken by transactions: # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename # for _all_ roundup-generated smtp messages to be sent to. # . the transaction cache had broken the roundupdb.Class set() reactors # . newly-created author users in the mailgw weren't being committed to the db # # Stuff that made it into CHANGES.txt (ie. the stuff I was actually working # on when I found that stuff :): # . #496356 ] Use threading in messages # . detectors were being registered multiple times # . added tests for mailgw # . much better attaching of erroneous messages in the mail gateway # # Revision 1.43 2001/12/20 06:13:24 rochecompaan # Bugs fixed: # . Exception handling in hyperdb for strings-that-look-like numbers got # lost somewhere # . Internet Explorer submits full path for filename - we now strip away # the path # Features added: # . Link and multilink properties are now displayed sorted in the cgi # interface # # Revision 1.42 2001/12/16 10:53:37 richard # take a copy of the node dict so that the subsequent set # operation doesn't modify the oldvalues structure # # Revision 1.41 2001/12/15 23:47:47 richard # Cleaned up some bare except statements # # Revision 1.40 2001/12/14 23:42:57 richard # yuck, a gdbm instance tests false :( # I've left the debugging code in - it should be removed one day if we're ever # _really_ anal about performace :) # # Revision 1.39 2001/12/02 05:06:16 richard # . We now use weakrefs in the Classes to keep the database reference, so # the close() method on the database is no longer needed. # I bumped the minimum python requirement up to 2.1 accordingly. # . #487480 ] roundup-server # . #487476 ] INSTALL.txt # # I also cleaned up the change message / post-edit stuff in the cgi client. # There's now a clearly marked "TODO: append the change note" where I believe # the change note should be added there. The "changes" list will obviously # have to be modified to be a dict of the changes, or somesuch. # # More testing needed. # # Revision 1.38 2001/12/01 07:17:50 richard # . We now have basic transaction support! Information is only written to # the database when the commit() method is called. Only the anydbm # backend is modified in this way - neither of the bsddb backends have been. # The mail, admin and cgi interfaces all use commit (except the admin tool # doesn't have a commit command, so interactive users can't commit...) # . Fixed login/registration forwarding the user to the right page (or not, # on a failure) # # Revision 1.37 2001/11/28 21:55:35 richard # . login_action and newuser_action return values were being ignored # . Woohoo! Found that bloody re-login bug that was killing the mail # gateway. # (also a minor cleanup in hyperdb) # # Revision 1.36 2001/11/27 03:16:09 richard # Another place that wasn't handling missing properties. # # Revision 1.35 2001/11/22 15:46:42 jhermann # Added module docstrings to all modules. # # Revision 1.34 2001/11/21 04:04:43 richard # *sigh* more missing value handling # # Revision 1.33 2001/11/21 03:40:54 richard # more new property handling # # Revision 1.32 2001/11/21 03:11:28 richard # Better handling of new properties. # # Revision 1.31 2001/11/12 22:01:06 richard # Fixed issues with nosy reaction and author copies. # # Revision 1.30 2001/11/09 10:11:08 richard # . roundup-admin now handles all hyperdb exceptions # # Revision 1.29 2001/10/27 00:17:41 richard # Made Class.stringFind() do caseless matching. # # Revision 1.28 2001/10/21 04:44:50 richard # bug #473124: UI inconsistency with Link fields. # This also prompted me to fix a fairly long-standing usability issue - # that of being able to turn off certain filters. # # Revision 1.27 2001/10/20 23:44:27 richard # Hyperdatabase sorts strings-that-look-like-numbers as numbers now. # # Revision 1.26 2001/10/16 03:48:01 richard # admin tool now complains if a "find" is attempted with a non-link property. # # Revision 1.25 2001/10/11 00:17:51 richard # Reverted a change in hyperdb so the default value for missing property # values in a create() is None and not '' (the empty string.) This obviously # breaks CSV import/export - the string 'None' will be created in an # export/import operation. # # Revision 1.24 2001/10/10 03:54:57 richard # Added database importing and exporting through CSV files. # Uses the csv module from object-craft for exporting if it's available. # Requires the csv module for importing. # # Revision 1.23 2001/10/09 23:58:10 richard # Moved the data stringification up into the hyperdb.Class class' get, set # and create methods. This means that the data is also stringified for the # journal call, and removes duplication of code from the backends. The # backend code now only sees strings. # # Revision 1.22 2001/10/09 07:25:59 richard # Added the Password property type. See "pydoc roundup.password" for # implementation details. Have updated some of the documentation too. # # Revision 1.21 2001/10/05 02:23:24 richard # . roundup-admin create now prompts for property info if none is supplied # on the command-line. # . hyperdb Class getprops() method may now return only the mutable # properties. # . Login now uses cookies, which makes it a whole lot more flexible. We can # now support anonymous user access (read-only, unless there's an # "anonymous" user, in which case write access is permitted). Login # handling has been moved into cgi_client.Client.main() # . The "extended" schema is now the default in roundup init. # . The schemas have had their page headings modified to cope with the new # login handling. Existing installations should copy the interfaces.py # file from the roundup lib directory to their instance home. # . Incorrectly had a Bizar Software copyright on the cgitb.py module from # Ping - has been removed. # . Fixed a whole bunch of places in the CGI interface where we should have # been returning Not Found instead of throwing an exception. # . Fixed a deviation from the spec: trying to modify the 'id' property of # an item now throws an exception. # # Revision 1.20 2001/10/04 02:12:42 richard # Added nicer command-line item adding: passing no arguments will enter an # interactive more which asks for each property in turn. While I was at it, I # fixed an implementation problem WRT the spec - I wasn't raising a # ValueError if the key property was missing from a create(). Also added a # protected=boolean argument to getprops() so we can list only the mutable # properties (defaults to yes, which lists the immutables). # # Revision 1.19 2001/08/29 04:47:18 richard # Fixed CGI client change messages so they actually include the properties # changed (again). # # Revision 1.18 2001/08/16 07:34:59 richard # better CGI text searching - but hidden filter fields are disappearing... # # Revision 1.17 2001/08/16 06:59:58 richard # all searches use re now - and they're all case insensitive # # Revision 1.16 2001/08/15 23:43:18 richard # Fixed some isFooTypes that I missed. # Refactored some code in the CGI code. # # Revision 1.15 2001/08/12 06:32:36 richard # using isinstance(blah, Foo) now instead of isFooType # # Revision 1.14 2001/08/07 00:24:42 richard # stupid typo # # Revision 1.13 2001/08/07 00:15:51 richard # Added the copyright/license notice to (nearly) all files at request of # Bizar Software. # # Revision 1.12 2001/08/02 06:38:17 richard # Roundupdb now appends "mailing list" information to its messages which # include the e-mail address and web interface address. Templates may # override this in their db classes to include specific information (support # instructions, etc). # # Revision 1.11 2001/08/01 04:24:21 richard # mailgw was assuming certain properties existed on the issues being created. # # Revision 1.10 2001/07/30 02:38:31 richard # get() now has a default arg - for migration only. # # Revision 1.9 2001/07/29 09:28:23 richard # Fixed sorting by clicking on column headings. # # Revision 1.8 2001/07/29 08:27:40 richard # Fixed handling of passed-in values in form elements (ie. during a # drill-down) # # Revision 1.7 2001/07/29 07:01:39 richard # Added vim command to all source so that we don't get no steenkin' tabs :) # # Revision 1.6 2001/07/29 05:36:14 richard # Cleanup of the link label generation. # # Revision 1.5 2001/07/29 04:05:37 richard # Added the fabricated property "id". # # Revision 1.4 2001/07/27 06:25:35 richard # Fixed some of the exceptions so they're the right type. # Removed the str()-ification of node ids so we don't mask oopsy errors any # more. # # Revision 1.3 2001/07/27 05:17:14 richard # just some comments # # Revision 1.2 2001/07/22 12:09:32 richard # Final commit of Grande Splite # # Revision 1.1 2001/07/22 11:58:35 richard # More Grande Splite # # # vim: set filetype=python ts=4 sw=4 et si code-c3e391d9c4e9/roundup/backends/back_metakit.py0000755000000000000000000012227007535063756020232 0ustar 00000000000000from roundup import hyperdb, date, password, roundupdb, security import metakit from sessions import Sessions import re, marshal, os, sys, weakref, time, calendar from roundup import indexer import locking _dbs = {} def Database(config, journaltag=None): db = _dbs.get(config.DATABASE, None) if db is None or db._db is None: db = _Database(config, journaltag) _dbs[config.DATABASE] = db else: db.journaltag = journaltag try: delattr(db, 'curuserid') except AttributeError: pass return db class _Database(hyperdb.Database): def __init__(self, config, journaltag=None): self.config = config self.journaltag = journaltag self.classes = {} self.dirty = 0 self.lockfile = None self._db = self.__open() self.indexer = Indexer(self.config.DATABASE, self._db) self.sessions = Sessions(self.config) self.security = security.Security(self) os.umask(0002) def post_init(self): if self.indexer.should_reindex(): self.reindex() def reindex(self): for klass in self.classes.values(): for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() # --- defined in ping's spec def __getattr__(self, classname): if classname == 'curuserid': try: self.curuserid = x = int(self.classes['user'].lookup(self.journaltag)) except KeyError: x = 0 return x elif classname == 'transactions': return self.dirty return self.getclass(classname) def getclass(self, classname): return self.classes[classname] def getclasses(self): return self.classes.keys() # --- end of ping's spec # --- exposed methods def commit(self): if self.dirty: self._db.commit() for cl in self.classes.values(): cl._commit() self.indexer.save_index() self.dirty = 0 def rollback(self): if self.dirty: for cl in self.classes.values(): cl._rollback() self._db.rollback() self.dirty = 0 def clear(self): for cl in self.classes.values(): cl._clear() def hasnode(self, classname, nodeid): return self.getclass(classname).hasnode(nodeid) def pack(self, pack_before): pass def addclass(self, cl): self.classes[cl.classname] = cl if self.tables.find(name=cl.classname) < 0: self.tables.append(name=cl.classname) def addjournal(self, tablenm, nodeid, action, params): tblid = self.tables.find(name=tablenm) if tblid == -1: tblid = self.tables.append(name=tablenm) # tableid:I,nodeid:I,date:I,user:I,action:I,params:B self.hist.append(tableid=tblid, nodeid=int(nodeid), date=int(time.time()), action=action, user = self.curuserid, params = marshal.dumps(params)) def gethistory(self, tablenm, nodeid): rslt = [] tblid = self.tables.find(name=tablenm) if tblid == -1: return rslt q = self.hist.select(tableid=tblid, nodeid=int(nodeid)) i = 0 userclass = self.getclass('user') for row in q: try: params = marshal.loads(row.params) except ValueError: print "history couldn't unmarshal %r" % row.params params = {} usernm = userclass.get(str(row.user), 'username') dt = date.Date(time.gmtime(row.date)) rslt.append((i, dt, usernm, _actionnames[row.action], params)) i += 1 return rslt def close(self): for cl in self.classes.values(): cl.db = None self._db = None locking.release_lock(self.lockfile) del _dbs[self.config.DATABASE] self.lockfile.close() self.classes = {} self.indexer = None # --- internal def __open(self): self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') lockfilenm = db[:-3]+'lck' self.lockfile = locking.acquire_lock(lockfilenm) self.lockfile.write(str(os.getpid())) self.lockfile.flush() self.fastopen = 0 if os.path.exists(db): dbtm = os.path.getmtime(db) pkgnm = self.config.__name__.split('.')[0] schemamod = sys.modules.get(pkgnm+'.dbinit', None) if schemamod: if os.path.exists(schemamod.__file__): schematm = os.path.getmtime(schemamod.__file__) if schematm < dbtm: # found schema mod - it's older than the db self.fastopen = 1 else: # can't find schemamod - must be frozen self.fastopen = 1 db = metakit.storage(db, 1) hist = db.view('history') tables = db.view('tables') if not self.fastopen: if not hist.structure(): hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]') if not tables.structure(): tables = db.getas('tables[name:S]') self.tables = tables self.hist = hist return db _STRINGTYPE = type('') _LISTTYPE = type([]) _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5) _actionnames = { _CREATE : 'create', _SET : 'set', _RETIRE : 'retire', _LINK : 'link', _UNLINK : 'unlink', } _marker = [] _ALLOWSETTINGPRIVATEPROPS = 0 class Class: privateprops = None def __init__(self, db, classname, **properties): #self.db = weakref.proxy(db) self.db = db self.classname = classname self.keyname = None self.ruprops = properties self.privateprops = { 'id' : hyperdb.String(), 'activity' : hyperdb.Date(), 'creation' : hyperdb.Date(), 'creator' : hyperdb.Link('user') } self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables self.reactors = {'create': [], 'set': [], 'retire': []} # ditto view = self.__getview() self.maxid = 1 if view: self.maxid = view[-1].id + 1 self.uncommitted = {} self.rbactions = [] # people reach inside!! self.properties = self.ruprops self.db.addclass(self) self.idcache = {} # default is to journal changes self.do_journal = 1 def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # --- the roundup.Class methods def audit(self, event, detector): l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def fireReactors(self, action, nodeid, oldvalues): for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) def react(self, event, detector): l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) # --- the hyperdb.Class methods def create(self, **propvalues): self.fireAuditors('create', None, propvalues) rowdict = {} rowdict['id'] = newid = self.maxid self.maxid += 1 ndx = self.getview(1).append(rowdict) propvalues['#ISNEW'] = 1 try: self.set(str(newid), **propvalues) except Exception: self.maxid -= 1 raise return str(newid) def get(self, nodeid, propname, default=_marker, cache=1): # default and cache aren't in the spec # cache=0 means "original value" view = self.getview() id = int(nodeid) if cache == 0: oldnode = self.uncommitted.get(id, None) if oldnode and oldnode.has_key(propname): return oldnode[propname] ndx = self.idcache.get(id, None) if ndx is None: ndx = view.find(id=id) if ndx < 0: raise IndexError, "%s has no node %s" % (self.classname, nodeid) self.idcache[id] = ndx try: raw = getattr(view[ndx], propname) except AttributeError: raise KeyError, propname rutyp = self.ruprops.get(propname, None) if rutyp is None: rutyp = self.privateprops[propname] converter = _converters.get(rutyp.__class__, None) if converter: raw = converter(raw) return raw def set(self, nodeid, **propvalues): isnew = 0 if propvalues.has_key('#ISNEW'): isnew = 1 del propvalues['#ISNEW'] if not isnew: self.fireAuditors('set', nodeid, propvalues) if not propvalues: return propvalues if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' view = self.getview(1) # node must exist & not be retired id = int(nodeid) ndx = view.find(id=id) if ndx < 0: raise IndexError, "%s has no node %s" % (self.classname, nodeid) row = view[ndx] if row._isdel: raise IndexError, "%s has no node %s" % (self.classname, nodeid) oldnode = self.uncommitted.setdefault(id, {}) changes = {} for key, value in propvalues.items(): # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. if _ALLOWSETTINGPRIVATEPROPS: prop = self.ruprops.get(key, None) if not prop: prop = self.privateprops[key] else: prop = self.ruprops[key] converter = _converters.get(prop.__class__, lambda v: v) # if the value's the same as the existing value, no sense in # doing anything oldvalue = converter(getattr(row, key)) if value == oldvalue: del propvalues[key] continue # check to make sure we're not duplicating an existing key if key == self.keyname: iv = self.getindexview(1) ndx = iv.find(k=value) if ndx == -1: iv.append(k=value, i=row.id) if not isnew: ndx = iv.find(k=oldvalue) if ndx > -1: iv.delete(ndx) else: raise ValueError, 'node with key "%s" exists'%value # do stuff based on the prop type if isinstance(prop, hyperdb.Link): link_class = prop.classname # must be a string or None if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( propname) # Roundup sets to "unselected" by passing None if value is None: value = 0 # if it isn't a number, it's a key try: int(value) except ValueError: try: value = self.db.getclass(link_class).lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) setattr(row, key, int(value)) changes[key] = oldvalue if self.do_journal and prop.do_journal: # register the unlink with the old linked node if oldvalue: self.db.addjournal(link_class, value, _UNLINK, (self.classname, str(row.id), key)) # register the link with the newly linked node if value: self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key)) elif isinstance(prop, hyperdb.Multilink): if type(value) != _LISTTYPE: raise TypeError, 'new property "%s" not a list of ids'%key link_class = prop.classname l = [] for entry in value: if type(entry) != _STRINGTYPE: raise ValueError, 'new property "%s" link value ' \ 'must be a string'%key # if it isn't a number, it's a key try: int(entry) except ValueError: try: entry = self.db.getclass(link_class).lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, prop.classname) l.append(entry) propvalues[key] = value = l # handle removals rmvd = [] for id in oldvalue: if id not in value: rmvd.append(id) # register the unlink with the old linked node if self.do_journal and prop.do_journal: self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key)) # handle additions adds = [] for id in value: if id not in oldvalue: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%( link_class, id) adds.append(id) # register the link with the newly linked node if self.do_journal and prop.do_journal: self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key)) sv = getattr(row, key) i = 0 while i < len(sv): if str(sv[i].fid) in rmvd: sv.delete(i) else: i += 1 for id in adds: sv.append(fid=int(id)) changes[key] = oldvalue if not rmvd and not adds: del propvalues[key] elif isinstance(prop, hyperdb.String): if value is not None and type(value) != _STRINGTYPE: raise TypeError, 'new property "%s" not a string'%key setattr(row, key, value) changes[key] = oldvalue if hasattr(prop, 'isfilename') and prop.isfilename: propvalues[key] = os.path.basename(value) if prop.indexme: self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain') elif isinstance(prop, hyperdb.Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'% key setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) elif value is not None and isinstance(prop, hyperdb.Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% key setattr(row, key, int(calendar.timegm(value.get_tuple()))) changes[key] = str(oldvalue) propvalues[key] = str(value) elif value is not None and isinstance(prop, hyperdb.Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'% key setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) elif value is not None and isinstance(prop, hyperdb.Number): setattr(row, key, int(value)) changes[key] = oldvalue propvalues[key] = value elif value is not None and isinstance(prop, hyperdb.Boolean): bv = value != 0 setattr(row, key, bv) changes[key] = oldvalue propvalues[key] = value oldnode[key] = oldvalue # nothing to do? if not propvalues: return propvalues if not propvalues.has_key('activity'): row.activity = int(time.time()) if isnew: if not row.creation: row.creation = int(time.time()) if not row.creator: row.creator = self.db.curuserid self.db.dirty = 1 if self.do_journal: if isnew: self.db.addjournal(self.classname, nodeid, _CREATE, {}) self.fireReactors('create', nodeid, None) else: self.db.addjournal(self.classname, nodeid, _SET, changes) self.fireReactors('set', nodeid, oldnode) return propvalues def retire(self, nodeid): self.fireAuditors('retire', nodeid, None) view = self.getview(1) ndx = view.find(id=int(nodeid)) if ndx < 0: raise KeyError, "nodeid %s not found" % nodeid row = view[ndx] oldvalues = self.uncommitted.setdefault(row.id, {}) oldval = oldvalues['_isdel'] = row._isdel row._isdel = 1 if self.do_journal: self.db.addjournal(self.classname, nodeid, _RETIRE, {}) if self.keyname: iv = self.getindexview(1) ndx = iv.find(k=getattr(row, self.keyname),i=row.id) if ndx > -1: iv.delete(ndx) self.db.dirty = 1 self.fireReactors('retire', nodeid, None) def history(self, nodeid): if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.gethistory(self.classname, nodeid) def setkey(self, propname): if self.keyname: if propname == self.keyname: return raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname) # first setkey for this run self.keyname = propname iv = self.db._db.view('_%s' % self.classname) if self.db.fastopen and iv.structure(): return # very first setkey ever self.db.dirty = 1 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname) iv = iv.ordered(1) # print "setkey building index" for row in self.getview(): iv.append(k=getattr(row, propname), i=row.id) self.db.commit() def getkey(self): return self.keyname def lookup(self, keyvalue): if type(keyvalue) is not _STRINGTYPE: raise TypeError, "%r is not a string" % keyvalue iv = self.getindexview() if iv: ndx = iv.find(k=keyvalue) if ndx > -1: return str(iv[ndx].i) else: view = self.getview() ndx = view.find({self.keyname:keyvalue, '_isdel':0}) if ndx > -1: return str(view[ndx].id) raise KeyError, keyvalue def destroy(self, keyvalue): #TODO clean this up once Richard's said how it should work iv = self.getindexview() if iv: ndx = iv.find(k=keyvalue) if ndx > -1: id = iv[ndx].i iv.delete(ndx) view = self.getview() ndx = view.find(id=id) if ndx > -1: view.delete(ndx) def find(self, **propspec): """Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname={nodeid:1,} 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose propname property links to any of the nodeids will be returned. Used by the full text indexing, which knows that "foo" occurs in msg1, msg3 and file7; so we have hits on these issues: db.issue.find(messages={'1':1,'3':1}, files={'7':1}) """ propspec = propspec.items() for propname, nodeid in propspec: # check the prop is OK prop = self.ruprops[propname] if (not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink)): raise TypeError, "'%s' not a Link/Multilink property"%propname vws = [] for propname, ids in propspec: if type(ids) is _STRINGTYPE: ids = {ids:1} prop = self.ruprops[propname] view = self.getview() if isinstance(prop, hyperdb.Multilink): view = view.flatten(getattr(view, propname)) def ff(row, nm=propname, ids=ids): return ids.has_key(str(row.fid)) else: def ff(row, nm=propname, ids=ids): return ids.has_key(str(getattr(row, nm))) ndxview = view.filter(ff) vws.append(ndxview.unique()) # handle the empty match case if not vws: return [] ndxview = vws[0] for v in vws[1:]: ndxview = ndxview.union(v) view = view.remapwith(ndxview) rslt = [] for row in view: rslt.append(str(row.id)) return rslt def list(self): l = [] for row in self.getview().select(_isdel=0): l.append(str(row.id)) return l def count(self): return len(self.getview()) def getprops(self, protected=1): # protected is not in ping's spec allprops = self.ruprops.copy() if protected and self.privateprops is not None: allprops.update(self.privateprops) return allprops def addprop(self, **properties): for key in properties.keys(): if self.ruprops.has_key(key): raise ValueError, "%s is already a property of %s" % (key, self.classname) self.ruprops.update(properties) self.db.fastopen = 0 view = self.__getview() self.db.commit() # ---- end of ping's spec def filter(self, search_matches, filterspec, sort, group): # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}}) # filterspec is a dict {propname:value} # sort and group are lists of propnames # sort and group are (dir, prop) where dir is '+', '-' or None # and prop is a prop name or None where = {'_isdel':0} mlcriteria = {} regexes = {} orcriteria = {} for propname, value in filterspec.items(): prop = self.ruprops.get(propname, None) if prop is None: prop = self.privateprops[propname] if isinstance(prop, hyperdb.Multilink): if type(value) is not _LISTTYPE: value = [value] # transform keys to ids u = [] for item in value: try: item = int(item) except (TypeError, ValueError): item = int(self.db.getclass(prop.classname).lookup(item)) if item == -1: item = 0 u.append(item) mlcriteria[propname] = u elif isinstance(prop, hyperdb.Link): if type(value) is not _LISTTYPE: value = [value] # transform keys to ids u = [] for item in value: try: item = int(item) except (TypeError, ValueError): item = int(self.db.getclass(prop.classname).lookup(item)) if item == -1: item = 0 u.append(item) if len(u) == 1: where[propname] = u[0] else: orcriteria[propname] = u elif isinstance(prop, hyperdb.String): # simple glob searching v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value) v = v.replace('?', '.') v = v.replace('*', '.*?') regexes[propname] = re.compile(v, re.I) elif propname == 'id': where[propname] = int(value) elif isinstance(prop, hyperdb.Boolean): if type(value) is _STRINGTYPE: bv = value.lower() in ('yes', 'true', 'on', '1') else: bv = value where[propname] = bv elif isinstance(prop, hyperdb.Number): where[propname] = int(value) else: where[propname] = str(value) v = self.getview() #print "filter start at %s" % time.time() if where: v = v.select(where) #print "filter where at %s" % time.time() if mlcriteria: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip # it def ff(row, ml=mlcriteria): for propname, values in ml.items(): sv = getattr(row, propname) for id in values: if sv.find(fid=id) == -1: return 0 return 1 iv = v.filter(ff) v = v.remapwith(iv) #print "filter mlcrit at %s" % time.time() if orcriteria: def ff(row, crit=orcriteria): for propname, allowed in crit.items(): val = getattr(row, propname) if val not in allowed: return 0 return 1 iv = v.filter(ff) v = v.remapwith(iv) #print "filter orcrit at %s" % time.time() if regexes: def ff(row, r=regexes): for propname, regex in r.items(): val = getattr(row, propname) if not regex.search(val): return 0 return 1 iv = v.filter(ff) v = v.remapwith(iv) #print "filter regexs at %s" % time.time() if sort or group: sortspec = [] rev = [] for dir, propname in group, sort: if propname is None: continue isreversed = 0 if dir == '-': isreversed = 1 try: prop = getattr(v, propname) except AttributeError: print "MK has no property %s" % propname continue propclass = self.ruprops.get(propname, None) if propclass is None: propclass = self.privateprops.get(propname, None) if propclass is None: print "Schema has no property %s" % propname continue if isinstance(propclass, hyperdb.Link): linkclass = self.db.getclass(propclass.classname) lv = linkclass.getview() lv = lv.rename('id', propname) v = v.join(lv, prop, 1) if linkclass.getprops().has_key('order'): propname = 'order' else: propname = linkclass.labelprop() prop = getattr(v, propname) if isreversed: rev.append(prop) sortspec.append(prop) v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug #print "filter sort at %s" % time.time() rslt = [] for row in v: id = str(row.id) if search_matches is not None: if search_matches.has_key(id): rslt.append(id) else: rslt.append(id) return rslt def hasnode(self, nodeid): return int(nodeid) < self.maxid def labelprop(self, default_to_id=0): ''' Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] def stringFind(self, **requirements): """Locate a particular node by matching a set of its String properties in a caseless search. If the property is not a String property, a TypeError is raised. The return is a list of the id of all nodes that match. """ for propname in requirements.keys(): prop = self.properties[propname] if isinstance(not prop, hyperdb.String): raise TypeError, "'%s' not a String property"%propname requirements[propname] = requirements[propname].lower() requirements['_isdel'] = 0 l = [] for row in self.getview().select(requirements): l.append(str(row.id)) return l def addjournal(self, nodeid, action, params): self.db.addjournal(self.classname, nodeid, action, params) def index(self, nodeid): ''' Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, hyperdb.String) and propclass.indexme: # index them under (classname, nodeid, property) self.db.indexer.add_text((self.classname, nodeid, prop), str(self.get(nodeid, prop))) # --- used by Database def _commit(self): """ called post commit of the DB. interested subclasses may override """ self.uncommitted = {} self.rbactions = [] self.idcache = {} def _rollback(self): """ called pre rollback of the DB. interested subclasses may override """ for action in self.rbactions: action() self.rbactions = [] self.uncommitted = {} self.idcache = {} def _clear(self): view = self.getview(1) if len(view): view[:] = [] self.db.dirty = 1 iv = self.getindexview(1) if iv: iv[:] = [] def rollbackaction(self, action): """ call this to register a callback called on rollback callback is removed on end of transaction """ self.rbactions.append(action) # --- internal def __getview(self): db = self.db._db view = db.view(self.classname) mkprops = view.structure() if mkprops and self.db.fastopen: return view.ordered(1) # is the definition the same? for nm, rutyp in self.ruprops.items(): for mkprop in mkprops: if mkprop.name == nm: break else: mkprop = None if mkprop is None: break if _typmap[rutyp.__class__] != mkprop.type: break else: return view.ordered(1) # need to create or restructure the mk view # id comes first, so MK will order it for us self.db.dirty = 1 s = ["%s[id:I" % self.classname] for nm, rutyp in self.ruprops.items(): mktyp = _typmap[rutyp.__class__] s.append('%s:%s' % (nm, mktyp)) if mktyp == 'V': s[-1] += ('[fid:I]') s.append('_isdel:I,activity:I,creation:I,creator:I]') v = self.db._db.getas(','.join(s)) self.db.commit() return v.ordered(1) def getview(self, RW=0): return self.db._db.view(self.classname).ordered(1) def getindexview(self, RW=0): return self.db._db.view("_%s" % self.classname).ordered(1) def _fetchML(sv): l = [] for row in sv: if row.fid: l.append(str(row.fid)) return l def _fetchPW(s): p = password.Password() p.unpack(s) return p def _fetchLink(n): return n and str(n) or None def _fetchDate(n): return date.Date(time.gmtime(n)) _converters = { hyperdb.Date : _fetchDate, hyperdb.Link : _fetchLink, hyperdb.Multilink : _fetchML, hyperdb.Interval : date.Interval, hyperdb.Password : _fetchPW, hyperdb.Boolean : lambda n: n, hyperdb.Number : lambda n: n, hyperdb.String : str, } class FileName(hyperdb.String): isfilename = 1 _typmap = { FileName : 'S', hyperdb.String : 'S', hyperdb.Date : 'I', hyperdb.Link : 'I', hyperdb.Multilink : 'V', hyperdb.Interval : 'S', hyperdb.Password : 'S', hyperdb.Boolean : 'I', hyperdb.Number : 'I', } class FileClass(Class): ' like Class but with a content property ' default_mime_type = 'text/plain' def __init__(self, db, classname, **properties): properties['content'] = FileName() if not properties.has_key('type'): properties['type'] = hyperdb.String() Class.__init__(self, db, classname, **properties) def get(self, nodeid, propname, default=_marker, cache=1): x = Class.get(self, nodeid, propname, default, cache) if propname == 'content': if x.startswith('file:'): fnm = x[5:] try: x = open(fnm, 'rb').read() except Exception, e: x = repr(e) return x def create(self, **propvalues): content = propvalues['content'] del propvalues['content'] newid = Class.create(self, **propvalues) if not content: return newid nm = bnm = '%s%s' % (self.classname, newid) sd = str(int(int(newid) / 1000)) d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd) if not os.path.exists(d): os.makedirs(d) nm = os.path.join(d, nm) open(nm, 'wb').write(content) self.set(newid, content = 'file:'+nm) mimetype = propvalues.get('type', self.default_mime_type) self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype) def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer): action1(fnm) self.rollbackaction(undo) return newid def index(self, nodeid): Class.index(self, nodeid) mimetype = self.get(nodeid, 'type') if not mimetype: mimetype = self.default_mime_type self.db.indexer.add_text((self.classname, nodeid, 'content'), self.get(nodeid, 'content'), mimetype) class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): """The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation" or "activity" property, a ValueError is raised.""" if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) CURVERSION = 1 class Indexer(indexer.Indexer): disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} def __init__(self, path, datadb): self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1) self.datadb = datadb self.reindex = 0 v = self.db.view('version') if not v.structure(): v = self.db.getas('version[vers:I]') self.db.commit() v.append(vers=CURVERSION) self.reindex = 1 elif v[0].vers != CURVERSION: v[0].vers = CURVERSION self.reindex = 1 if self.reindex: self.db.getas('ids[tblid:I,nodeid:I,propid:I]') self.db.getas('index[word:S,hits[pos:I]]') self.db.commit() self.reindex = 1 self.changed = 0 self.propcache = {} def force_reindex(self): v = self.db.view('ids') v[:] = [] v = self.db.view('index') v[:] = [] self.db.commit() self.reindex = 1 def should_reindex(self): return self.reindex def _getprops(self, classname): props = self.propcache.get(classname, None) if props is None: props = self.datadb.view(classname).structure() props = [prop.name for prop in props] self.propcache[classname] = props return props def _getpropid(self, classname, propname): return self._getprops(classname).index(propname) def _getpropname(self, classname, propid): return self._getprops(classname)[propid] def add_text(self, identifier, text, mime_type='text/plain'): if mime_type != 'text/plain': return classname, nodeid, property = identifier tbls = self.datadb.view('tables') tblid = tbls.find(name=classname) if tblid < 0: raise KeyError, "unknown class %r"%classname nodeid = int(nodeid) propid = self._getpropid(classname, property) pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid) wordlist = re.findall(r'\b\w{3,25}\b', text) words = {} for word in wordlist: word = word.upper() if not self.disallows.has_key(word): words[word] = 1 words = words.keys() index = self.db.view('index').ordered(1) for word in words: ndx = index.find(word=word) if ndx < 0: ndx = index.append(word=word) hits = index[ndx].hits if len(hits)==0 or hits.find(pos=pos) < 0: hits.append(pos=pos) self.changed = 1 def find(self, wordlist): hits = None index = self.db.view('index').ordered(1) for word in wordlist: if not 2 < len(word) < 26: continue ndx = index.find(word=word) if ndx < 0: return {} if hits is None: hits = index[ndx].hits else: hits = hits.intersect(index[ndx].hits) if len(hits) == 0: return {} if hits is None: return {} rslt = {} ids = self.db.view('ids').remapwith(hits) tbls = self.datadb.view('tables') for i in range(len(ids)): hit = ids[i] classname = tbls[hit.tblid].name nodeid = str(hit.nodeid) property = self._getpropname(classname, hit.propid) rslt[i] = (classname, nodeid, property) return rslt def save_index(self): if self.changed: self.db.commit() self.changed = 0 code-c3e391d9c4e9/roundup/backends/blobfiles.py0000644000000000000000000001466507535063756017562 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: blobfiles.py,v 1.8 2002-07-19 03:36:34 richard Exp $ ''' This module exports file storage for roundup backends. Files are stored into a directory hierarchy. ''' import os def files_in_dir(dir): if not os.path.exists(dir): return 0 num_files = 0 for dir_entry in os.listdir(dir): full_filename = os.path.join(dir,dir_entry) if os.path.isfile(full_filename): num_files = num_files + 1 elif os.path.isdir(full_filename): num_files = num_files + files_in_dir(full_filename) return num_files class FileStorage: """Store files in some directory structure""" def filename(self, classname, nodeid, property=None): '''Determine what the filename for the given node and optionally property is. ''' if property: name = '%s%s.%s'%(classname, nodeid, property) else: # roundupdb.FileClass never specified the property name, so don't # include it name = '%s%s'%(classname, nodeid) # have a separate subdir for every thousand messages subdir = str(int(nodeid) / 1000) return os.path.join(self.dir, 'files', classname, subdir, name) def filename_flat(self, classname, nodeid, property=None): '''Determine what the filename for the given node and optionally property is. ''' if property: return os.path.join(self.dir, 'files', '%s%s.%s'%(classname, nodeid, property)) else: # roundupdb.FileClass never specified the property name, so don't # include it return os.path.join(self.dir, 'files', '%s%s'%(classname, nodeid)) def storefile(self, classname, nodeid, property, content): '''Store the content of the file in the database. The property may be None, in which case the filename does not indicate which property is being saved. ''' # determine the name of the file to write to name = self.filename(classname, nodeid, property) # make sure the file storage dir exists if not os.path.exists(os.path.dirname(name)): os.makedirs(os.path.dirname(name)) # open the temp file for writing open(name + '.tmp', 'wb').write(content) # save off the commit action self.transactions.append((self.doStoreFile, (classname, nodeid, property))) def getfile(self, classname, nodeid, property): '''Get the content of the file in the database. ''' filename = self.filename(classname, nodeid, property) try: return open(filename, 'rb').read() except: # now try the temp pre-commit filename try: return open(filename+'.tmp', 'rb').read() except: # fallback to flat file storage filename = self.filename_flat(classname, nodeid, property) return open(filename, 'rb').read() def numfiles(self): '''Get number of files in storage, even across subdirectories. ''' files_dir = os.path.join(self.dir, 'files') return files_in_dir(files_dir) def doStoreFile(self, classname, nodeid, property, **databases): '''Store the file as part of a transaction commit. ''' # determine the name of the file to write to name = self.filename(classname, nodeid, property) # the file is currently ".tmp" - move it to its real name to commit os.rename(name+".tmp", name) # return the classname, nodeid so we reindex this content return (classname, nodeid) def rollbackStoreFile(self, classname, nodeid, property, **databases): '''Remove the temp file as a part of a rollback ''' # determine the name of the file to delete name = self.filename(classname, nodeid, property) if os.path.exists(name+".tmp"): os.remove(name+".tmp") # $Log: not supported by cvs2svn $ # Revision 1.7 2002/07/14 06:14:40 richard # Some more TODOs # # Revision 1.6 2002/07/09 03:02:52 richard # More indexer work: # - all String properties may now be indexed too. Currently there's a bit of # "issue" specific code in the actual searching which needs to be # addressed. In a nutshell: # + pass 'indexme="yes"' as a String() property initialisation arg, eg: # file = FileClass(db, "file", name=String(), type=String(), # comment=String(indexme="yes")) # + the comment will then be indexed and be searchable, with the results # related back to the issue that the file is linked to # - as a result of this work, the FileClass has a default MIME type that may # be overridden in a subclass, or by the use of a "type" property as is # done in the default templates. # - the regeneration of the indexes (if necessary) is done once the schema is # set up in the dbinit. # # Revision 1.5 2002/07/08 06:58:15 richard # cleaned up the indexer code: # - it splits more words out (much simpler, faster splitter) # - removed code we'll never use (roundup.roundup_indexer has the full # implementation, and replaces roundup.indexer) # - only index text/plain and rfc822/message (ideas for other text formats to # index are welcome) # - added simple unit test for indexer. Needs more tests for regression. # # Revision 1.4 2002/06/19 03:07:19 richard # Moved the file storage commit into blobfiles where it belongs. # # Revision 1.3 2002/02/27 07:33:34 grubert # . add, vim line and cvs log key. # # # vim: set filetype=python ts=4 sw=4 et si code-c3e391d9c4e9/roundup/backends/locking.py0000644000000000000000000000341407535063756017235 0ustar 00000000000000#! /usr/bin/env python # Copyright (c) 2002 ekit.com Inc (http://www.ekit-inc.com/) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # $Id: locking.py,v 1.1 2002-04-15 23:25:15 richard Exp $ '''This module provides a generic interface to acquire and release exclusive access to a file. It should work on Unix and Windows. ''' import portalocker def acquire_lock(path, block=1): '''Acquire a lock for the given path ''' import portalocker file = open(path, 'w') if block: portalocker.lock(file, portalocker.LOCK_EX) else: portalocker.lock(file, portalocker.LOCK_EX|portalocker.LOCK_NB) return file def release_lock(file): '''Release our lock on the given path ''' portalocker.unlock(file) # # $Log: not supported by cvs2svn $ # # code-c3e391d9c4e9/roundup/backends/portalocker.py0000644000000000000000000000445607535063756020143 0ustar 00000000000000# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking. # Requires python 1.5.2 or better. # ID line added by richard for Roundup file tracking # $Id: portalocker.py,v 1.1 2002-04-15 23:25:15 richard Exp $ """Cross-platform (posix/nt) API for flock-style file locking. Synopsis: import portalocker file = open("somefile", "r+") portalocker.lock(file, portalocker.LOCK_EX) file.seek(12) file.write("foo") file.close() If you know what you're doing, you may choose to portalocker.unlock(file) before closing the file, but why? Methods: lock( file, flags ) unlock( file ) Constants: LOCK_EX LOCK_SH LOCK_NB I learned the win32 technique for locking files from sample code provided by John Nielsen in the documentation that accompanies the win32 modules. Author: Jonathan Feinberg Version: Id: portalocker.py,v 1.3 2001/05/29 18:47:55 Administrator Exp **un-cvsified by richard so the version doesn't change** """ import os if os.name == 'nt': import win32con import win32file import pywintypes LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK LOCK_SH = 0 # the default LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # is there any reason not to reuse the following structure? __overlapped = pywintypes.OVERLAPPED() elif os.name == 'posix': import fcntl LOCK_EX = fcntl.LOCK_EX LOCK_SH = fcntl.LOCK_SH LOCK_NB = fcntl.LOCK_NB else: raise RuntimeError("PortaLocker only defined for nt and posix platforms") if os.name == 'nt': def lock(file, flags): hfile = win32file._get_osfhandle(file.fileno()) win32file.LockFileEx(hfile, flags, 0, 0xffff0000, __overlapped) def unlock(file): hfile = win32file._get_osfhandle(file.fileno()) win32file.UnlockFileEx(hfile, 0, 0xffff0000, __overlapped) elif os.name =='posix': def lock(file, flags): fcntl.flock(file.fileno(), flags) def unlock(file): fcntl.flock(file.fileno(), fcntl.LOCK_UN) if __name__ == '__main__': from time import time, strftime, localtime import sys import portalocker log = open('log.txt', "a+") portalocker.lock(log, portalocker.LOCK_EX) timestamp = strftime("%m/%d/%Y %H:%M:%S\n", localtime(time())) log.write( timestamp ) print "Wrote lines. Hit enter to release lock." dummy = sys.stdin.readline() log.close() code-c3e391d9c4e9/roundup/backends/sessions.py0000644000000000000000000000541207535063756017455 0ustar 00000000000000#$Id: sessions.py,v 1.1 2002-07-30 08:22:38 richard Exp $ ''' This module defines a very basic store that's used by the CGI interface to store session information. ''' import anydbm, whichdb, os, marshal class Sessions: ''' Back onto an anydbm store. Keys are session id strings, values are marshalled data. ''' def __init__(self, config): self.config = config self.dir = config.DATABASE # ensure files are group readable and writable os.umask(0002) def clear(self): path = os.path.join(self.dir, 'sessions') if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') def determine_db_type(self, path): ''' determine which DB wrote the class file ''' db_type = '' if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! db_type = 'dbm' return db_type def get(self, sessionid, value): db = self.opendb('c') try: if db.has_key(sessionid): values = marshal.loads(db[sessionid]) else: return None return values.get(value, None) finally: db.close() def set(self, sessionid, **newvalues): db = self.opendb('c') try: if db.has_key(sessionid): values = marshal.loads(db[sessionid]) else: values = {} values.update(newvalues) db[sessionid] = marshal.dumps(values) finally: db.close() def list(self): db = self.opendb('r') try: return db.keys() finally: db.close() def destroy(self, sessionid): db = self.opendb('c') try: if db.has_key(sessionid): del db[sessionid] finally: db.close() def opendb(self, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' # figure the class db type path = os.path.join(os.getcwd(), self.dir, 'sessions') db_type = self.determine_db_type(path) # new database? let anydbm pick the best dbm if not db_type: return anydbm.open(path, 'n') # open the database with the correct module dbm = __import__(db_type) return dbm.open(path, mode) def commit(self): pass # #$Log: not supported by cvs2svn $ # #