code-963003d2ffdd/roundup/backends/.cvsignore0000644000000000000000000000002410344271362017176 0ustar 00000000000000*.pyc *.pyo *.cover code-963003d2ffdd/roundup/backends/__init__.py0000644000000000000000000000547410344271362017325 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # # $Id: __init__.py,v 1.35 2005-10-07 05:35:03 richard Exp $ '''Container for the hyperdb storage backend implementations. ''' __docformat__ = 'restructuredtext' import sys # These names are used to suppress import errors. # If get_backend raises an ImportError with appropriate # module name, have_backend quietly returns False. # Otherwise the error is reraised. _modules = { 'mysql': 'MySQLdb', 'postgresql': 'psycopg', 'tsearch2': 'psycopg', } def get_backend(name): '''Get a specific backend by name.''' vars = globals() # if requested backend has been imported yet, return current instance if vars.has_key(name): return vars[name] # import the backend module module_name = 'back_%s' % name try: module = __import__(module_name, vars) except: # import failed, but in versions prior to 2.4, a (broken) # module is left in sys.modules and package globals; # subsequent imports would succeed and get the broken module. # This no longer happens in Python 2.4 and later. if sys.version_info < (2, 4): del sys.modules['.'.join((__name__, module_name))] del vars[module_name] raise else: vars[name] = module return module def have_backend(name): '''Is backend "name" available?''' if name == 'tsearch2': return 0 try: get_backend(name) return 1 except ImportError, e: global _modules if not str(e).startswith('No module named %s' % _modules.get(name, name)): raise return 0 def list_backends(): '''List all available backend names. This function has side-effect of registering backward-compatible globals for all available backends. ''' l = [] for name in 'anydbm', 'mysql', 'sqlite', 'metakit', 'postgresql': if have_backend(name): l.append(name) return l # vim: set filetype=python sts=4 sw=4 et si : code-963003d2ffdd/roundup/backends/back_anydbm.py0000644000000000000000000024610310344271362020014 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_anydbm.py,v 1.190 2005-10-07 05:35:03 richard Exp $ '''This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several serious bugs, and is not available) ''' __docformat__ = 'restructuredtext' try: import anydbm, sys # dumbdbm only works in python 2.1.2+ if sys.version_info < (2,1,2): import dumbdbm assert anydbm._defaultmod != dumbdbm del dumbdbm except AssertionError: print "WARNING: you should upgrade to python 2.1.3" import whichdb, os, marshal, re, weakref, string, copy, time, shutil, logging from roundup import hyperdb, date, password, roundupdb, security from roundup.backends import locking from roundup.i18n import _ from blobfiles import FileStorage from sessions_dbm import Sessions, OneTimeKeys try: # re-enable once Xapian is fixed from indexer_xapian import Indexer_disabled except ImportError: from indexer_dbm import Indexer def db_exists(config): # check for the user db for db in 'nodes.user nodes.user.db'.split(): if os.path.exists(os.path.join(config.DATABASE, db)): return 1 return 0 def db_nuke(config): shutil.rmtree(config.DATABASE) # # Now the database # class Database(FileStorage, hyperdb.Database, roundupdb.Database): '''A database for storing records containing flexible data types. Transaction stuff TODO: - check the timestamp of the class file and nuke the cache if it's modified. Do some sort of conflict checking on the dirty stuff. - perhaps detect write collisions (related to above)? ''' def __init__(self, config, journaltag=None): '''Open a hyperdatabase given a specifier to some storage. The 'storagelocator' is obtained from config.DATABASE. The meaning of 'storagelocator' depends on the particular implementation of the hyperdatabase. It could be a file name, a directory path, a socket descriptor for a connection to a database over the network, etc. The 'journaltag' is a token that will be attached to the journal entries for any edits done on the database. If 'journaltag' is None, the database is opened in read-only mode: the Class.create(), Class.set(), Class.retire(), and Class.restore() methods are disabled. ''' self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.cache = {} # cache of nodes loaded or created self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, 'filtering': 0} self.dirtynodes = {} # keep track of the dirty nodes by class self.newnodes = {} # keep track of the new nodes by class self.destroyednodes = {}# keep track of the destroyed nodes by class self.transactions = [] self.indexer = Indexer(self) self.security = security.Security(self) # ensure files are group readable and writable os.umask(0002) # lock it lockfilenm = os.path.join(self.dir, 'lock') self.lockfile = locking.acquire_lock(lockfilenm) self.lockfile.write(str(os.getpid())) self.lockfile.flush() def post_init(self): '''Called once the schema initialisation has finished. ''' # reindex the db if necessary if self.indexer.should_reindex(): self.reindex() def refresh_database(self): """Rebuild the database """ self.reindex() def getSessionManager(self): return Sessions(self) def getOTKManager(self): return OneTimeKeys(self) def reindex(self, classname=None): if classname: classes = [self.getclass(classname)] else: classes = self.classes.values() for klass in classes: for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() def __repr__(self): return ''%id(self) # # Classes # def __getattr__(self, classname): '''A convenient way of calling self.getclass(classname).''' if self.classes.has_key(classname): return self.classes[classname] raise AttributeError, classname def addclass(self, cl): cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl # add default Edit and View permissions self.security.addPermission(name="Create", klass=cn, description="User is allowed to create "+cn) self.security.addPermission(name="Edit", klass=cn, description="User is allowed to edit "+cn) self.security.addPermission(name="View", klass=cn, description="User is allowed to access "+cn) def getclasses(self): '''Return a list of the names of all existing classes.''' l = self.classes.keys() l.sort() return l def getclass(self, classname): '''Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. ''' try: return self.classes[classname] except KeyError: raise KeyError, 'There is no class called "%s"'%classname # # Class DBs # def clear(self): '''Delete all database contents ''' logging.getLogger('hyperdb').info('clear') for cn in self.classes.keys(): for dummy in 'nodes', 'journals': path = os.path.join(self.dir, 'journals.%s'%cn) if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') # reset id sequences path = os.path.join(os.getcwd(), self.dir, '_ids') if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' return self.opendb('nodes.%s'%classname, mode) def determine_db_type(self, path): ''' determine which DB wrote the class file ''' db_type = '' if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! db_type = 'dbm' return db_type def opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' # figure the class db type path = os.path.join(os.getcwd(), self.dir, name) db_type = self.determine_db_type(path) # new database? let anydbm pick the best dbm if not db_type: if __debug__: logging.getLogger('hyperdb').debug("opendb anydbm.open(%r, 'c')"%path) return anydbm.open(path, 'c') # open the database with the correct module try: dbm = __import__(db_type) except ImportError: raise hyperdb.DatabaseError, \ "Couldn't open database - the required module '%s'"\ " is not available"%db_type if __debug__: logging.getLogger('hyperdb').debug("opendb %r.open(%r, %r)"%(db_type, path, mode)) return dbm.open(path, mode) # # Node IDs # def newid(self, classname): ''' Generate a new id for the given class ''' # open the ids DB - create if if doesn't exist db = self.opendb('_ids', 'c') if db.has_key(classname): newid = db[classname] = str(int(db[classname]) + 1) else: # the count() bit is transitional - older dbs won't start at 1 newid = str(self.getclass(classname).count()+1) db[classname] = newid db.close() return newid def setid(self, classname, setid): ''' Set the id counter: used during import of database ''' # open the ids DB - create if if doesn't exist db = self.opendb('_ids', 'c') db[classname] = str(setid) db.close() # # Nodes # def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' # we'll be supplied these props if we're doing an import if not node.has_key('creator'): # add in the "calculated" properties (dupe so we don't affect # calling code's node assumptions) node = node.copy() node['creator'] = self.getuid() node['actor'] = self.getuid() node['creation'] = node['activity'] = date.Date() self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node self.savenode(classname, nodeid, node) def setnode(self, classname, nodeid, node): ''' change the specified node ''' self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node self.cache[classname][nodeid] = node self.savenode(classname, nodeid, node) def savenode(self, classname, nodeid, node): ''' perform the saving of data specified by the set/addnode ''' if __debug__: logging.getLogger('hyperdb').debug('save %s%s %r'%(classname, nodeid, node)) self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database Note the "cache" parameter is not used, and exists purely for backward compatibility! ''' # try the cache cache_dict = self.cache.setdefault(classname, {}) if cache_dict.has_key(nodeid): if __debug__: logging.getLogger('hyperdb').debug('get %s%s cached'%(classname, nodeid)) self.stats['cache_hits'] += 1 return cache_dict[nodeid] if __debug__: self.stats['cache_misses'] += 1 start_t = time.time() logging.getLogger('hyperdb').debug('get %s%s'%(classname, nodeid)) # get from the database and save in the cache if db is None: db = self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, "no such %s %s"%(classname, nodeid) # check the uncommitted, destroyed nodes if (self.destroyednodes.has_key(classname) and self.destroyednodes[classname].has_key(nodeid)): raise IndexError, "no such %s %s"%(classname, nodeid) # decode res = marshal.loads(db[nodeid]) # reverse the serialisation res = self.unserialise(classname, res) # store off in the cache dict if cache: cache_dict[nodeid] = res if __debug__: self.stats['get_items'] += (time.time() - start_t) return res def destroynode(self, classname, nodeid): '''Remove a node from the database. Called exclusively by the destroy() method on Class. ''' logging.getLogger('hyperdb').info('destroy %s%s'%(classname, nodeid)) # remove from cache and newnodes if it's there if (self.cache.has_key(classname) and self.cache[classname].has_key(nodeid)): del self.cache[classname][nodeid] if (self.newnodes.has_key(classname) and self.newnodes[classname].has_key(nodeid)): del self.newnodes[classname][nodeid] # see if there's any obvious commit actions that we should get rid of for entry in self.transactions[:]: if entry[1][:2] == (classname, nodeid): self.transactions.remove(entry) # add to the destroyednodes map self.destroyednodes.setdefault(classname, {})[nodeid] = 1 # add the destroy commit action self.transactions.append((self.doDestroyNode, (classname, nodeid))) def serialise(self, classname, node): '''Copy the node contents, converting non-marshallable data into marshallable data. ''' properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): if k == self.RETIRED_FLAG: d[k] = v continue # if the property doesn't exist then we really don't care if not properties.has_key(k): continue # get the property spec prop = properties[k] if isinstance(prop, hyperdb.Password) and v is not None: d[k] = str(v) elif isinstance(prop, hyperdb.Date) and v is not None: d[k] = v.serialise() elif isinstance(prop, hyperdb.Interval) and v is not None: d[k] = v.serialise() else: d[k] = v return d def unserialise(self, classname, node): '''Decode the marshalled node data ''' properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): # if the property doesn't exist, or is the "retired" flag then # it won't be in the properties dict if not properties.has_key(k): d[k] = v continue # get the property spec prop = properties[k] if isinstance(prop, hyperdb.Date) and v is not None: d[k] = date.Date(v) elif isinstance(prop, hyperdb.Interval) and v is not None: d[k] = date.Interval(v) elif isinstance(prop, hyperdb.Password) and v is not None: p = password.Password() p.unpack(v) d[k] = p else: d[k] = v return d def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): return 1 # not in the cache - check the database if db is None: db = self.getclassdb(classname) res = db.has_key(nodeid) return res def countnodes(self, classname, db=None): count = 0 # include the uncommitted nodes if self.newnodes.has_key(classname): count += len(self.newnodes[classname]) if self.destroyednodes.has_key(classname): count -= len(self.destroyednodes[classname]) # and count those in the DB if db is None: db = self.getclassdb(classname) count = count + len(db.keys()) return count # # Files - special node properties # inherited from FileStorage # # Journal # def addjournal(self, classname, nodeid, action, params, creator=None, creation=None): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None 'creator' -- the user performing the action, which defaults to the current user. ''' if __debug__: logging.getLogger('hyperdb').debug('addjournal %s%s %s %r %s %r'%(classname, nodeid, action, params, creator, creation)) if creator is None: creator = self.getuid() self.transactions.append((self.doSaveJournal, (classname, nodeid, action, params, creator, creation))) def setjournal(self, classname, nodeid, journal): '''Set the journal to the "journal" list.''' if __debug__: logging.getLogger('hyperdb').debug('setjournal %s%s %r'%(classname, nodeid, journal)) self.transactions.append((self.doSetJournal, (classname, nodeid, journal))) def getjournal(self, classname, nodeid): ''' get the journal for id Raise IndexError if the node doesn't exist (as per history()'s API) ''' # our journal result res = [] # add any journal entries for transactions not committed to the # database for method, args in self.transactions: if method != self.doSaveJournal: continue (cache_classname, cache_nodeid, cache_action, cache_params, cache_creator, cache_creation) = args if cache_classname == classname and cache_nodeid == nodeid: if not cache_creator: cache_creator = self.getuid() if not cache_creation: cache_creation = date.Date() res.append((cache_nodeid, cache_creation, cache_creator, cache_action, cache_params)) # attempt to open the journal - in some rare cases, the journal may # not exist try: db = self.opendb('journals.%s'%classname, 'r') except anydbm.error, error: if str(error) == "need 'c' or 'n' flag to open new db": raise IndexError, 'no such %s %s'%(classname, nodeid) elif error.args[0] != 2: # this isn't a "not found" error, be alarmed! raise if res: # we have unsaved journal entries, return them return res raise IndexError, 'no such %s %s'%(classname, nodeid) try: journal = marshal.loads(db[nodeid]) except KeyError: db.close() if res: # we have some unsaved journal entries, be happy! return res raise IndexError, 'no such %s %s'%(classname, nodeid) db.close() # add all the saved journal entries for this node for nodeid, date_stamp, user, action, params in journal: res.append((nodeid, date.Date(date_stamp), user, action, params)) return res def pack(self, pack_before): ''' Delete all journal entries except "create" before 'pack_before'. ''' pack_before = pack_before.serialise() for classname in self.getclasses(): packed = 0 # get the journal db db_name = 'journals.%s'%classname path = os.path.join(os.getcwd(), self.dir, classname) db_type = self.determine_db_type(path) db = self.opendb(db_name, 'w') for key in db.keys(): # get the journal for this db entry journal = marshal.loads(db[key]) l = [] last_set_entry = None for entry in journal: # unpack the entry (nodeid, date_stamp, self.journaltag, action, params) = entry # if the entry is after the pack date, _or_ the initial # create entry, then it stays if date_stamp > pack_before or action == 'create': l.append(entry) else: packed += 1 db[key] = marshal.dumps(l) logging.getLogger('hyperdb').info('packed %d %s items'%(packed, classname)) if db_type == 'gdbm': db.reorganize() db.close() # # Basic transaction support # def commit(self): ''' Commit the current transactions. ''' logging.getLogger('hyperdb').info('commit %s transactions'%( len(self.transactions))) # keep a handle to all the database files opened self.databases = {} try: # now, do all the transactions reindex = {} for method, args in self.transactions: reindex[method(*args)] = 1 finally: # make sure we close all the database files for db in self.databases.values(): db.close() del self.databases # reindex the nodes that request it for classname, nodeid in filter(None, reindex.keys()): self.getclass(classname).index(nodeid) # save the indexer state self.indexer.save_index() self.clearCache() def clearCache(self): # all transactions committed, back to normal self.cache = {} self.dirtynodes = {} self.newnodes = {} self.destroyednodes = {} self.transactions = [] def getCachedClassDB(self, classname): ''' get the class db, looking in our cache of databases for commit ''' # get the database handle db_name = 'nodes.%s'%classname if not self.databases.has_key(db_name): self.databases[db_name] = self.getclassdb(classname, 'c') return self.databases[db_name] def doSaveNode(self, classname, nodeid, node): db = self.getCachedClassDB(classname) # now save the marshalled data db[nodeid] = marshal.dumps(self.serialise(classname, node)) # return the classname, nodeid so we reindex this content return (classname, nodeid) def getCachedJournalDB(self, classname): ''' get the journal db, looking in our cache of databases for commit ''' # get the database handle db_name = 'journals.%s'%classname if not self.databases.has_key(db_name): self.databases[db_name] = self.opendb(db_name, 'c') return self.databases[db_name] def doSaveJournal(self, classname, nodeid, action, params, creator, creation): # serialise the parameters now if necessary if isinstance(params, type({})): if action in ('set', 'create'): params = self.serialise(classname, params) # handle supply of the special journalling parameters (usually # supplied on importing an existing database) journaltag = creator if creation: journaldate = creation.serialise() else: journaldate = date.Date().serialise() # create the journal entry entry = (nodeid, journaldate, journaltag, action, params) db = self.getCachedJournalDB(classname) # now insert the journal entry if db.has_key(nodeid): # append to existing s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) def doSetJournal(self, classname, nodeid, journal): l = [] for nodeid, journaldate, journaltag, action, params in journal: # serialise the parameters now if necessary if isinstance(params, type({})): if action in ('set', 'create'): params = self.serialise(classname, params) journaldate = journaldate.serialise() l.append((nodeid, journaldate, journaltag, action, params)) db = self.getCachedJournalDB(classname) db[nodeid] = marshal.dumps(l) def doDestroyNode(self, classname, nodeid): # delete from the class database db = self.getCachedClassDB(classname) if db.has_key(nodeid): del db[nodeid] # delete from the database db = self.getCachedJournalDB(classname) if db.has_key(nodeid): del db[nodeid] # return the classname, nodeid so we reindex this content return (classname, nodeid) def rollback(self): ''' Reverse all actions from the current transaction. ''' logging.getLogger('hyperdb').info('rollback %s transactions'%( len(self.transactions))) for method, args in self.transactions: # delete temporary files if method == self.doStoreFile: self.rollbackStoreFile(*args) self.cache = {} self.dirtynodes = {} self.newnodes = {} self.destroyednodes = {} self.transactions = [] def close(self): ''' Nothing to do ''' if self.lockfile is not None: locking.release_lock(self.lockfile) self.lockfile.close() self.lockfile = None _marker = [] class Class(hyperdb.Class): '''The handle to a particular class of nodes in a hyperdatabase.''' def __init__(self, db, classname, **properties): '''Create a new class with a given name and property specification. 'classname' must not collide with the name of an existing class, or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. ''' for name in 'creation activity creator actor'.split(): if properties.has_key(name): raise ValueError, '"creation", "activity", "creator" and '\ '"actor" are reserved' self.classname = classname self.properties = properties self.db = weakref.proxy(db) # use a weak ref to avoid circularity self.key = '' # should we journal changes (default yes) self.do_journal = 1 # do the db-related init stuff db.addclass(self) self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []} self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []} def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # Editing nodes: def create(self, **propvalues): '''Create a new node of this class and return its id. The keyword arguments in 'propvalues' map property names to values. The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' self.fireAuditors('create', None, propvalues) newid = self.create_inner(**propvalues) self.fireReactors('create', newid, None) return newid def create_inner(self, **propvalues): ''' Called by create, in-between the audit and react calls. ''' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' # new node's id newid = self.db.newid(self.classname) # validate propvalues num_re = re.compile('^\d+$') for key, value in propvalues.items(): if key == self.key: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # try to handle this property try: prop = self.properties[key] except KeyError: raise KeyError, '"%s" has no property "%s"'%(self.classname, key) if value is not None and isinstance(prop, hyperdb.Link): if type(value) != type(''): raise ValueError, 'link value must be String' link_class = self.properties[key].classname # if it isn't a number, it's a key if not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, link_class) elif not self.db.getclass(link_class).hasnode(value): raise IndexError, '%s has no node %s'%(link_class, value) # save off the value propvalues[key] = value # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, value, 'link', (self.classname, newid, key)) elif isinstance(prop, hyperdb.Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of ids'%key # clean up and validate the list of links link_class = self.properties[key].classname l = [] for entry in value: if type(entry) != type(''): raise ValueError, '"%s" multilink value (%r) '\ 'must contain Strings'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, self.properties[key].classname) l.append(entry) value = l propvalues[key] = value # handle additions for nodeid in value: if not self.db.getclass(link_class).hasnode(nodeid): raise IndexError, '%s has no node %s'%(link_class, nodeid) # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) elif isinstance(prop, hyperdb.String): if type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%key if prop.indexme: self.db.indexer.add_text((self.classname, newid, key), value) elif isinstance(prop, hyperdb.Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%key elif isinstance(prop, hyperdb.Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'%key elif isinstance(prop, hyperdb.Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'%key elif value is not None and isinstance(prop, hyperdb.Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%key elif value is not None and isinstance(prop, hyperdb.Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%key # make sure there's data where there needs to be for key, prop in self.properties.items(): if propvalues.has_key(key): continue if key == self.key: raise ValueError, 'key property "%s" is required'%key if isinstance(prop, hyperdb.Multilink): propvalues[key] = [] # done self.db.addnode(self.classname, newid, propvalues) if self.do_journal: self.db.addjournal(self.classname, newid, 'create', {}) return newid def get(self, nodeid, propname, default=_marker, cache=1): '''Get the value of a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. 'cache' exists for backward compatibility, and is not used. Attempts to get the "creation" or "activity" properties should do the right thing. ''' if propname == 'id': return nodeid # get the node's dict d = self.db.getnode(self.classname, nodeid) # check for one of the special props if propname == 'creation': if d.has_key('creation'): return d['creation'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[0][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'activity': if d.has_key('activity'): return d['activity'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: return self.db.getjournal(self.classname, nodeid)[-1][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'creator': if d.has_key('creator'): return d['creator'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: num_re = re.compile('^\d+$') value = journal[0][2] if num_re.match(value): return value else: # old-style "username" journal tag try: return self.db.user.lookup(value) except KeyError: # user's been retired, return admin return '1' else: return self.db.getuid() if propname == 'actor': if d.has_key('actor'): return d['actor'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: num_re = re.compile('^\d+$') value = journal[-1][2] if num_re.match(value): return value else: # old-style "username" journal tag try: return self.db.user.lookup(value) except KeyError: # user's been retired, return admin return '1' else: return self.db.getuid() # get the property (raises KeyErorr if invalid) prop = self.properties[propname] if not d.has_key(propname): if default is _marker: if isinstance(prop, hyperdb.Multilink): return [] else: return None else: return default # return a dupe of the list so code doesn't get confused if isinstance(prop, hyperdb.Multilink): return d[propname][:] return d[propname] def set(self, nodeid, **propvalues): '''Modify a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. Each key in 'propvalues' must be the name of a property of this class or a KeyError is raised. All values in 'propvalues' must be acceptable types for their corresponding properties or a TypeError is raised. If the value of the key property is set, it must not collide with other key strings or a ValueError is raised. If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' self.fireAuditors('set', nodeid, propvalues) oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) for name,prop in self.getprops(protected=0).items(): if oldvalues.has_key(name): continue if isinstance(prop, hyperdb.Multilink): oldvalues[name] = [] else: oldvalues[name] = None propvalues = self.set_inner(nodeid, **propvalues) self.fireReactors('set', nodeid, oldvalues) return propvalues def set_inner(self, nodeid, **propvalues): ''' Called by set, in-between the audit and react calls. ''' if not propvalues: return propvalues if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' node = self.db.getnode(self.classname, nodeid) if node.has_key(self.db.RETIRED_FLAG): raise IndexError num_re = re.compile('^\d+$') # if the journal value is to be different, store it in here journalvalues = {} for propname, value in propvalues.items(): # check to make sure we're not duplicating an existing key if propname == self.key and node[propname] != value: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. try: prop = self.properties[propname] except KeyError: raise KeyError, '"%s" has no property named "%s"'%( self.classname, propname) # if the value's the same as the existing value, no sense in # doing anything current = node.get(propname, None) if value == current: del propvalues[propname] continue journalvalues[propname] = current # do stuff based on the prop type if isinstance(prop, hyperdb.Link): link_class = prop.classname # if it isn't a number, it's a key if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( propname) if isinstance(value, type('')) and not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) if self.do_journal and prop.do_journal: # register the unlink with the old linked node if node.has_key(propname) and node[propname] is not None: self.db.addjournal(link_class, node[propname], 'unlink', (self.classname, nodeid, propname)) # register the link with the newly linked node if value is not None: self.db.addjournal(link_class, value, 'link', (self.classname, nodeid, propname)) elif isinstance(prop, hyperdb.Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of'\ ' ids'%propname link_class = self.properties[propname].classname l = [] for entry in value: # if it isn't a number, it's a key if type(entry) != type(''): raise ValueError, 'new property "%s" link value ' \ 'must be a string'%propname if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, entry, self.properties[propname].classname) l.append(entry) value = l propvalues[propname] = value # figure the journal entry for this property add = [] remove = [] # handle removals if node.has_key(propname): l = node[propname] else: l = [] for id in l[:]: if id in value: continue # register the unlink with the old linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'unlink', (self.classname, nodeid, propname)) l.remove(id) remove.append(id) # handle additions for id in value: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%(link_class, id) if id in l: continue # register the link with the newly linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'link', (self.classname, nodeid, propname)) l.append(id) add.append(id) # figure the journal entry l = [] if add: l.append(('+', add)) if remove: l.append(('-', remove)) if l: journalvalues[propname] = tuple(l) elif isinstance(prop, hyperdb.String): if value is not None and type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%propname if prop.indexme: self.db.indexer.add_text((self.classname, nodeid, propname), value) elif isinstance(prop, hyperdb.Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%propname propvalues[propname] = value elif value is not None and isinstance(prop, hyperdb.Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% propname propvalues[propname] = value elif value is not None and isinstance(prop, hyperdb.Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an '\ 'Interval'%propname propvalues[propname] = value elif value is not None and isinstance(prop, hyperdb.Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%propname elif value is not None and isinstance(prop, hyperdb.Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%propname node[propname] = value # nothing to do? if not propvalues: return propvalues # update the activity time node['activity'] = date.Date() node['actor'] = self.db.getuid() # do the set, and journal it self.db.setnode(self.classname, nodeid, node) if self.do_journal: self.db.addjournal(self.classname, nodeid, 'set', journalvalues) return propvalues def retire(self, nodeid): '''Retire a node. The properties on the node remain available from the get() method, and the node's id is never reused. Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' self.fireAuditors('retire', nodeid, None) node = self.db.getnode(self.classname, nodeid) node[self.db.RETIRED_FLAG] = 1 self.db.setnode(self.classname, nodeid, node) if self.do_journal: self.db.addjournal(self.classname, nodeid, 'retired', None) self.fireReactors('retire', nodeid, None) def restore(self, nodeid): '''Restpre a retired node. Make node available for all operations like it was before retirement. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' node = self.db.getnode(self.classname, nodeid) # check if key property was overrided key = self.getkey() try: id = self.lookup(node[key]) except KeyError: pass else: raise KeyError, "Key property (%s) of retired node clashes with \ existing one (%s)" % (key, node[key]) # Now we can safely restore node self.fireAuditors('restore', nodeid, None) del node[self.db.RETIRED_FLAG] self.db.setnode(self.classname, nodeid, node) if self.do_journal: self.db.addjournal(self.classname, nodeid, 'restored', None) self.fireReactors('restore', nodeid, None) def is_retired(self, nodeid, cldb=None): '''Return true if the node is retired. ''' node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): return 1 return 0 def destroy(self, nodeid): '''Destroy a node. WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted WARNING: use retire() instead WARNING: the properties of this node will not be available ever again WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' self.db.destroynode(self.classname, nodeid) def history(self, nodeid): '''Retrieve the journal of edits on a particular node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. The returned list contains tuples of the form (nodeid, date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.getjournal(self.classname, nodeid) # Locating nodes: def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' return self.db.hasnode(self.classname, nodeid) def setkey(self, propname): '''Select a String property of this class to be the key property. 'propname' must be the name of a String property of this class or None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. If the property doesn't exist, KeyError is raised. ''' prop = self.getprops()[propname] if not isinstance(prop, hyperdb.String): raise TypeError, 'key properties must be String' self.key = propname def getkey(self): '''Return the name of the key property for this class or None.''' return self.key def labelprop(self, default_to_id=0): '''Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] # TODO: set up a separate index db file for this? profile? def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. If this class has no key property, a TypeError is raised. If the 'keyvalue' matches one of the values for the key property among the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. ''' if not self.key: raise TypeError, 'No key property set for class %s'%self.classname cldb = self.db.getclassdb(self.classname) try: for nodeid in self.getnodeids(cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue if not node.has_key(self.key): continue if node[self.key] == keyvalue: return nodeid finally: cldb.close() raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key, keyvalue, self.classname) # change from spec - allows multiple props to match def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname=nodeid or propname={nodeid:1, } 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Examples:: db.issue.find(messages='1') db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' propspec = propspec.items() for propname, itemids in propspec: # check the prop is OK prop = self.properties[propname] if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink): raise TypeError, "'%s' not a Link/Multilink property"%propname # ok, now do the find cldb = self.db.getclassdb(self.classname) l = [] try: for id in self.getnodeids(db=cldb): item = self.db.getnode(self.classname, id, db=cldb) if item.has_key(self.db.RETIRED_FLAG): continue for propname, itemids in propspec: if type(itemids) is not type({}): itemids = {itemids:1} # special case if the item doesn't have this property if not item.has_key(propname): if itemids.has_key(None): l.append(id) break continue # grab the property definition and its value on this item prop = self.properties[propname] value = item[propname] if isinstance(prop, hyperdb.Link) and itemids.has_key(value): l.append(id) break elif isinstance(prop, hyperdb.Multilink): hit = 0 for v in value: if itemids.has_key(v): l.append(id) hit = 1 break if hit: break finally: cldb.close() return l def stringFind(self, **requirements): '''Locate a particular node by matching a set of its String properties in a caseless search. If the property is not a String property, a TypeError is raised. The return is a list of the id of all nodes that match. ''' for propname in requirements.keys(): prop = self.properties[propname] if not isinstance(prop, hyperdb.String): raise TypeError, "'%s' not a String property"%propname requirements[propname] = requirements[propname].lower() l = [] cldb = self.db.getclassdb(self.classname) try: for nodeid in self.getnodeids(cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue for key, value in requirements.items(): if not node.has_key(key): break if node[key] is None or node[key].lower() != value: break else: l.append(nodeid) finally: cldb.close() return l def list(self): ''' Return a list of the ids of the active nodes in this class. ''' l = [] cn = self.classname cldb = self.db.getclassdb(cn) try: for nodeid in self.getnodeids(cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue l.append(nodeid) finally: cldb.close() l.sort() return l def getnodeids(self, db=None): ''' Return a list of ALL nodeids ''' res = [] # start off with the new nodes if self.db.newnodes.has_key(self.classname): res += self.db.newnodes[self.classname].keys() if db is None: db = self.db.getclassdb(self.classname) res = res + db.keys() # remove the uncommitted, destroyed nodes if self.db.destroyednodes.has_key(self.classname): for nodeid in self.db.destroyednodes[self.classname].keys(): if db.has_key(nodeid): res.remove(nodeid) return res def filter(self, search_matches, filterspec, sort=(None,None), group=(None,None), num_re = re.compile('^\d+$')): """Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec. "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} or None The filter must match all properties specificed - but if the property value to match is a list, any one of the values in the list may match for that property to match. """ if __debug__: start_t = time.time() cn = self.classname # optimise filterspec l = [] props = self.getprops() LINK = 'spec:link' MULTILINK = 'spec:multilink' STRING = 'spec:string' DATE = 'spec:date' INTERVAL = 'spec:interval' OTHER = 'spec:other' for k, v in filterspec.items(): propclass = props[k] if isinstance(propclass, hyperdb.Link): if type(v) is not type([]): v = [v] u = [] for entry in v: # the value -1 is a special "not set" sentinel if entry == '-1': entry = None u.append(entry) l.append((LINK, k, u)) elif isinstance(propclass, hyperdb.Multilink): # the value -1 is a special "not set" sentinel if v in ('-1', ['-1']): v = [] elif type(v) is not type([]): v = [v] l.append((MULTILINK, k, v)) elif isinstance(propclass, hyperdb.String) and k != 'id': if type(v) is not type([]): v = [v] for v in v: # simple glob searching v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) v = v.replace('?', '.') v = v.replace('*', '.*?') l.append((STRING, k, re.compile(v, re.I))) elif isinstance(propclass, hyperdb.Date): try: date_rng = propclass.range_from_raw (v, self.db) l.append((DATE, k, date_rng)) except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(propclass, hyperdb.Interval): try: intv_rng = date.Range(v, date.Interval) l.append((INTERVAL, k, intv_rng)) except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(propclass, hyperdb.Boolean): if type(v) != type([]): v = v.split(',') bv = [] for val in v: if type(val) is type(''): bv.append(val.lower() in ('yes', 'true', 'on', '1')) else: bv.append(val) l.append((OTHER, k, bv)) elif k == 'id': if type(v) != type([]): v = v.split(',') l.append((OTHER, k, [str(int(val)) for val in v])) elif isinstance(propclass, hyperdb.Number): if type(v) != type([]): v = v.split(',') l.append((OTHER, k, [float(val) for val in v])) filterspec = l # now, find all the nodes that are active and pass filtering matches = [] cldb = self.db.getclassdb(cn) t = 0 try: # TODO: only full-scan once (use items()) for nodeid in self.getnodeids(cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue # apply filter for t, k, v in filterspec: # handle the id prop if k == 'id': if nodeid not in v: break continue # get the node value nv = node.get(k, None) match = 0 # now apply the property filter if t == LINK: # link - if this node's property doesn't appear in the # filterspec's nodeid list, skip it match = nv in v elif t == MULTILINK: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip # it nv = node.get(k, []) # check for matching the absence of multilink values if not v: match = not nv else: # othewise, make sure this node has each of the # required values for want in v: if want in nv: match = 1 break elif t == STRING: if nv is None: nv = '' # RE search match = v.search(nv) elif t == DATE or t == INTERVAL: if nv is None: match = v is None else: if v.to_value: if v.from_value <= nv and v.to_value >= nv: match = 1 else: if v.from_value <= nv: match = 1 elif t == OTHER: # straight value comparison for the other types match = nv in v if not match: break else: matches.append([nodeid, node]) # filter based on full text search if search_matches is not None: k = [] for v in matches: if search_matches.has_key(v[0]): k.append(v) matches = k # always sort by id if no other sort is specified if sort == (None, None): sort = ('+', 'id') # add sorting information to the match entries directions = [] JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1} for dir, prop in sort, group: if dir is None or prop is None: continue directions.append(dir) propclass = props[prop] try: # cache the opened link class db, if needed. lcldb = None # cache the linked class items too lcache = {} for entry in matches: itemid = entry[-2] item = entry[-1] # handle the properties that might be "faked" # also, handle possible missing properties try: v = item[prop] except KeyError: if JPROPS.has_key(prop): # force lookup of the special journal prop v = self.get(itemid, prop) else: # the node doesn't have a value for this # property if isinstance(propclass, hyperdb.Multilink): v = [] else: v = None entry.insert(0, v) continue # missing (None) values are always sorted first if v is None: entry.insert(0, v) continue if isinstance(propclass, hyperdb.String): # it might be a string that's really an integer try: tv = int(v) except: v = v.lower() else: v = tv elif isinstance(propclass, hyperdb.Link): lcn = propclass.classname link = self.db.classes[lcn] key = None if link.getprops().has_key('order'): key = 'order' elif link.getkey(): key = link.getkey() if key: if not lcache.has_key(v): # open the link class db if it's not already if lcldb is None: lcldb = self.db.getclassdb(lcn) lcache[v] = self.db.getnode(lcn, v, lcldb) v = lcache[v][key] entry.insert(0, v) finally: # if we opened the link class db, close it now if lcldb is not None: lcldb.close() del lcache finally: cldb.close() # sort vals are inserted, but directions are appended, so reverse directions.reverse() if '-' in directions: # one or more of the sort specs is in reverse order, so we have # to use this icky function to sort def sortfun(a, b, directions=directions, n=range(len(directions))): for i in n: if not cmp(a[i], b[i]): continue if directions[i] == '+': # compare in the usual, ascending direction return cmp(a[i],b[i]) else: # compare in the reverse, descending direction return cmp(b[i],a[i]) # for consistency, sort by the id if the items are equal return cmp(a[-2], b[-2]) matches.sort(sortfun) else: # sorting is in the normal, ascending direction matches.sort() # pull the id out of the individual entries matches = [entry[-2] for entry in matches] if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return matches def count(self): '''Get the number of nodes in this class. If the returned integer is 'numnodes', the ids of all the nodes in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. ''' return self.db.countnodes(self.classname) # Manipulating properties: def getprops(self, protected=1): '''Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - those which may not be modified. In addition to the actual properties on the node, these methods provide the "creation" and "activity" properties. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = self.properties.copy() if protected: d['id'] = hyperdb.String() d['creation'] = hyperdb.Date() d['activity'] = hyperdb.Date() d['creator'] = hyperdb.Link('user') d['actor'] = hyperdb.Link('user') return d def addprop(self, **properties): '''Add properties to this class. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. None of the keys in 'properties' may collide with the names of existing properties, or a ValueError is raised before any properties have been added. ''' for key in properties.keys(): if self.properties.has_key(key): raise ValueError, key self.properties.update(properties) def index(self, nodeid): ''' Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, hyperdb.String) and propclass.indexme: # index them under (classname, nodeid, property) try: value = str(self.get(nodeid, prop)) except IndexError: # node has been destroyed continue self.db.indexer.add_text((self.classname, nodeid, prop), value) # # Detector interface # def audit(self, event, detector): '''Register a detector ''' l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): '''Fire all registered auditors. ''' for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def react(self, event, detector): '''Register a detector ''' l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) def fireReactors(self, action, nodeid, oldvalues): '''Fire all registered reactors. ''' for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) # # import / export support # def export_list(self, propnames, nodeid): ''' Export a node - generate a list of CSV-able data in the order specified by propnames for the given node. ''' properties = self.getprops() l = [] for prop in propnames: proptype = properties[prop] value = self.get(nodeid, prop) # "marshal" data where needed if value is None: pass elif isinstance(proptype, hyperdb.Date): value = value.get_tuple() elif isinstance(proptype, hyperdb.Interval): value = value.get_tuple() elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) # append retired flag l.append(repr(self.is_retired(nodeid))) return l def import_list(self, propnames, proplist): ''' Import a node - all information including "id" is present and should not be sanity checked. Triggers are not triggered. The journal should be initialised using the "creator" and "created" information. Return the nodeid of the node imported. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' properties = self.getprops() # make the new node's property map d = {} newid = None for i in range(len(propnames)): # Figure the property for this column propname = propnames[i] # Use eval to reverse the repr() used to output the CSV value = eval(proplist[i]) # "unmarshal" where necessary if propname == 'id': newid = value continue elif propname == 'is retired': # is the item retired? if int(value): d[self.db.RETIRED_FLAG] = 1 continue elif value is None: d[propname] = None continue prop = properties[propname] if isinstance(prop, hyperdb.Date): value = date.Date(value) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value) elif isinstance(prop, hyperdb.Password): pwd = password.Password() pwd.unpack(value) value = pwd d[propname] = value # get a new id if necessary if newid is None: newid = self.db.newid(self.classname) # add the node and journal self.db.addnode(self.classname, newid, d) return newid def export_journals(self): '''Export a class's journal - generate a list of lists of CSV-able data: nodeid, date, user, action, params No heading here - the columns are fixed. ''' properties = self.getprops() r = [] for nodeid in self.getnodeids(): for nodeid, date, user, action, params in self.history(nodeid): date = date.get_tuple() if action == 'set': export_data = {} for propname, value in params.items(): if not properties.has_key(propname): # property no longer in the schema continue prop = properties[propname] # make sure the params are eval()'able if value is None: # don't export empties continue elif isinstance(prop, hyperdb.Date): # this is a hack - some dates are stored as strings if not isinstance(value, type('')): value = value.get_tuple() elif isinstance(prop, hyperdb.Interval): # hack too - some intervals are stored as strings if not isinstance(value, type('')): value = value.get_tuple() elif isinstance(prop, hyperdb.Password): value = str(value) export_data[propname] = value params = export_data l = [nodeid, date, user, action, params] r.append(map(repr, l)) return r def import_journals(self, entries): '''Import a class's journal. Uses setjournal() to set the journal for each item.''' properties = self.getprops() d = {} for l in entries: l = map(eval, l) nodeid, jdate, user, action, params = l r = d.setdefault(nodeid, []) if action == 'set': for propname, value in params.items(): prop = properties[propname] if value is None: pass elif isinstance(prop, hyperdb.Date): if type(value) == type(()): print _('WARNING: invalid date tuple %r')%(value,) value = date.Date( "2000-1-1" ) value = date.Date(value) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value) elif isinstance(prop, hyperdb.Password): pwd = password.Password() pwd.unpack(value) value = pwd params[propname] = value r.append((nodeid, date.Date(jdate), user, action, params)) for nodeid, l in d.items(): self.db.setjournal(self.classname, nodeid, l) class FileClass(hyperdb.FileClass, Class): '''This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. The default MIME type of this data is defined by the "default_mime_type" class attribute, which may be overridden by each node if the class defines a "type" String property. ''' default_mime_type = 'text/plain' def create(self, **propvalues): ''' Snarf the "content" propvalue and store in a file ''' # we need to fire the auditors now, or the content property won't # be in propvalues for the auditors to play with self.fireAuditors('create', None, propvalues) # now remove the content property so it's not stored in the db content = propvalues['content'] del propvalues['content'] # make sure we have a MIME type mime_type = propvalues.get('type', self.default_mime_type) # do the database create newid = self.create_inner(**propvalues) # fire reactors self.fireReactors('create', newid, None) # store off the content as a file self.db.storefile(self.classname, newid, None, content) return newid def get(self, nodeid, propname, default=_marker, cache=1): ''' Trap the content propname and get it from the file 'cache' exists for backwards compatibility, and is not used. ''' poss_msg = 'Possibly an access right configuration problem.' if propname == 'content': try: return self.db.getfile(self.classname, nodeid, None) except IOError, (strerror): # XXX by catching this we don't see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) if default is not _marker: return Class.get(self, nodeid, propname, default) else: return Class.get(self, nodeid, propname) def set(self, itemid, **propvalues): ''' Snarf the "content" propvalue and update it in a file ''' self.fireAuditors('set', itemid, propvalues) # create the oldvalues dict - fill in any missing values oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) for name,prop in self.getprops(protected=0).items(): if oldvalues.has_key(name): continue if isinstance(prop, hyperdb.Multilink): oldvalues[name] = [] else: oldvalues[name] = None # now remove the content property so it's not stored in the db content = None if propvalues.has_key('content'): content = propvalues['content'] del propvalues['content'] # do the database update propvalues = self.set_inner(itemid, **propvalues) # do content? if content: # store and index self.db.storefile(self.classname, itemid, None, content) mime_type = None if self.getprops().has_key('type'): mime_type = propvalues.get('type', self.get(itemid, 'type')) if not mime_type: mime_type = self.default_mime_type self.db.indexer.add_text((self.classname, itemid, 'content'), content, mime_type) propvalues['content'] = content # fire reactors self.fireReactors('set', itemid, oldvalues) return propvalues def getprops(self, protected=1): '''In addition to the actual properties on the node, these methods provide the "content" property. If the "protected" flag is true, we include protected properties - those which may not be modified. Note that the content prop is indexed separately, hence no indexme. ''' d = Class.getprops(self, protected=protected).copy() d['content'] = hyperdb.String() return d def index(self, nodeid): '''Add (or refresh) the node to search indexes. Pass on the content-type property for the content property. ''' Class.index(self, nodeid) mime_type = None if self.getprops().has_key('type'): mime_type = self.get(nodeid, 'type', self.default_mime_type) if not mime_type: mime_type = self.default_mime_type self.db.indexer.add_text((self.classname, nodeid, 'content'), str(self.get(nodeid, 'content')), mime_type) # deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): '''The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation" or "activity" property, a ValueError is raised. ''' if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) # vim: set et sts=4 sw=4 : code-963003d2ffdd/roundup/backends/back_metakit.py0000755000000000000000000023022110344271362020175 0ustar 00000000000000# $Id: back_metakit.py,v 1.98 2005-10-07 05:35:04 richard Exp $ '''Metakit backend for Roundup, originally by Gordon McMillan. Known Current Bugs: - You can't change a class' key properly. This shouldn't be too hard to fix. - Some unit tests are overridden. Notes by Richard: This backend has some behaviour specific to metakit: - there's no concept of an explicit "unset" in metakit, so all types have some "unset" value: ========= ===== ====================================================== Type Value Action when fetching from mk ========= ===== ====================================================== Strings '' convert to None Date 0 (seconds since 1970-01-01.00:00:00) convert to None Interval '' convert to None Number 0 ambiguious :( - do nothing (see BACKWARDS_COMPATIBLE) Boolean 0 ambiguious :( - do nothing (see BACKWARDS_COMPATABILE) Link 0 convert to None Multilink [] actually, mk can handle this one ;) Password '' convert to None ========= ===== ====================================================== The get/set routines handle these values accordingly by converting to/from None where they can. The Number/Boolean types are not able to handle an "unset" at all, so they default the "unset" to 0. - Metakit relies in reference counting to close the database, there is no explicit close call. This can cause issues if a metakit database is referenced multiple times, one might not actually be closing the db. - probably a bunch of stuff that I'm not aware of yet because I haven't fully read through the source. One of these days.... ''' __docformat__ = 'restructuredtext' # Enable this flag to break backwards compatibility (i.e. can't read old # databases) but comply with more roundup features, like adding NULL support. BACKWARDS_COMPATIBLE = 1 from roundup import hyperdb, date, password, roundupdb, security import logging import metakit from sessions_dbm import Sessions, OneTimeKeys import re, marshal, os, sys, time, calendar, shutil from indexer_common import Indexer, is_stopword import locking from roundup.date import Range from blobfiles import files_in_dir # view modes for opening # XXX FIXME BPK -> these don't do anything, they are ignored # should we just get rid of them for simplicities sake? READ = 0 READWRITE = 1 def db_exists(config): return os.path.exists(os.path.join(config.TRACKER_HOME, 'db', 'tracker.mk4')) def db_nuke(config): shutil.rmtree(os.path.join(config.TRACKER_HOME, 'db')) # general metakit error class MKBackendError(Exception): pass _dbs = {} def Database(config, journaltag=None): ''' Only have a single instance of the Database class for each instance ''' db = _dbs.get(config.DATABASE, None) if db is None or db._db is None: db = _Database(config, journaltag) _dbs[config.DATABASE] = db else: db.journaltag = journaltag return db class _Database(hyperdb.Database, roundupdb.Database): def __init__(self, config, journaltag=None): self.config = config self.journaltag = journaltag self.classes = {} self.dirty = 0 self.lockfile = None self._db = self.__open() self.indexer = Indexer(self) self.security = security.Security(self) self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, 'filtering': 0} os.umask(0002) def post_init(self): if self.indexer.should_reindex(): self.reindex() def refresh_database(self): # XXX handle refresh self.reindex() def reindex(self, classname=None): if classname: classes = [self.getclass(classname)] else: classes = self.classes.values() for klass in classes: for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() def getSessionManager(self): return Sessions(self) def getOTKManager(self): return OneTimeKeys(self) # --- defined in ping's spec def __getattr__(self, classname): if classname == 'transactions': return self.dirty # fall back on the classes try: return self.getclass(classname) except KeyError, msg: # KeyError's not appropriate here raise AttributeError, str(msg) def getclass(self, classname): try: return self.classes[classname] except KeyError: raise KeyError, 'There is no class called "%s"'%classname def getclasses(self): return self.classes.keys() # --- end of ping's spec # --- exposed methods def commit(self): '''commit all changes to the database''' if self.dirty: self._db.commit() for cl in self.classes.values(): cl._commit() self.indexer.save_index() self.dirty = 0 def rollback(self): '''roll back all changes since the last commit''' if self.dirty: for cl in self.classes.values(): cl._rollback() self._db.rollback() self._db = None self._db = metakit.storage(self.dbnm, 1) self.hist = self._db.view('history') self.tables = self._db.view('tables') self.indexer.rollback() self.indexer.datadb = self._db self.dirty = 0 def clearCache(self): '''clear the internal cache by committing all pending database changes''' for cl in self.classes.values(): cl._commit() def clear(self): '''clear the internal cache but don't commit any changes''' for cl in self.classes.values(): cl._clear() def hasnode(self, classname, nodeid): '''does a particular class contain a nodeid?''' return self.getclass(classname).hasnode(nodeid) def pack(self, pack_before): ''' Delete all journal entries except "create" before 'pack_before'. ''' mindate = int(calendar.timegm(pack_before.get_tuple())) i = 0 while i < len(self.hist): if self.hist[i].date < mindate and self.hist[i].action != _CREATE: self.hist.delete(i) else: i = i + 1 def addclass(self, cl): ''' Add a Class to the hyperdatabase. ''' cn = cl.classname self.classes[cn] = cl if self.tables.find(name=cn) < 0: self.tables.append(name=cn) # add default Edit and View permissions self.security.addPermission(name="Create", klass=cn, description="User is allowed to create "+cn) self.security.addPermission(name="Edit", klass=cn, description="User is allowed to edit "+cn) self.security.addPermission(name="View", klass=cn, description="User is allowed to access "+cn) def addjournal(self, tablenm, nodeid, action, params, creator=None, creation=None): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' tblid = self.tables.find(name=tablenm) if tblid == -1: tblid = self.tables.append(name=tablenm) if creator is None: creator = int(self.getuid()) else: try: creator = int(creator) except TypeError: creator = int(self.getclass('user').lookup(creator)) if creation is None: creation = int(time.time()) elif isinstance(creation, date.Date): creation = int(calendar.timegm(creation.get_tuple())) # tableid:I,nodeid:I,date:I,user:I,action:I,params:B self.hist.append(tableid=tblid, nodeid=int(nodeid), date=creation, action=action, user=creator, params=marshal.dumps(params)) def setjournal(self, tablenm, nodeid, journal): '''Set the journal to the "journal" list.''' tblid = self.tables.find(name=tablenm) if tblid == -1: tblid = self.tables.append(name=tablenm) for nodeid, date, user, action, params in journal: # tableid:I,nodeid:I,date:I,user:I,action:I,params:B self.hist.append(tableid=tblid, nodeid=int(nodeid), date=date, action=action, user=int(user), params=marshal.dumps(params)) def getjournal(self, tablenm, nodeid): ''' get the journal for id ''' rslt = [] tblid = self.tables.find(name=tablenm) if tblid == -1: return rslt q = self.hist.select(tableid=tblid, nodeid=int(nodeid)) if len(q) == 0: raise IndexError, "no history for id %s in %s" % (nodeid, tablenm) i = 0 #userclass = self.getclass('user') for row in q: try: params = marshal.loads(row.params) except ValueError: logging.getLogger("hyperdb").error( "history couldn't unmarshal %r" % row.params) params = {} #usernm = userclass.get(str(row.user), 'username') dt = date.Date(time.gmtime(row.date)) #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params)) rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params)) return rslt def destroyjournal(self, tablenm, nodeid): nodeid = int(nodeid) tblid = self.tables.find(name=tablenm) if tblid == -1: return i = 0 hist = self.hist while i < len(hist): if hist[i].tableid == tblid and hist[i].nodeid == nodeid: hist.delete(i) else: i = i + 1 self.dirty = 1 def close(self): ''' Close off the connection. ''' # de-reference count the metakit databases, # as this is the only way they will be closed for cl in self.classes.values(): cl.db = None self._db = None if self.lockfile is not None: locking.release_lock(self.lockfile) if _dbs.has_key(self.config.DATABASE): del _dbs[self.config.DATABASE] if self.lockfile is not None: self.lockfile.close() self.lockfile = None self.classes = {} # force the indexer to close self.indexer.close() self.indexer = None # --- internal def __open(self): ''' Open the metakit database ''' # make the database dir if it doesn't exist if not os.path.exists(self.config.DATABASE): os.makedirs(self.config.DATABASE) # figure the file names self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') lockfilenm = db[:-3]+'lck' # get the database lock self.lockfile = locking.acquire_lock(lockfilenm) self.lockfile.write(str(os.getpid())) self.lockfile.flush() # see if the schema has changed since last db access self.fastopen = 0 if os.path.exists(db): dbtm = os.path.getmtime(db) schemafile = os.path.join(self.config['HOME'], 'schema.py') if not os.path.isfile(schemafile): # try old-style schema schemafile = os.path.join(self.config['HOME'], 'dbinit.py') if os.path.isfile(schemafile) \ and (os.path.getmtime(schemafile) < dbtm): # found schema file - it's older than the db self.fastopen = 1 # open the db db = metakit.storage(db, 1) hist = db.view('history') tables = db.view('tables') if not self.fastopen: # create the database if it's brand new if not hist.structure(): hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]') if not tables.structure(): tables = db.getas('tables[name:S]') db.commit() # we now have an open, initialised database self.tables = tables self.hist = hist return db def setid(self, classname, maxid): ''' No-op in metakit ''' cls = self.getclass(classname) cls.setid(int(maxid)) def numfiles(self): '''Get number of files in storage, even across subdirectories. ''' files_dir = os.path.join(self.config.DATABASE, 'files') return files_in_dir(files_dir) _STRINGTYPE = type('') _LISTTYPE = type([]) _CREATE, _SET, _RETIRE, _LINK, _UNLINK, _RESTORE = range(6) _actionnames = { _CREATE : 'create', _SET : 'set', _RETIRE : 'retire', _RESTORE : 'restore', _LINK : 'link', _UNLINK : 'unlink', } _names_to_actionnames = { 'create': _CREATE, 'set': _SET, 'retire': _RETIRE, 'restore': _RESTORE, 'link': _LINK, 'unlink': _UNLINK, } _marker = [] _ALLOWSETTINGPRIVATEPROPS = 0 class Class(hyperdb.Class): ''' The handle to a particular class of nodes in a hyperdatabase. All methods except __repr__ and getnode must be implemented by a concrete backend Class of which this is one. ''' privateprops = None def __init__(self, db, classname, **properties): if (properties.has_key('creation') or properties.has_key('activity') or properties.has_key('creator') or properties.has_key('actor')): raise ValueError, '"creation", "activity" and "creator" are '\ 'reserved' if hasattr(db, classname): raise ValueError, "Class %s already exists"%classname self.db = db self.classname = classname self.key = None self.ruprops = properties self.privateprops = { 'id' : hyperdb.String(), 'activity' : hyperdb.Date(), 'actor' : hyperdb.Link('user'), 'creation' : hyperdb.Date(), 'creator' : hyperdb.Link('user') } # event -> list of callables self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []} self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []} view = self.__getview() self.maxid = 1 if view: self.maxid = view[-1].id + 1 self.uncommitted = {} self.comactions = [] self.rbactions = [] # people reach inside!! self.properties = self.ruprops self.db.addclass(self) self.idcache = {} # default is to journal changes self.do_journal = 1 def setid(self, maxid): self.maxid = maxid + 1 def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # # Detector/reactor interface # def audit(self, event, detector): '''Register a detector ''' l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): '''Fire all registered auditors. ''' for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def react(self, event, detector): '''Register a reactor ''' l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) def fireReactors(self, action, nodeid, oldvalues): '''Fire all registered reactors. ''' for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) # --- the hyperdb.Class methods def create(self, **propvalues): ''' Create a new node of this class and return its id. The keyword arguments in 'propvalues' map property names to values. The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. ''' if not propvalues: raise ValueError, "Need something to create!" self.fireAuditors('create', None, propvalues) newid = self.create_inner(**propvalues) self.fireReactors('create', newid, None) return newid def create_inner(self, **propvalues): ''' Called by create, in-between the audit and react calls. ''' rowdict = {} rowdict['id'] = newid = self.maxid self.maxid += 1 ndx = self.getview(READWRITE).append(rowdict) propvalues['#ISNEW'] = 1 try: self.set_inner(str(newid), **propvalues) except Exception: self.maxid -= 1 raise return str(newid) def get(self, nodeid, propname, default=_marker, cache=1): '''Get the value of a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. 'cache' exists for backwards compatibility, and is not used. ''' view = self.getview() id = int(nodeid) if cache == 0: oldnode = self.uncommitted.get(id, None) if oldnode and oldnode.has_key(propname): raw = oldnode[propname] converter = _converters.get(raw.__class__, None) if converter: return converter(raw) return raw ndx = self.idcache.get(id, None) if ndx is None: ndx = view.find(id=id) if ndx < 0: raise IndexError, "%s has no node %s" % (self.classname, nodeid) self.idcache[id] = ndx try: raw = getattr(view[ndx], propname) except AttributeError: raise KeyError, propname rutyp = self.ruprops.get(propname, None) if rutyp is None: rutyp = self.privateprops[propname] converter = _converters.get(rutyp.__class__, None) if converter: raw = converter(raw) return raw def set(self, nodeid, **propvalues): '''Modify a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. Each key in 'propvalues' must be the name of a property of this class or a KeyError is raised. All values in 'propvalues' must be acceptable types for their corresponding properties or a TypeError is raised. If the value of the key property is set, it must not collide with other key strings or a ValueError is raised. If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. ''' self.fireAuditors('set', nodeid, propvalues) propvalues, oldnode = self.set_inner(nodeid, **propvalues) self.fireReactors('set', nodeid, oldnode) def set_inner(self, nodeid, **propvalues): '''Called outside of auditors''' isnew = 0 if propvalues.has_key('#ISNEW'): isnew = 1 del propvalues['#ISNEW'] if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' view = self.getview(READWRITE) # node must exist & not be retired id = int(nodeid) ndx = view.find(id=id) if ndx < 0: raise IndexError, "%s has no node %s" % (self.classname, nodeid) row = view[ndx] if row._isdel: raise IndexError, "%s has no node %s" % (self.classname, nodeid) oldnode = self.uncommitted.setdefault(id, {}) changes = {} for key, value in propvalues.items(): # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. if _ALLOWSETTINGPRIVATEPROPS: prop = self.ruprops.get(key, None) if not prop: prop = self.privateprops[key] else: prop = self.ruprops[key] converter = _converters.get(prop.__class__, lambda v: v) # if the value's the same as the existing value, no sense in # doing anything oldvalue = converter(getattr(row, key)) if value == oldvalue: del propvalues[key] continue # check to make sure we're not duplicating an existing key if key == self.key: iv = self.getindexview(READWRITE) ndx = iv.find(k=value) if ndx == -1: iv.append(k=value, i=row.id) if not isnew: ndx = iv.find(k=oldvalue) if ndx > -1: iv.delete(ndx) else: raise ValueError, 'node with key "%s" exists'%value # do stuff based on the prop type if isinstance(prop, hyperdb.Link): link_class = prop.classname # must be a string or None if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( key) # Roundup sets to "unselected" by passing None if value is None: value = 0 # if it isn't a number, it's a key try: int(value) except ValueError: try: value = self.db.getclass(link_class).lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) setattr(row, key, int(value)) changes[key] = oldvalue if self.do_journal and prop.do_journal: # register the unlink with the old linked node if oldvalue: self.db.addjournal(link_class, oldvalue, _UNLINK, (self.classname, str(row.id), key)) # register the link with the newly linked node if value: self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key)) elif isinstance(prop, hyperdb.Multilink): if value is not None and type(value) != _LISTTYPE: raise TypeError, 'new property "%s" not a list of ids'%key link_class = prop.classname l = [] if value is None: value = [] for entry in value: if type(entry) != _STRINGTYPE: raise ValueError, 'new property "%s" link value ' \ 'must be a string'%key # if it isn't a number, it's a key try: int(entry) except ValueError: try: entry = self.db.getclass(link_class).lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, prop.classname) l.append(entry) propvalues[key] = value = l # handle removals rmvd = [] for id in oldvalue: if id not in value: rmvd.append(id) # register the unlink with the old linked node if self.do_journal and prop.do_journal: self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key)) # handle additions adds = [] for id in value: if id not in oldvalue: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%( link_class, id) adds.append(id) # register the link with the newly linked node if self.do_journal and prop.do_journal: self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key)) # perform the modifications on the actual property value sv = getattr(row, key) i = 0 while i < len(sv): if str(sv[i].fid) in rmvd: sv.delete(i) else: i += 1 for id in adds: sv.append(fid=int(id)) # figure the journal entry l = [] if adds: l.append(('+', adds)) if rmvd: l.append(('-', rmvd)) if l: changes[key] = tuple(l) #changes[key] = oldvalue if not rmvd and not adds: del propvalues[key] elif isinstance(prop, hyperdb.String): if value is not None and type(value) != _STRINGTYPE: raise TypeError, 'new property "%s" not a string'%key if value is None: value = '' setattr(row, key, value) changes[key] = oldvalue if hasattr(prop, 'isfilename') and prop.isfilename: propvalues[key] = os.path.basename(value) if prop.indexme: self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain') elif isinstance(prop, hyperdb.Password): if value is not None and not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'% key if value is None: value = '' setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) elif isinstance(prop, hyperdb.Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% key if value is None: setattr(row, key, 0) else: setattr(row, key, int(calendar.timegm(value.get_tuple()))) if oldvalue is None: changes[key] = oldvalue else: changes[key] = str(oldvalue) propvalues[key] = str(value) elif isinstance(prop, hyperdb.Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'% key if value is None: setattr(row, key, '') else: # kedder: we should store interval values serialized setattr(row, key, value.serialise()) changes[key] = str(oldvalue) propvalues[key] = str(value) elif isinstance(prop, hyperdb.Number): if value is None: v = 0 else: try: v = float(value) except ValueError: raise TypeError, "%s (%s) is not numeric"%(key, repr(value)) if not BACKWARDS_COMPATIBLE: if v >=0: v = v + 1 setattr(row, key, v) changes[key] = oldvalue propvalues[key] = value elif isinstance(prop, hyperdb.Boolean): if value is None: bv = 0 elif value not in (0,1): raise TypeError, "%s (%s) is not boolean"%(key, repr(value)) else: bv = value if not BACKWARDS_COMPATIBLE: bv += 1 setattr(row, key, bv) changes[key] = oldvalue propvalues[key] = value oldnode[key] = oldvalue # nothing to do? if not isnew and not propvalues: return propvalues, oldnode if not propvalues.has_key('activity'): row.activity = int(time.time()) if not propvalues.has_key('actor'): row.actor = int(self.db.getuid()) if isnew: if not row.creation: row.creation = int(time.time()) if not row.creator: row.creator = int(self.db.getuid()) self.db.dirty = 1 if self.do_journal: if isnew: self.db.addjournal(self.classname, nodeid, _CREATE, {}) else: self.db.addjournal(self.classname, nodeid, _SET, changes) return propvalues, oldnode def retire(self, nodeid): '''Retire a node. The properties on the node remain available from the get() method, and the node's id is never reused. Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' self.fireAuditors('retire', nodeid, None) view = self.getview(READWRITE) ndx = view.find(id=int(nodeid)) if ndx < 0: raise KeyError, "nodeid %s not found" % nodeid row = view[ndx] oldvalues = self.uncommitted.setdefault(row.id, {}) oldval = oldvalues['_isdel'] = row._isdel row._isdel = 1 if self.do_journal: self.db.addjournal(self.classname, nodeid, _RETIRE, {}) if self.key: iv = self.getindexview(READWRITE) ndx = iv.find(k=getattr(row, self.key)) # find is broken with multiple attribute lookups # on ordered views #ndx = iv.find(k=getattr(row, self.key),i=row.id) if ndx > -1 and iv[ndx].i == row.id: iv.delete(ndx) self.db.dirty = 1 self.fireReactors('retire', nodeid, None) def restore(self, nodeid): '''Restore a retired node. Make node available for all operations like it was before retirement. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' # check if key property was overrided key = self.getkey() keyvalue = self.get(nodeid, key) try: id = self.lookup(keyvalue) except KeyError: pass else: raise KeyError, "Key property (%s) of retired node clashes with \ existing one (%s)" % (key, keyvalue) # Now we can safely restore node self.fireAuditors('restore', nodeid, None) view = self.getview(READWRITE) ndx = view.find(id=int(nodeid)) if ndx < 0: raise KeyError, "nodeid %s not found" % nodeid row = view[ndx] oldvalues = self.uncommitted.setdefault(row.id, {}) oldval = oldvalues['_isdel'] = row._isdel row._isdel = 0 if self.do_journal: self.db.addjournal(self.classname, nodeid, _RESTORE, {}) if self.key: iv = self.getindexview(READWRITE) ndx = iv.find(k=getattr(row, self.key),i=row.id) if ndx > -1: iv.delete(ndx) self.db.dirty = 1 self.fireReactors('restore', nodeid, None) def is_retired(self, nodeid): '''Return true if the node is retired ''' view = self.getview(READWRITE) # node must exist & not be retired id = int(nodeid) ndx = view.find(id=id) if ndx < 0: raise IndexError, "%s has no node %s" % (self.classname, nodeid) row = view[ndx] return row._isdel def history(self, nodeid): '''Retrieve the journal of edits on a particular node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. The returned list contains tuples of the form (nodeid, date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.getjournal(self.classname, nodeid) def setkey(self, propname): '''Select a String property of this class to be the key property. 'propname' must be the name of a String property of this class or None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. ''' if self.key: if propname == self.key: return else: # drop the old key table tablename = "_%s.%s"%(self.classname, self.key) self.db._db.getas(tablename) #raise ValueError, "%s already indexed on %s"%(self.classname, # self.key) prop = self.properties.get(propname, None) if prop is None: prop = self.privateprops.get(propname, None) if prop is None: raise KeyError, "no property %s" % propname if not isinstance(prop, hyperdb.String): raise TypeError, "%s is not a String" % propname # the way he index on properties is by creating a # table named _%(classname)s.%(key)s, if this table # exists then everything is okay. If this table # doesn't exist, then generate a new table on the # key value. # first setkey for this run or key has been changed self.key = propname tablename = "_%s.%s"%(self.classname, self.key) iv = self.db._db.view(tablename) if self.db.fastopen and iv.structure(): return # very first setkey ever or the key has changed self.db.dirty = 1 iv = self.db._db.getas('_%s[k:S,i:I]' % tablename) iv = iv.ordered(1) for row in self.getview(): iv.append(k=getattr(row, propname), i=row.id) self.db.commit() def getkey(self): '''Return the name of the key property for this class or None.''' return self.key def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. If this class has no key property, a TypeError is raised. If the keyvalue matches one of the values for the key property among the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. ''' if not self.key: raise TypeError, 'No key property set for class %s'%self.classname if type(keyvalue) is not _STRINGTYPE: raise TypeError, '%r is not a string'%keyvalue # XXX FIX ME -> this is a bit convoluted # First we search the index view to get the id # which is a quicker look up. # Then we lookup the row with id=id # if the _isdel property of the row is 0, return the # string version of the id. (Why string version???) # # Otherwise, just lookup the non-indexed key # in the non-index table and check the _isdel property iv = self.getindexview() if iv: # look up the index view for the id, # then instead of looking up the keyvalue, lookup the # quicker id ndx = iv.find(k=keyvalue) if ndx > -1: view = self.getview() ndx = view.find(id=iv[ndx].i) if ndx > -1: row = view[ndx] if not row._isdel: return str(row.id) else: # perform the slower query view = self.getview() ndx = view.find({self.key:keyvalue}) if ndx > -1: row = view[ndx] if not row._isdel: return str(row.id) raise KeyError, keyvalue def destroy(self, id): '''Destroy a node. WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted WARNING: use retire() instead WARNING: the properties of this node will not be available ever again WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. The node is completely removed from the hyperdb, including all journal entries. It will no longer be available, and will generally break code if there are any references to the node. ''' view = self.getview(READWRITE) ndx = view.find(id=int(id)) if ndx > -1: if self.key: keyvalue = getattr(view[ndx], self.key) iv = self.getindexview(READWRITE) if iv: ivndx = iv.find(k=keyvalue) if ivndx > -1: iv.delete(ivndx) view.delete(ndx) self.db.destroyjournal(self.classname, id) self.db.dirty = 1 def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname=nodeid or propname={nodeid:1, } 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Examples:: db.issue.find(messages='1') db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' propspec = propspec.items() for propname, nodeid in propspec: # check the prop is OK prop = self.ruprops[propname] if (not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink)): raise TypeError, "'%s' not a Link/Multilink property"%propname vws = [] for propname, ids in propspec: if type(ids) is _STRINGTYPE: ids = {int(ids):1} elif ids is None: ids = {0:1} else: d = {} for id in ids.keys(): if id is None: d[0] = 1 else: d[int(id)] = 1 ids = d prop = self.ruprops[propname] view = self.getview() if isinstance(prop, hyperdb.Multilink): def ff(row, nm=propname, ids=ids): if not row._isdel: sv = getattr(row, nm) for sr in sv: if ids.has_key(sr.fid): return 1 return 0 else: def ff(row, nm=propname, ids=ids): return not row._isdel and ids.has_key(getattr(row, nm)) ndxview = view.filter(ff) vws.append(ndxview.unique()) # handle the empty match case if not vws: return [] ndxview = vws[0] for v in vws[1:]: ndxview = ndxview.union(v) view = self.getview().remapwith(ndxview) rslt = [] for row in view: rslt.append(str(row.id)) return rslt def list(self): ''' Return a list of the ids of the active nodes in this class. ''' l = [] for row in self.getview().select(_isdel=0): l.append(str(row.id)) return l def getnodeids(self): ''' Retrieve all the ids of the nodes for a particular Class. Set retired=None to get all nodes. Otherwise it'll get all the retired or non-retired nodes, depending on the flag. ''' l = [] for row in self.getview(): l.append(str(row.id)) return l def count(self): return len(self.getview()) def getprops(self, protected=1): # protected is not in ping's spec allprops = self.ruprops.copy() if protected and self.privateprops is not None: allprops.update(self.privateprops) return allprops def addprop(self, **properties): for key in properties.keys(): if self.ruprops.has_key(key): raise ValueError, "%s is already a property of %s"%(key, self.classname) self.ruprops.update(properties) # Class structure has changed self.db.fastopen = 0 view = self.__getview() self.db.commit() # ---- end of ping's spec def filter(self, search_matches, filterspec, sort=(None,None), group=(None,None)): '''Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} or None The filter must match all properties specificed - but if the property value to match is a list, any one of the values in the list may match for that property to match. ''' if __debug__: start_t = time.time() where = {'_isdel':0} wherehigh = {} mlcriteria = {} regexes = [] orcriteria = {} for propname, value in filterspec.items(): prop = self.ruprops.get(propname, None) if prop is None: prop = self.privateprops[propname] if isinstance(prop, hyperdb.Multilink): if value in ('-1', ['-1']): value = [] elif type(value) is not _LISTTYPE: value = [value] # transform keys to ids u = [] for item in value: try: item = int(item) except (TypeError, ValueError): item = int(self.db.getclass(prop.classname).lookup(item)) if item == -1: item = 0 u.append(item) mlcriteria[propname] = u elif isinstance(prop, hyperdb.Link): if type(value) is not _LISTTYPE: value = [value] # transform keys to ids u = [] for item in value: if item is None: item = -1 else: try: item = int(item) except (TypeError, ValueError): linkcl = self.db.getclass(prop.classname) item = int(linkcl.lookup(item)) if item == -1: item = 0 u.append(item) if len(u) == 1: where[propname] = u[0] else: orcriteria[propname] = u elif isinstance(prop, hyperdb.String): if type(value) is not type([]): value = [value] for v in value: # simple glob searching v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) v = v.replace('?', '.') v = v.replace('*', '.*?') regexes.append((propname, re.compile(v, re.I))) elif propname == 'id': where[propname] = int(value) elif isinstance(prop, hyperdb.Boolean): if type(value) is _STRINGTYPE: bv = value.lower() in ('yes', 'true', 'on', '1') else: bv = value where[propname] = bv elif isinstance(prop, hyperdb.Date): try: # Try to filter on range of dates date_rng = prop.range_from_raw (value, self.db) if date_rng.from_value: t = date_rng.from_value.get_tuple() where[propname] = int(calendar.timegm(t)) else: # use minimum possible value to exclude items without # 'prop' property where[propname] = 0 if date_rng.to_value: t = date_rng.to_value.get_tuple() wherehigh[propname] = int(calendar.timegm(t)) else: wherehigh[propname] = None except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(prop, hyperdb.Interval): try: # Try to filter on range of intervals date_rng = Range(value, date.Interval) if date_rng.from_value: #t = date_rng.from_value.get_tuple() where[propname] = date_rng.from_value.serialise() else: # use minimum possible value to exclude items without # 'prop' property where[propname] = '-99999999999999' if date_rng.to_value: #t = date_rng.to_value.get_tuple() wherehigh[propname] = date_rng.to_value.serialise() else: wherehigh[propname] = None except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(prop, hyperdb.Number): if type(value) is _LISTTYPE: orcriteria[propname] = [float(v) for v in value] else: where[propname] = float(value) else: where[propname] = str(value) v = self.getview() if where: where_higherbound = where.copy() where_higherbound.update(wherehigh) v = v.select(where, where_higherbound) if mlcriteria: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip it def ff(row, ml=mlcriteria): for propname, values in ml.items(): sv = getattr(row, propname) if not values and not sv: return 1 for id in values: if sv.find(fid=id) != -1: return 1 return 0 iv = v.filter(ff) v = v.remapwith(iv) if orcriteria: def ff(row, crit=orcriteria): for propname, allowed in crit.items(): val = getattr(row, propname) if val not in allowed: return 0 return 1 iv = v.filter(ff) v = v.remapwith(iv) if regexes: def ff(row, r=regexes): for propname, regex in r: val = str(getattr(row, propname)) if not regex.search(val): return 0 return 1 iv = v.filter(ff) v = v.remapwith(iv) if sort or group: sortspec = [] rev = [] for dir, propname in group, sort: if propname is None: continue isreversed = 0 if dir == '-': isreversed = 1 try: prop = getattr(v, propname) except AttributeError: logging.getLogger("hyperdb").error( "MK has no property %s" % propname) continue propclass = self.ruprops.get(propname, None) if propclass is None: propclass = self.privateprops.get(propname, None) if propclass is None: logging.getLogger("hyperdb").error( "Schema has no property %s" % propname) continue if isinstance(propclass, hyperdb.Link): linkclass = self.db.getclass(propclass.classname) lv = linkclass.getview() lv = lv.rename('id', propname) v = v.join(lv, prop, 1) if linkclass.getprops().has_key('order'): propname = 'order' else: propname = linkclass.labelprop() prop = getattr(v, propname) if isreversed: rev.append(prop) sortspec.append(prop) v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug rslt = [] for row in v: id = str(row.id) if search_matches is not None: if search_matches.has_key(id): rslt.append(id) else: rslt.append(id) if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return rslt def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' return int(nodeid) < self.maxid def labelprop(self, default_to_id=0): '''Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] def stringFind(self, **requirements): '''Locate a particular node by matching a set of its String properties in a caseless search. If the property is not a String property, a TypeError is raised. The return is a list of the id of all nodes that match. ''' for propname in requirements.keys(): prop = self.properties[propname] if isinstance(not prop, hyperdb.String): raise TypeError, "'%s' not a String property"%propname requirements[propname] = requirements[propname].lower() requirements['_isdel'] = 0 l = [] for row in self.getview().select(requirements): l.append(str(row.id)) return l def addjournal(self, nodeid, action, params): '''Add a journal to the given nodeid, 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' self.db.addjournal(self.classname, nodeid, action, params) def index(self, nodeid): ''' Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, hyperdb.String) and propclass.indexme: # index them under (classname, nodeid, property) self.db.indexer.add_text((self.classname, nodeid, prop), str(self.get(nodeid, prop))) # --- used by Database def _commit(self): ''' called post commit of the DB. interested subclasses may override ''' self.uncommitted = {} for action in self.comactions: action() self.comactions = [] self.rbactions = [] self.idcache = {} def _rollback(self): ''' called pre rollback of the DB. interested subclasses may override ''' self.comactions = [] for action in self.rbactions: action() self.rbactions = [] self.uncommitted = {} self.idcache = {} def _clear(self): view = self.getview(READWRITE) if len(view): view[:] = [] self.db.dirty = 1 iv = self.getindexview(READWRITE) if iv: iv[:] = [] def commitaction(self, action): ''' call this to register a callback called on commit callback is removed on end of transaction ''' self.comactions.append(action) def rollbackaction(self, action): ''' call this to register a callback called on rollback callback is removed on end of transaction ''' self.rbactions.append(action) # --- internal def __getview(self): ''' Find the interface for a specific Class in the hyperdb. This method checks to see whether the schema has changed and re-works the underlying metakit structure if it has. ''' db = self.db._db view = db.view(self.classname) mkprops = view.structure() # if we have structure in the database, and the structure hasn't # changed # note on view.ordered -> # return a metakit view ordered on the id column # id is always the first column. This speeds up # look-ups on the id column. if mkprops and self.db.fastopen: return view.ordered(1) # is the definition the same? for nm, rutyp in self.ruprops.items(): for mkprop in mkprops: if mkprop.name == nm: break else: mkprop = None if mkprop is None: break if _typmap[rutyp.__class__] != mkprop.type: break else: # make sure we have the 'actor' property too for mkprop in mkprops: if mkprop.name == 'actor': return view.ordered(1) # The schema has changed. We need to create or restructure the mk view # id comes first, so we can use view.ordered(1) so that # MK will order it for us to allow binary-search quick lookups on # the id column self.db.dirty = 1 s = ["%s[id:I" % self.classname] # these columns will always be added, we can't trample them :) _columns = {"id":"I", "_isdel":"I", "activity":"I", "actor": "I", "creation":"I", "creator":"I"} for nm, rutyp in self.ruprops.items(): mktyp = _typmap[rutyp.__class__].upper() if nm in _columns and _columns[nm] != mktyp: # oops, two columns with the same name and different properties raise MKBackendError("column %s for table %sis defined with multiple types"%(nm, self.classname)) _columns[nm] = mktyp s.append('%s:%s' % (nm, mktyp)) if mktyp == 'V': s[-1] += ('[fid:I]') # XXX FIX ME -> in some tests, creation:I becomes creation:S is this # okay? Does this need to be supported? s.append('_isdel:I,activity:I,actor:I,creation:I,creator:I]') view = self.db._db.getas(','.join(s)) self.db.commit() return view.ordered(1) def getview(self, RW=0): # XXX FIX ME -> The RW flag doesn't do anything. return self.db._db.view(self.classname).ordered(1) def getindexview(self, RW=0): # XXX FIX ME -> The RW flag doesn't do anything. tablename = "_%s.%s"%(self.classname, self.key) return self.db._db.view("_%s" % tablename).ordered(1) # # import / export # def export_list(self, propnames, nodeid): ''' Export a node - generate a list of CSV-able data in the order specified by propnames for the given node. ''' properties = self.getprops() l = [] for prop in propnames: proptype = properties[prop] value = self.get(nodeid, prop) # "marshal" data where needed if value is None: pass elif isinstance(proptype, hyperdb.Date): value = value.get_tuple() elif isinstance(proptype, hyperdb.Interval): value = value.get_tuple() elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) # append retired flag l.append(repr(self.is_retired(nodeid))) return l def import_list(self, propnames, proplist): ''' Import a node - all information including "id" is present and should not be sanity checked. Triggers are not triggered. The journal should be initialised using the "creator" and "creation" information. Return the nodeid of the node imported. ''' if self.db.journaltag is None: raise hyperdb.DatabaseError, 'Database open read-only' properties = self.getprops() d = {} view = self.getview(READWRITE) for i in range(len(propnames)): value = eval(proplist[i]) if not value: continue propname = propnames[i] if propname == 'id': newid = value = int(value) elif propname == 'is retired': # is the item retired? if int(value): d['_isdel'] = 1 continue elif value is None: d[propname] = None continue prop = properties[propname] if isinstance(prop, hyperdb.Date): value = int(calendar.timegm(value)) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value).serialise() elif isinstance(prop, hyperdb.Number): value = float(value) elif isinstance(prop, hyperdb.Boolean): value = int(value) elif isinstance(prop, hyperdb.Link) and value: value = int(value) elif isinstance(prop, hyperdb.Multilink): # we handle multilinks separately continue d[propname] = value # possibly make a new node if not d.has_key('id'): d['id'] = newid = self.maxid self.maxid += 1 # save off the node view.append(d) # fix up multilinks ndx = view.find(id=newid) row = view[ndx] for i in range(len(propnames)): value = eval(proplist[i]) propname = propnames[i] if propname == 'is retired': continue prop = properties[propname] if not isinstance(prop, hyperdb.Multilink): continue sv = getattr(row, propname) for entry in value: sv.append((int(entry),)) self.db.dirty = 1 return newid def export_journals(self): '''Export a class's journal - generate a list of lists of CSV-able data: nodeid, date, user, action, params No heading here - the columns are fixed. ''' from roundup.hyperdb import Interval, Date, Password properties = self.getprops() r = [] for nodeid in self.getnodeids(): for nodeid, date, user, action, params in self.history(nodeid): date = date.get_tuple() if action == 'set': export_data = {} for propname, value in params.items(): if not properties.has_key(propname): # property no longer in the schema continue prop = properties[propname] # make sure the params are eval()'able if value is None: pass elif isinstance(prop, Date): value = value.get_tuple() elif isinstance(prop, Interval): value = value.get_tuple() elif isinstance(prop, Password): value = str(value) export_data[propname] = value params = export_data l = [nodeid, date, user, action, params] r.append(map(repr, l)) return r def import_journals(self, entries): '''Import a class's journal. Uses setjournal() to set the journal for each item.''' properties = self.getprops() d = {} for l in entries: l = map(eval, l) nodeid, jdate, user, action, params = l jdate = int(calendar.timegm(date.Date(jdate).get_tuple())) r = d.setdefault(nodeid, []) if action == 'set': for propname, value in params.items(): prop = properties[propname] if value is None: pass elif isinstance(prop, hyperdb.Date): value = date.Date(value) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value) elif isinstance(prop, hyperdb.Password): pwd = password.Password() pwd.unpack(value) value = pwd params[propname] = value action = _names_to_actionnames[action] r.append((nodeid, jdate, user, action, params)) for nodeid, l in d.items(): self.db.setjournal(self.classname, nodeid, l) def _fetchML(sv): l = [] for row in sv: if row.fid: l.append(str(row.fid)) return l def _fetchPW(s): ''' Convert to a password.Password unless the password is '' which is our sentinel for "unset". ''' if s == '': return None p = password.Password() p.unpack(s) return p def _fetchLink(n): ''' Return None if the link is 0 - otherwise strify it. ''' return n and str(n) or None def _fetchDate(n): ''' Convert the timestamp to a date.Date instance - unless it's 0 which is our sentinel for "unset". ''' if n == 0: return None return date.Date(time.gmtime(n)) def _fetchInterval(n): ''' Convert to a date.Interval unless the interval is '' which is our sentinel for "unset". ''' if n == '': return None return date.Interval(n) # Converters for boolean and numbers to properly # return None values. # These are in conjunction with the setters above # look for hyperdb.Boolean and hyperdb.Number if BACKWARDS_COMPATIBLE: def getBoolean(bool): return bool def getNumber(number): return number else: def getBoolean(bool): if not bool: res = None else: res = bool - 1 return res def getNumber(number): if number == 0: res = None elif number < 0: res = number else: res = number - 1 return res _converters = { hyperdb.Date : _fetchDate, hyperdb.Link : _fetchLink, hyperdb.Multilink : _fetchML, hyperdb.Interval : _fetchInterval, hyperdb.Password : _fetchPW, hyperdb.Boolean : getBoolean, hyperdb.Number : getNumber, hyperdb.String : lambda s: s and str(s) or None, } class FileName(hyperdb.String): isfilename = 1 _typmap = { FileName : 'S', hyperdb.String : 'S', hyperdb.Date : 'I', hyperdb.Link : 'I', hyperdb.Multilink : 'V', hyperdb.Interval : 'S', hyperdb.Password : 'S', hyperdb.Boolean : 'I', hyperdb.Number : 'D', } class FileClass(hyperdb.FileClass, Class): ''' like Class but with a content property ''' default_mime_type = 'text/plain' def __init__(self, db, classname, **properties): properties['content'] = FileName() if not properties.has_key('type'): properties['type'] = hyperdb.String() Class.__init__(self, db, classname, **properties) def gen_filename(self, nodeid): nm = '%s%s' % (self.classname, nodeid) sd = str(int(int(nodeid) / 1000)) d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd) if not os.path.exists(d): os.makedirs(d) return os.path.join(d, nm) def export_files(self, dirname, nodeid): ''' Export the "content" property as a file, not csv column ''' source = self.gen_filename(nodeid) x, filename = os.path.split(source) x, subdir = os.path.split(x) dest = os.path.join(dirname, self.classname+'-files', subdir, filename) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) shutil.copyfile(source, dest) def import_files(self, dirname, nodeid): ''' Import the "content" property as a file ''' dest = self.gen_filename(nodeid) x, filename = os.path.split(dest) x, subdir = os.path.split(x) source = os.path.join(dirname, self.classname+'-files', subdir, filename) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) shutil.copyfile(source, dest) def get(self, nodeid, propname, default=_marker, cache=1): if propname == 'content': poss_msg = 'Possibly an access right configuration problem.' fnm = self.gen_filename(nodeid) if not os.path.exists(fnm): fnm = fnm + '.tmp' try: f = open(fnm, 'rb') except IOError, (strerror): # XXX by catching this we donot see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) x = f.read() f.close() else: x = Class.get(self, nodeid, propname, default) return x def create(self, **propvalues): if not propvalues: raise ValueError, "Need something to create!" self.fireAuditors('create', None, propvalues) content = propvalues['content'] del propvalues['content'] newid = Class.create_inner(self, **propvalues) if not content: return newid # figure a filename nm = self.gen_filename(newid) # make sure we don't register the rename action more than once if not os.path.exists(nm + '.tmp'): # register commit and rollback actions def commit(fnm=nm): os.rename(fnm + '.tmp', fnm) self.commitaction(commit) def undo(fnm=nm): os.remove(fnm + '.tmp') self.rollbackaction(undo) # save the tempfile f = open(nm + '.tmp', 'wb') f.write(content) f.close() mimetype = propvalues.get('type', self.default_mime_type) self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype) return newid def set(self, itemid, **propvalues): if not propvalues: return self.fireAuditors('set', None, propvalues) content = propvalues.get('content', None) if content is not None: del propvalues['content'] propvalues, oldnode = Class.set_inner(self, itemid, **propvalues) # figure a filename if content is not None: nm = self.gen_filename(itemid) # make sure we don't register the rename action more than once if not os.path.exists(nm + '.tmp'): # register commit and rollback actions def commit(fnm=nm): if os.path.exists(fnm): os.remove(fnm) os.rename(fnm + '.tmp', fnm) self.commitaction(commit) def undo(fnm=nm): os.remove(fnm + '.tmp') self.rollbackaction(undo) f = open(nm + '.tmp', 'wb') f.write(content) f.close() mimetype = propvalues.get('type', self.default_mime_type) self.db.indexer.add_text((self.classname, itemid, 'content'), content, mimetype) self.fireReactors('set', oldnode, propvalues) def index(self, nodeid): '''Add (or refresh) the node to search indexes. Pass on the content-type property for the content property. ''' Class.index(self, nodeid) try: mime_type = self.get(nodeid, 'type', self.default_mime_type) except KeyError: mime_type = self.default_mime_type self.db.indexer.add_text((self.classname, nodeid, 'content'), str(self.get(nodeid, 'content')), mime_type) class IssueClass(Class, roundupdb.IssueClass): ''' The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation" or "activity" property, a ValueError is raised. ''' def __init__(self, db, classname, **properties): if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) CURVERSION = 2 class MetakitIndexer(Indexer): def __init__(self, db): self.path = os.path.join(db.config.DATABASE, 'index.mk4') self.db = metakit.storage(self.path, 1) self.datadb = db._db self.reindex = 0 v = self.db.view('version') if not v.structure(): v = self.db.getas('version[vers:I]') self.db.commit() v.append(vers=CURVERSION) self.reindex = 1 elif v[0].vers != CURVERSION: v[0].vers = CURVERSION self.reindex = 1 if self.reindex: self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]') self.db.getas('index[word:S,hits[pos:I]]') self.db.commit() self.reindex = 1 self.changed = 0 self.propcache = {} def close(self): '''close the indexing database''' del self.db self.db = None def force_reindex(self): '''Force a reindexing of the database. This essentially empties the tables ids and index and sets a flag so that the databases are reindexed''' v = self.db.view('ids') v[:] = [] v = self.db.view('index') v[:] = [] self.db.commit() self.reindex = 1 def should_reindex(self): '''returns True if the indexes need to be rebuilt''' return self.reindex def _getprops(self, classname): props = self.propcache.get(classname, None) if props is None: props = self.datadb.view(classname).structure() props = [prop.name for prop in props] self.propcache[classname] = props return props def _getpropid(self, classname, propname): return self._getprops(classname).index(propname) def _getpropname(self, classname, propid): return self._getprops(classname)[propid] def add_text(self, identifier, text, mime_type='text/plain'): if mime_type != 'text/plain': return classname, nodeid, property = identifier tbls = self.datadb.view('tables') tblid = tbls.find(name=classname) if tblid < 0: raise KeyError, "unknown class %r"%classname nodeid = int(nodeid) propid = self._getpropid(classname, property) ids = self.db.view('ids') oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0) if oldpos > -1: ids[oldpos].ignore = 1 self.changed = 1 pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid) wordlist = re.findall(r'\b\w{2,25}\b', text.upper()) words = {} for word in wordlist: if not is_stopword(word): words[word] = 1 words = words.keys() index = self.db.view('index').ordered(1) for word in words: ndx = index.find(word=word) if ndx < 0: index.append(word=word) ndx = index.find(word=word) index[ndx].hits.append(pos=pos) self.changed = 1 def find(self, wordlist): '''look up all the words in the wordlist. If none are found return an empty dictionary * more rules here ''' hits = None index = self.db.view('index').ordered(1) for word in wordlist: word = word.upper() if not 2 < len(word) < 26: continue ndx = index.find(word=word) if ndx < 0: return {} if hits is None: hits = index[ndx].hits else: hits = hits.intersect(index[ndx].hits) if len(hits) == 0: return {} if hits is None: return [] rslt = [] ids = self.db.view('ids').remapwith(hits) tbls = self.datadb.view('tables') for i in range(len(ids)): hit = ids[i] if not hit.ignore: classname = tbls[hit.tblid].name nodeid = str(hit.nodeid) property = self._getpropname(classname, hit.propid) rslt.append((classname, nodeid, property)) return rslt def save_index(self): if self.changed: self.db.commit() self.changed = 0 def rollback(self): if self.changed: self.db.rollback() self.db = metakit.storage(self.path, 1) self.changed = 0 try: # re-enable once Xapian is fixed from indexer_xapian import Indexer_disabled except ImportError: Indexer = MetakitIndexer # vim: set et sts=4 sw=4 : code-963003d2ffdd/roundup/backends/back_mysql.py0000644000000000000000000006734610344271362017721 0ustar 00000000000000#$Id: back_mysql.py,v 1.60 2005-10-24 03:37:11 richard Exp $ # # Copyright (c) 2003 Martynas Sklyzmantas, Andrey Lebedev # # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # '''This module defines a backend implementation for MySQL. How to implement AUTO_INCREMENT: mysql> create table foo (num integer auto_increment primary key, name varchar(255)) AUTO_INCREMENT=1 type=InnoDB; ql> insert into foo (name) values ('foo5'); Query OK, 1 row affected (0.00 sec) mysql> SELECT num FROM foo WHERE num IS NULL; +-----+ | num | +-----+ | 4 | +-----+ 1 row in set (0.00 sec) mysql> SELECT num FROM foo WHERE num IS NULL; Empty set (0.00 sec) NOTE: we don't need an index on the id column if it's PRIMARY KEY ''' __docformat__ = 'restructuredtext' from roundup.backends.rdbms_common import * from roundup.backends import rdbms_common import MySQLdb import os, shutil from MySQLdb.constants import ER import logging def connection_dict(config, dbnamestr=None): d = rdbms_common.connection_dict(config, dbnamestr) if d.has_key('password'): d['passwd'] = d['password'] del d['password'] if d.has_key('port'): d['port'] = int(d['port']) return d def db_nuke(config): """Clear all database contents and drop database itself""" if db_exists(config): kwargs = connection_dict(config) conn = MySQLdb.connect(**kwargs) try: conn.select_db(config.RDBMS_NAME) except: # no, it doesn't exist pass else: cursor = conn.cursor() cursor.execute("SHOW TABLES") tables = cursor.fetchall() # stupid MySQL bug requires us to drop all the tables first for table in tables: command = 'DROP TABLE %s'%table[0] if __debug__: logging.getLogger('hyperdb').debug(command) cursor.execute(command) command = "DROP DATABASE %s"%config.RDBMS_NAME logging.getLogger('hyperdb').info(command) cursor.execute(command) conn.commit() conn.close() if os.path.exists(config.DATABASE): shutil.rmtree(config.DATABASE) def db_create(config): """Create the database.""" kwargs = connection_dict(config) conn = MySQLdb.connect(**kwargs) cursor = conn.cursor() command = "CREATE DATABASE %s"%config.RDBMS_NAME logging.getLogger('hyperdb').info(command) cursor.execute(command) conn.commit() conn.close() def db_exists(config): """Check if database already exists.""" kwargs = connection_dict(config) conn = MySQLdb.connect(**kwargs) try: try: conn.select_db(config.RDBMS_NAME) except MySQLdb.OperationalError: return 0 finally: conn.close() return 1 class Database(Database): arg = '%s' # used by some code to switch styles of query implements_intersect = 0 # Backend for MySQL to use. # InnoDB is faster, but if you're running <4.0.16 then you'll need to # use BDB to pass all unit tests. mysql_backend = 'InnoDB' #mysql_backend = 'BDB' hyperdb_to_sql_datatypes = { hyperdb.String : 'TEXT', hyperdb.Date : 'DATETIME', hyperdb.Link : 'INTEGER', hyperdb.Interval : 'VARCHAR(255)', hyperdb.Password : 'VARCHAR(255)', hyperdb.Boolean : 'BOOL', hyperdb.Number : 'REAL', } hyperdb_to_sql_value = { hyperdb.String : str, # no fractional seconds for MySQL hyperdb.Date : lambda x: x.formal(sep=' '), hyperdb.Link : int, hyperdb.Interval : str, hyperdb.Password : str, hyperdb.Boolean : int, hyperdb.Number : lambda x: x, hyperdb.Multilink : lambda x: x, # used in journal marshalling } def sql_open_connection(self): kwargs = connection_dict(self.config, 'db') logging.getLogger('hyperdb').info('open database %r'%(kwargs['db'],)) try: conn = MySQLdb.connect(**kwargs) except MySQLdb.OperationalError, message: raise DatabaseError, message cursor = conn.cursor() cursor.execute("SET AUTOCOMMIT=0") cursor.execute("START TRANSACTION") return (conn, cursor) def open_connection(self): # make sure the database actually exists if not db_exists(self.config): db_create(self.config) self.conn, self.cursor = self.sql_open_connection() try: self.load_dbschema() except MySQLdb.OperationalError, message: if message[0] != ER.NO_DB_ERROR: raise except MySQLdb.ProgrammingError, message: if message[0] != ER.NO_SUCH_TABLE: raise DatabaseError, message self.init_dbschema() self.sql("CREATE TABLE schema (schema TEXT) TYPE=%s"% self.mysql_backend) self.sql('''CREATE TABLE ids (name VARCHAR(255), num INTEGER) TYPE=%s'''%self.mysql_backend) self.sql('create index ids_name_idx on ids(name)') self.create_version_2_tables() def create_version_2_tables(self): # OTK store self.sql('''CREATE TABLE otks (otk_key VARCHAR(255), otk_value TEXT, otk_time FLOAT(20)) TYPE=%s'''%self.mysql_backend) self.sql('CREATE INDEX otks_key_idx ON otks(otk_key)') # Sessions store self.sql('''CREATE TABLE sessions (session_key VARCHAR(255), session_time FLOAT(20), session_value TEXT) TYPE=%s'''%self.mysql_backend) self.sql('''CREATE INDEX sessions_key_idx ON sessions(session_key)''') # full-text indexing store self.sql('''CREATE TABLE __textids (_class VARCHAR(255), _itemid VARCHAR(255), _prop VARCHAR(255), _textid INT) TYPE=%s'''%self.mysql_backend) self.sql('''CREATE TABLE __words (_word VARCHAR(30), _textid INT) TYPE=%s'''%self.mysql_backend) self.sql('CREATE INDEX words_word_ids ON __words(_word)') sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg) self.sql(sql, ('__textids', 1)) def add_new_columns_v2(self): '''While we're adding the actor column, we need to update the tables to have the correct datatypes.''' for klass in self.classes.values(): cn = klass.classname properties = klass.getprops() old_spec = self.database_schema['tables'][cn] # figure the non-Multilink properties to copy over propnames = ['activity', 'creation', 'creator'] # figure actions based on data type for name, s_prop in old_spec[1]: # s_prop is a repr() string of a hyperdb type object if s_prop.find('Multilink') == -1: if properties.has_key(name): propnames.append(name) continue tn = '%s_%s'%(cn, name) if properties.has_key(name): # grabe the current values sql = 'select linkid, nodeid from %s'%tn self.sql(sql) rows = self.cursor.fetchall() # drop the old table self.drop_multilink_table_indexes(cn, name) sql = 'drop table %s'%tn self.sql(sql) if properties.has_key(name): # re-create and populate the new table self.create_multilink_table(klass, name) sql = '''insert into %s (linkid, nodeid) values (%s, %s)'''%(tn, self.arg, self.arg) for linkid, nodeid in rows: self.sql(sql, (int(linkid), int(nodeid))) # figure the column names to fetch fetch = ['_%s'%name for name in propnames] # select the data out of the old table fetch.append('id') fetch.append('__retired__') fetchcols = ','.join(fetch) sql = 'select %s from _%s'%(fetchcols, cn) self.sql(sql) # unserialise the old data olddata = [] propnames = propnames + ['id', '__retired__'] cols = [] first = 1 for entry in self.cursor.fetchall(): l = [] olddata.append(l) for i in range(len(propnames)): name = propnames[i] v = entry[i] if name in ('id', '__retired__'): if first: cols.append(name) l.append(int(v)) continue if first: cols.append('_' + name) prop = properties[name] if isinstance(prop, Date) and v is not None: v = date.Date(v) elif isinstance(prop, Interval) and v is not None: v = date.Interval(v) elif isinstance(prop, Password) and v is not None: v = password.Password(encrypted=v) elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None: v = float(v) # convert to new MySQL data type prop = properties[name] if v is not None: e = self.hyperdb_to_sql_value[prop.__class__](v) else: e = None l.append(e) # Intervals store the seconds value too if isinstance(prop, Interval): if first: cols.append('__' + name + '_int__') if v is not None: l.append(v.as_seconds()) else: l.append(e) first = 0 self.drop_class_table_indexes(cn, old_spec[0]) # drop the old table self.sql('drop table _%s'%cn) # create the new table self.create_class_table(klass) # do the insert of the old data args = ','.join([self.arg for x in cols]) cols = ','.join(cols) sql = 'insert into _%s (%s) values (%s)'%(cn, cols, args) for entry in olddata: self.sql(sql, tuple(entry)) # now load up the old journal data to migrate it cols = ','.join('nodeid date tag action params'.split()) sql = 'select %s from %s__journal'%(cols, cn) self.sql(sql) # data conversions olddata = [] for nodeid, journaldate, journaltag, action, params in \ self.cursor.fetchall(): #nodeid = int(nodeid) journaldate = date.Date(journaldate) #params = eval(params) olddata.append((nodeid, journaldate, journaltag, action, params)) # drop journal table and indexes self.drop_journal_table_indexes(cn) sql = 'drop table %s__journal'%cn self.sql(sql) # re-create journal table self.create_journal_table(klass) dc = self.hyperdb_to_sql_value[hyperdb.Date] for nodeid, journaldate, journaltag, action, params in olddata: self.save_journal(cn, cols, nodeid, dc(journaldate), journaltag, action, params) # make sure the normal schema update code doesn't try to # change things self.database_schema['tables'][cn] = klass.schema() def fix_version_2_tables(self): # Convert journal date column to TIMESTAMP, params column to TEXT self._convert_journal_tables() # Convert all String properties to TEXT self._convert_string_properties() def __repr__(self): return ''%id(self) def sql_fetchone(self): return self.cursor.fetchone() def sql_fetchall(self): return self.cursor.fetchall() def sql_index_exists(self, table_name, index_name): self.sql('show index from %s'%table_name) for index in self.cursor.fetchall(): if index[2] == index_name: return 1 return 0 def create_class_table(self, spec, create_sequence=1): cols, mls = self.determine_columns(spec.properties.items()) # add on our special columns cols.append(('id', 'INTEGER PRIMARY KEY')) cols.append(('__retired__', 'INTEGER DEFAULT 0')) # create the base table scols = ','.join(['%s %s'%x for x in cols]) sql = 'create table _%s (%s) type=%s'%(spec.classname, scols, self.mysql_backend) self.sql(sql) self.create_class_table_indexes(spec) return cols, mls def drop_class_table_indexes(self, cn, key): # drop the old table indexes first l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn] if key: l.append('_%s_%s_idx'%(cn, key)) table_name = '_%s'%cn for index_name in l: if not self.sql_index_exists(table_name, index_name): continue index_sql = 'drop index %s on %s'%(index_name, table_name) self.sql(index_sql) def create_journal_table(self, spec): ''' create the journal table for a class given the spec and already-determined cols ''' # journal table cols = ','.join(['%s varchar'%x for x in 'nodeid date tag action params'.split()]) sql = '''create table %s__journal ( nodeid integer, date datetime, tag varchar(255), action varchar(255), params text) type=%s'''%( spec.classname, self.mysql_backend) self.sql(sql) self.create_journal_table_indexes(spec) def drop_journal_table_indexes(self, classname): index_name = '%s_journ_idx'%classname if not self.sql_index_exists('%s__journal'%classname, index_name): return index_sql = 'drop index %s on %s__journal'%(index_name, classname) self.sql(index_sql) def create_multilink_table(self, spec, ml): sql = '''CREATE TABLE `%s_%s` (linkid VARCHAR(255), nodeid VARCHAR(255)) TYPE=%s'''%(spec.classname, ml, self.mysql_backend) self.sql(sql) self.create_multilink_table_indexes(spec, ml) def drop_multilink_table_indexes(self, classname, ml): l = [ '%s_%s_l_idx'%(classname, ml), '%s_%s_n_idx'%(classname, ml) ] table_name = '%s_%s'%(classname, ml) for index_name in l: if not self.sql_index_exists(table_name, index_name): continue sql = 'drop index %s on %s'%(index_name, table_name) self.sql(sql) def drop_class_table_key_index(self, cn, key): table_name = '_%s'%cn index_name = '_%s_%s_idx'%(cn, key) if not self.sql_index_exists(table_name, index_name): return sql = 'drop index %s on %s'%(index_name, table_name) self.sql(sql) # old-skool id generation def newid(self, classname): ''' Generate a new id for the given class ''' # get the next ID - "FOR UPDATE" will lock the row for us sql = 'select num from ids where name=%s FOR UPDATE'%self.arg self.sql(sql, (classname, )) newid = int(self.cursor.fetchone()[0]) # update the counter sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg) vals = (int(newid)+1, classname) self.sql(sql, vals) # return as string return str(newid) def setid(self, classname, setid): ''' Set the id counter: used during import of database We add one to make it behave like the seqeunces in postgres. ''' sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg) vals = (int(setid)+1, classname) self.sql(sql, vals) def clear(self): rdbms_common.Database.clear(self) # set the id counters to 0 (setid adds one) so we start at 1 for cn in self.classes.keys(): self.setid(cn, 0) def create_class(self, spec): rdbms_common.Database.create_class(self, spec) sql = 'insert into ids (name, num) values (%s, %s)' vals = (spec.classname, 1) self.sql(sql, vals) def sql_commit(self): ''' Actually commit to the database. ''' logging.getLogger('hyperdb').info('commit') self.conn.commit() # open a new cursor for subsequent work self.cursor = self.conn.cursor() # make sure we're in a new transaction and not autocommitting self.sql("SET AUTOCOMMIT=0") self.sql("START TRANSACTION") class MysqlClass: # we're overriding this method for ONE missing bit of functionality. # look for "I can't believe it's not a toy RDBMS" below def filter(self, search_matches, filterspec, sort=(None,None), group=(None,None)): '''Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} or None The filter must match all properties specificed - but if the property value to match is a list, any one of the values in the list may match for that property to match. ''' # we can't match anything if search_matches is empty if search_matches == {}: return [] if __debug__: start_t = time.time() cn = self.classname # vars to hold the components of the SQL statement frum = ['_'+cn] # FROM clauses loj = [] # LEFT OUTER JOIN clauses where = [] # WHERE clauses args = [] # *any* positional arguments a = self.db.arg # figure the WHERE clause from the filterspec props = self.getprops() mlfilt = 0 # are we joining with Multilink tables? for k, v in filterspec.items(): propclass = props[k] # now do other where clause stuff if isinstance(propclass, Multilink): mlfilt = 1 tn = '%s_%s'%(cn, k) if v in ('-1', ['-1']): # only match rows that have count(linkid)=0 in the # corresponding multilink table) # "I can't believe it's not a toy RDBMS" # see, even toy RDBMSes like gadfly and sqlite can do # sub-selects... self.db.sql('select nodeid from %s'%tn) s = ','.join([x[0] for x in self.db.sql_fetchall()]) where.append('_%s.id not in (%s)'%(cn, s)) elif isinstance(v, type([])): frum.append(tn) s = ','.join([a for x in v]) where.append('_%s.id=%s.nodeid and %s.linkid in (%s)'%(cn, tn, tn, s)) args = args + v else: frum.append(tn) where.append('_%s.id=%s.nodeid and %s.linkid=%s'%(cn, tn, tn, a)) args.append(v) elif k == 'id': if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s.%s in (%s)'%(cn, k, s)) args = args + v else: where.append('_%s.%s=%s'%(cn, k, a)) args.append(v) elif isinstance(propclass, String): if not isinstance(v, type([])): v = [v] # Quote the bits in the string that need it and then embed # in a "substring" search. Note - need to quote the '%' so # they make it through the python layer happily v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v] # now add to the where clause where.append('(' +' and '.join(["_%s._%s LIKE '%s'"%(cn, k, s) for s in v]) +')') # note: args are embedded in the query string now elif isinstance(propclass, Link): if isinstance(v, type([])): d = {} for entry in v: if entry == '-1': entry = None d[entry] = entry l = [] if d.has_key(None) or not d: del d[None] l.append('_%s._%s is NULL'%(cn, k)) if d: v = d.keys() s = ','.join([a for x in v]) l.append('(_%s._%s in (%s))'%(cn, k, s)) args = args + v if l: where.append('(' + ' or '.join(l) +')') else: if v in ('-1', None): v = None where.append('_%s._%s is NULL'%(cn, k)) else: where.append('_%s._%s=%s'%(cn, k, a)) args.append(v) elif isinstance(propclass, Date): dc = self.db.hyperdb_to_sql_value[hyperdb.Date] if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s._%s in (%s)'%(cn, k, s)) args = args + [dc(date.Date(x)) for x in v] else: try: # Try to filter on range of dates date_rng = propclass.range_from_raw (v, self.db) if date_rng.from_value: where.append('_%s._%s >= %s'%(cn, k, a)) args.append(dc(date_rng.from_value)) if date_rng.to_value: where.append('_%s._%s <= %s'%(cn, k, a)) args.append(dc(date_rng.to_value)) except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(propclass, Interval): # filter using the ___int__ column if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s.__%s_int__ in (%s)'%(cn, k, s)) args = args + [date.Interval(x).as_seconds() for x in v] else: try: # Try to filter on range of intervals date_rng = Range(v, date.Interval) if date_rng.from_value: where.append('_%s.__%s_int__ >= %s'%(cn, k, a)) args.append(date_rng.from_value.as_seconds()) if date_rng.to_value: where.append('_%s.__%s_int__ <= %s'%(cn, k, a)) args.append(date_rng.to_value.as_seconds()) except ValueError: # If range creation fails - ignore that search parameter pass else: if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s._%s in (%s)'%(cn, k, s)) args = args + v else: where.append('_%s._%s=%s'%(cn, k, a)) args.append(v) # don't match retired nodes where.append('_%s.__retired__ <> 1'%cn) # add results of full text search if search_matches is not None: v = search_matches.keys() s = ','.join([a for x in v]) where.append('_%s.id in (%s)'%(cn, s)) args = args + v # sanity check: sorting *and* grouping on the same property? if group[1] == sort[1]: sort = (None, None) # "grouping" is just the first-order sorting in the SQL fetch orderby = [] ordercols = [] mlsort = [] for sortby in group, sort: sdir, prop = sortby if sdir and prop: if isinstance(props[prop], Multilink): mlsort.append(sortby) continue elif isinstance(props[prop], Interval): # use the int column for sorting o = '__'+prop+'_int__' ordercols.append(o) elif isinstance(props[prop], Link): # determine whether the linked Class has an order property lcn = props[prop].classname link = self.db.classes[lcn] o = '_%s._%s'%(cn, prop) if link.getprops().has_key('order'): tn = '_' + lcn loj.append('LEFT OUTER JOIN %s on %s=%s.id'%(tn, o, tn)) ordercols.append(tn + '._order') o = tn + '._order' else: ordercols.append(o) elif prop == 'id': o = '_%s.id'%cn else: o = '_%s._%s'%(cn, prop) ordercols.append(o) if sdir == '-': o += ' desc' orderby.append(o) # construct the SQL frum = ','.join(frum) if where: where = ' where ' + (' and '.join(where)) else: where = '' if mlfilt: # we're joining tables on the id, so we will get dupes if we # don't distinct() cols = ['distinct(_%s.id)'%cn] else: cols = ['_%s.id'%cn] if orderby: cols = cols + ordercols order = ' order by %s'%(','.join(orderby)) else: order = '' cols = ','.join(cols) loj = ' '.join(loj) sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order) args = tuple(args) self.db.sql(sql, args) l = self.db.cursor.fetchall() # return the IDs (the first column) # XXX numeric ids l = [str(row[0]) for row in l] if not mlsort: if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return l # ergh. someone wants to sort by a multilink. r = [] for id in l: m = [] for ml in mlsort: m.append(self.get(id, ml[1])) r.append((id, m)) i = 0 for sortby in mlsort: def sortfun(a, b, dir=sortby[i], i=i): if dir == '-': return cmp(b[1][i], a[1][i]) else: return cmp(a[1][i], b[1][i]) r.sort(sortfun) i += 1 r = [i[0] for i in r] if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return r class Class(MysqlClass, rdbms_common.Class): pass class IssueClass(MysqlClass, rdbms_common.IssueClass): pass class FileClass(MysqlClass, rdbms_common.FileClass): pass # vim: set et sts=4 sw=4 : code-963003d2ffdd/roundup/backends/back_postgresql.py0000644000000000000000000001712010344271362020740 0ustar 00000000000000#$Id: back_postgresql.py,v 1.30 2005-05-02 05:48:59 richard Exp $ # # Copyright (c) 2003 Martynas Sklyzmantas, Andrey Lebedev # # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # '''Postgresql backend via psycopg for Roundup.''' __docformat__ = 'restructuredtext' import os, shutil, popen2, time import psycopg import logging from roundup import hyperdb, date from roundup.backends import rdbms_common def connection_dict(config, dbnamestr=None): ''' read_default_group is MySQL-specific, ignore it ''' d = rdbms_common.connection_dict(config, dbnamestr) if d.has_key('read_default_group'): del d['read_default_group'] if d.has_key('read_default_file'): del d['read_default_file'] return d def db_create(config): """Clear all database contents and drop database itself""" command = 'CREATE DATABASE %s'%config.RDBMS_NAME logging.getLogger('hyperdb').info(command) db_command(config, command) def db_nuke(config, fail_ok=0): """Clear all database contents and drop database itself""" command = 'DROP DATABASE %s'% config.RDBMS_NAME logging.getLogger('hyperdb').info(command) db_command(config, command) if os.path.exists(config.DATABASE): shutil.rmtree(config.DATABASE) def db_command(config, command): '''Perform some sort of database-level command. Retry 10 times if we fail by conflicting with another user. ''' template1 = connection_dict(config) template1['database'] = 'template1' try: conn = psycopg.connect(**template1) except psycopg.OperationalError, message: raise hyperdb.DatabaseError, message conn.set_isolation_level(0) cursor = conn.cursor() try: for n in range(10): if pg_command(cursor, command): return finally: conn.close() raise RuntimeError, '10 attempts to create database failed' def pg_command(cursor, command): '''Execute the postgresql command, which may be blocked by some other user connecting to the database, and return a true value if it succeeds. ''' try: cursor.execute(command) except psycopg.ProgrammingError, err: response = str(err).split('\n')[0] if response.find('FATAL') != -1: raise RuntimeError, response elif response.find('ERROR') != -1: if response.find('is being accessed by other users') == -1: raise RuntimeError, response time.sleep(1) return 0 return 1 def db_exists(config): """Check if database already exists""" db = connection_dict(config, 'database') try: conn = psycopg.connect(**db) conn.close() return 1 except: return 0 class Database(rdbms_common.Database): arg = '%s' # used by some code to switch styles of query implements_intersect = 1 def sql_open_connection(self): db = connection_dict(self.config, 'database') logging.getLogger('hyperdb').info('open database %r'%db['database']) try: conn = psycopg.connect(**db) except psycopg.OperationalError, message: raise hyperdb.DatabaseError, message cursor = conn.cursor() return (conn, cursor) def open_connection(self): if not db_exists(self.config): db_create(self.config) self.conn, self.cursor = self.sql_open_connection() try: self.load_dbschema() except psycopg.ProgrammingError, message: if str(message).find('"schema" does not exist') == -1: raise self.rollback() self.init_dbschema() self.sql("CREATE TABLE schema (schema TEXT)") self.sql("CREATE TABLE dual (dummy integer)") self.sql("insert into dual values (1)") self.create_version_2_tables() def create_version_2_tables(self): # OTK store self.sql('''CREATE TABLE otks (otk_key VARCHAR(255), otk_value TEXT, otk_time REAL)''') self.sql('CREATE INDEX otks_key_idx ON otks(otk_key)') # Sessions store self.sql('''CREATE TABLE sessions ( session_key VARCHAR(255), session_time REAL, session_value TEXT)''') self.sql('''CREATE INDEX sessions_key_idx ON sessions(session_key)''') # full-text indexing store self.sql('CREATE SEQUENCE ___textids_ids') self.sql('''CREATE TABLE __textids ( _textid integer primary key, _class VARCHAR(255), _itemid VARCHAR(255), _prop VARCHAR(255))''') self.sql('''CREATE TABLE __words (_word VARCHAR(30), _textid integer)''') self.sql('CREATE INDEX words_word_idx ON __words(_word)') def fix_version_2_tables(self): # Convert journal date column to TIMESTAMP, params column to TEXT self._convert_journal_tables() # Convert all String properties to TEXT self._convert_string_properties() # convert session / OTK *_time columns to REAL for name in ('otk', 'session'): self.sql('drop index %ss_key_idx'%name) self.sql('drop table %ss'%name) self.sql('''CREATE TABLE %ss (%s_key VARCHAR(255), %s_value VARCHAR(255), %s_time REAL)'''%(name, name, name, name)) self.sql('CREATE INDEX %ss_key_idx ON %ss(%s_key)'%(name, name, name)) def fix_version_3_tables(self): rdbms_common.Database.fix_version_3_tables(self) self.sql('''CREATE INDEX words_both_idx ON public.__words USING btree (_word, _textid)''') def add_actor_column(self): # update existing tables to have the new actor column tables = self.database_schema['tables'] for name in tables.keys(): self.sql('ALTER TABLE _%s add __actor VARCHAR(255)'%name) def __repr__(self): return '' % id(self) def sql_stringquote(self, value): ''' psycopg.QuotedString returns a "buffer" object with the single-quotes around it... ''' return str(psycopg.QuotedString(str(value)))[1:-1] def sql_index_exists(self, table_name, index_name): sql = 'select count(*) from pg_indexes where ' \ 'tablename=%s and indexname=%s'%(self.arg, self.arg) self.sql(sql, (table_name, index_name)) return self.cursor.fetchone()[0] def create_class_table(self, spec, create_sequence=1): if create_sequence: sql = 'CREATE SEQUENCE _%s_ids'%spec.classname self.sql(sql) return rdbms_common.Database.create_class_table(self, spec) def drop_class_table(self, cn): sql = 'drop table _%s'%cn self.sql(sql) sql = 'drop sequence _%s_ids'%cn self.sql(sql) def newid(self, classname): sql = "select nextval('_%s_ids') from dual"%classname self.sql(sql) return self.cursor.fetchone()[0] def setid(self, classname, setid): sql = "select setval('_%s_ids', %s) from dual"%(classname, int(setid)) self.sql(sql) def clear(self): rdbms_common.Database.clear(self) # reset the sequences for cn in self.classes.keys(): self.cursor.execute('DROP SEQUENCE _%s_ids'%cn) self.cursor.execute('CREATE SEQUENCE _%s_ids'%cn) class Class(rdbms_common.Class): pass class IssueClass(rdbms_common.IssueClass): pass class FileClass(rdbms_common.FileClass): pass code-963003d2ffdd/roundup/backends/back_sqlite.py0000644000000000000000000003222310344271362020037 0ustar 00000000000000# $Id: back_sqlite.py,v 1.43 2005-06-08 03:41:21 anthonybaxter Exp $ '''Implements a backend for SQLite. See https://pysqlite.sourceforge.net/ for pysqlite info NOTE: we use the rdbms_common table creation methods which define datatypes for the columns, but sqlite IGNORES these specifications. ''' __docformat__ = 'restructuredtext' import os, base64, marshal, shutil, time, logging from roundup import hyperdb, date, password from roundup.backends import rdbms_common import sqlite def db_exists(config): return os.path.exists(os.path.join(config.DATABASE, 'db')) def db_nuke(config): shutil.rmtree(config.DATABASE) class Database(rdbms_common.Database): # char to use for positional arguments arg = '%s' # used by some code to switch styles of query implements_intersect = 1 hyperdb_to_sql_datatypes = { hyperdb.String : 'VARCHAR(255)', hyperdb.Date : 'VARCHAR(30)', hyperdb.Link : 'INTEGER', hyperdb.Interval : 'VARCHAR(255)', hyperdb.Password : 'VARCHAR(255)', hyperdb.Boolean : 'BOOLEAN', hyperdb.Number : 'REAL', } hyperdb_to_sql_value = { hyperdb.String : str, hyperdb.Date : lambda x: x.serialise(), hyperdb.Link : int, hyperdb.Interval : str, hyperdb.Password : str, hyperdb.Boolean : int, hyperdb.Number : lambda x: x, hyperdb.Multilink : lambda x: x, # used in journal marshalling } sql_to_hyperdb_value = { hyperdb.String : str, hyperdb.Date : lambda x: date.Date(str(x)), hyperdb.Link : str, # XXX numeric ids hyperdb.Interval : date.Interval, hyperdb.Password : lambda x: password.Password(encrypted=x), hyperdb.Boolean : int, hyperdb.Number : rdbms_common._num_cvt, hyperdb.Multilink : lambda x: x, # used in journal marshalling } def sqlite_busy_handler(self, data, table, count): """invoked whenever SQLite tries to access a database that is locked""" if count == 1: # use a 30 second timeout (extraordinarily generous) # for handling locked database self._busy_handler_endtime = time.time() + 30 elif time.time() > self._busy_handler_endtime: # timeout expired - no more retries return 0 # sleep adaptively as retry count grows, # starting from about half a second time_to_sleep = 0.01 * (2 << min(5, count)) time.sleep(time_to_sleep) return 1 def sql_open_connection(self): '''Open a standard, non-autocommitting connection. pysqlite will automatically BEGIN TRANSACTION for us. ''' # make sure the database directory exists # database itself will be created by sqlite if needed if not os.path.isdir(self.config.DATABASE): os.makedirs(self.config.DATABASE) db = os.path.join(self.config.DATABASE, 'db') logging.getLogger('hyperdb').info('open database %r'%db) conn = sqlite.connect(db=db) # set a 30 second timeout (extraordinarily generous) for handling # locked database conn.db.sqlite_busy_handler(self.sqlite_busy_handler) cursor = conn.cursor() return (conn, cursor) def open_connection(self): # ensure files are group readable and writable os.umask(0002) (self.conn, self.cursor) = self.sql_open_connection() try: self.load_dbschema() except sqlite.DatabaseError, error: if str(error) != 'no such table: schema': raise self.init_dbschema() self.sql('create table schema (schema varchar)') self.sql('create table ids (name varchar, num integer)') self.sql('create index ids_name_idx on ids(name)') self.create_version_2_tables() def create_version_2_tables(self): self.sql('create table otks (otk_key varchar, ' 'otk_value varchar, otk_time integer)') self.sql('create index otks_key_idx on otks(otk_key)') self.sql('create table sessions (session_key varchar, ' 'session_time integer, session_value varchar)') self.sql('create index sessions_key_idx on ' 'sessions(session_key)') # full-text indexing store self.sql('CREATE TABLE __textids (_class varchar, ' '_itemid varchar, _prop varchar, _textid integer primary key) ') self.sql('CREATE TABLE __words (_word varchar, ' '_textid integer)') self.sql('CREATE INDEX words_word_ids ON __words(_word)') sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg) self.sql(sql, ('__textids', 1)) def add_new_columns_v2(self): # update existing tables to have the new actor column tables = self.database_schema['tables'] for classname, spec in self.classes.items(): if tables.has_key(classname): dbspec = tables[classname] self.update_class(spec, dbspec, force=1, adding_v2=1) # we've updated - don't try again tables[classname] = spec.schema() def fix_version_3_tables(self): # NOOP - no restriction on column length here pass def update_class(self, spec, old_spec, force=0, adding_v2=0): ''' Determine the differences between the current spec and the database version of the spec, and update where necessary. If 'force' is true, update the database anyway. SQLite doesn't have ALTER TABLE, so we have to copy and regenerate the tables with the new schema. ''' new_has = spec.properties.has_key new_spec = spec.schema() new_spec[1].sort() old_spec[1].sort() if not force and new_spec == old_spec: # no changes return 0 logging.getLogger('hyperdb').info('update_class %s'%spec.classname) # detect multilinks that have been removed, and drop their table old_has = {} for name, prop in old_spec[1]: old_has[name] = 1 if new_has(name) or not isinstance(prop, hyperdb.Multilink): continue # it's a multilink, and it's been removed - drop the old # table. First drop indexes. self.drop_multilink_table_indexes(spec.classname, name) sql = 'drop table %s_%s'%(spec.classname, prop) self.sql(sql) old_has = old_has.has_key # now figure how we populate the new table if adding_v2: fetch = ['_activity', '_creation', '_creator'] else: fetch = ['_actor', '_activity', '_creation', '_creator'] properties = spec.getprops() for propname,x in new_spec[1]: prop = properties[propname] if isinstance(prop, hyperdb.Multilink): if not old_has(propname): # we need to create the new table self.create_multilink_table(spec, propname) elif force: tn = '%s_%s'%(spec.classname, propname) # grabe the current values sql = 'select linkid, nodeid from %s'%tn self.sql(sql) rows = self.cursor.fetchall() # drop the old table self.drop_multilink_table_indexes(spec.classname, propname) sql = 'drop table %s'%tn self.sql(sql) # re-create and populate the new table self.create_multilink_table(spec, propname) sql = '''insert into %s (linkid, nodeid) values (%s, %s)'''%(tn, self.arg, self.arg) for linkid, nodeid in rows: self.sql(sql, (int(linkid), int(nodeid))) elif old_has(propname): # we copy this col over from the old table fetch.append('_'+propname) # select the data out of the old table fetch.append('id') fetch.append('__retired__') fetchcols = ','.join(fetch) cn = spec.classname sql = 'select %s from _%s'%(fetchcols, cn) self.sql(sql) olddata = self.cursor.fetchall() # TODO: update all the other index dropping code self.drop_class_table_indexes(cn, old_spec[0]) # drop the old table self.sql('drop table _%s'%cn) # create the new table self.create_class_table(spec) if olddata: inscols = ['id', '_actor', '_activity', '_creation', '_creator'] for propname,x in new_spec[1]: prop = properties[propname] if isinstance(prop, hyperdb.Multilink): continue elif isinstance(prop, hyperdb.Interval): inscols.append('_'+propname) inscols.append('__'+propname+'_int__') elif old_has(propname): # we copy this col over from the old table inscols.append('_'+propname) # do the insert of the old data - the new columns will have # NULL values args = ','.join([self.arg for x in inscols]) cols = ','.join(inscols) sql = 'insert into _%s (%s) values (%s)'%(cn, cols, args) for entry in olddata: d = [] for name in inscols: # generate the new value for the Interval int column if name.endswith('_int__'): name = name[2:-6] if entry.has_key(name): v = hyperdb.Interval(entry[name]).as_seconds() else: v = None elif entry.has_key(name): v = entry[name] else: v = None d.append(v) self.sql(sql, tuple(d)) return 1 def sql_close(self): ''' Squash any error caused by us already having closed the connection. ''' try: self.conn.close() except sqlite.ProgrammingError, value: if str(value) != 'close failed - Connection is closed.': raise def sql_rollback(self): ''' Squash any error caused by us having closed the connection (and therefore not having anything to roll back) ''' try: self.conn.rollback() except sqlite.ProgrammingError, value: if str(value) != 'rollback failed - Connection is closed.': raise def __repr__(self): return ''%id(self) def sql_commit(self): ''' Actually commit to the database. Ignore errors if there's nothing to commit. ''' try: self.conn.commit() except sqlite.DatabaseError, error: if str(error) != 'cannot commit - no transaction is active': raise # open a new cursor for subsequent work self.cursor = self.conn.cursor() def sql_index_exists(self, table_name, index_name): self.sql('pragma index_list(%s)'%table_name) for entry in self.cursor.fetchall(): if entry[1] == index_name: return 1 return 0 # old-skool id generation def newid(self, classname): ''' Generate a new id for the given class ''' # get the next ID sql = 'select num from ids where name=%s'%self.arg self.sql(sql, (classname, )) newid = int(self.cursor.fetchone()[0]) # update the counter sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg) vals = (int(newid)+1, classname) self.sql(sql, vals) # return as string return str(newid) def setid(self, classname, setid): ''' Set the id counter: used during import of database We add one to make it behave like the sequences in postgres. ''' sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg) vals = (int(setid)+1, classname) self.sql(sql, vals) def clear(self): rdbms_common.Database.clear(self) # set the id counters to 0 (setid adds one) so we start at 1 for cn in self.classes.keys(): self.setid(cn, 0) def create_class(self, spec): rdbms_common.Database.create_class(self, spec) sql = 'insert into ids (name, num) values (%s, %s)' vals = (spec.classname, 1) self.sql(sql, vals) class sqliteClass: def filter(self, search_matches, filterspec, sort=(None,None), group=(None,None)): ''' If there's NO matches to a fetch, sqlite returns NULL instead of nothing ''' return filter(None, rdbms_common.Class.filter(self, search_matches, filterspec, sort=sort, group=group)) class Class(sqliteClass, rdbms_common.Class): pass class IssueClass(sqliteClass, rdbms_common.IssueClass): pass class FileClass(sqliteClass, rdbms_common.FileClass): pass # vim: set et sts=4 sw=4 : code-963003d2ffdd/roundup/backends/back_tsearch2.py0000644000000000000000000001555610344271362020263 0ustar 00000000000000#$Id: back_tsearch2.py,v 1.9 2005-01-08 16:16:59 jlgijsbers Exp $ # Note: this backend is EXPERIMENTAL. Do not use if you value your data. import re import psycopg from roundup import hyperdb from roundup.support import ensureParentsExist from roundup.backends import back_postgresql, tsearch2_setup, indexer_rdbms from roundup.backends.back_postgresql import db_create, db_nuke, db_command from roundup.backends.back_postgresql import pg_command, db_exists, Class, IssueClass, FileClass from roundup.backends.indexer_common import _isLink, Indexer # XXX: Should probably be on the Class class. def _indexedProps(spec): """Get a list of properties to be indexed on 'spec'.""" return [prop for prop, propclass in spec.getprops().items() if isinstance(propclass, hyperdb.String) and propclass.indexme] def _getQueryDict(spec): """Get a convenience dictionary for creating tsearch2 indexes.""" query_dict = {'classname': spec.classname, 'indexedColumns': ['_' + prop for prop in _indexedProps(spec)]} query_dict['tablename'] = "_%(classname)s" % query_dict query_dict['triggername'] = "%(tablename)s_tsvectorupdate" % query_dict return query_dict class Database(back_postgresql.Database): def __init__(self, config, journaltag=None): back_postgresql.Database.__init__(self, config, journaltag) self.indexer = Indexer(self) def create_version_2_tables(self): back_postgresql.Database.create_version_2_tables(self) tsearch2_setup.setup(self.cursor) def create_class_table_indexes(self, spec): back_postgresql.Database.create_class_table_indexes(self, spec) self.cursor.execute("""CREATE INDEX _%(classname)s_idxFTI_idx ON %(tablename)s USING gist(idxFTI);""" % _getQueryDict(spec)) self.create_tsearch2_trigger(spec) def create_tsearch2_trigger(self, spec): d = _getQueryDict(spec) if d['indexedColumns']: d['joined'] = " || ' ' ||".join(d['indexedColumns']) query = """UPDATE %(tablename)s SET idxFTI = to_tsvector('default', %(joined)s)""" % d self.cursor.execute(query) d['joined'] = ", ".join(d['indexedColumns']) query = """CREATE TRIGGER %(triggername)s BEFORE UPDATE OR INSERT ON %(tablename)s FOR EACH ROW EXECUTE PROCEDURE tsearch2(idxFTI, %(joined)s);""" % d self.cursor.execute(query) def drop_tsearch2_trigger(self, spec): # Check whether the trigger exists before trying to drop it. query_dict = _getQueryDict(spec) self.sql("""SELECT tgname FROM pg_catalog.pg_trigger WHERE tgname = '%(triggername)s'""" % query_dict) if self.cursor.fetchall(): self.sql("""DROP TRIGGER %(triggername)s ON %(tablename)s""" % query_dict) def update_class(self, spec, old_spec, force=0): result = back_postgresql.Database.update_class(self, spec, old_spec, force) # Drop trigger... self.drop_tsearch2_trigger(spec) # and recreate if necessary. self.create_tsearch2_trigger(spec) return result def determine_all_columns(self, spec): cols, mls = back_postgresql.Database.determine_all_columns(self, spec) cols.append(('idxFTI', 'tsvector')) return cols, mls class Indexer(Indexer): def __init__(self, db): self.db = db # This indexer never needs to reindex. def should_reindex(self): return 0 def getHits(self, search_terms, klass): return self.find(search_terms, klass) def find(self, search_terms, klass): if not search_terms: return None hits = self.tsearchQuery(klass.classname, search_terms) designator_propname = {} for nm, propclass in klass.getprops().items(): if _isLink(propclass): hits.extend(self.tsearchQuery(propclass.classname, search_terms)) return hits def tsearchQuery(self, classname, search_terms): query = """SELECT id FROM _%(classname)s WHERE idxFTI @@ to_tsquery('default', '%(terms)s')""" query = query % {'classname': classname, 'terms': ' & '.join(search_terms)} self.db.cursor.execute(query) klass = self.db.getclass(classname) nodeids = [str(row[0]) for row in self.db.cursor.fetchall()] # filter out files without text/plain mime type # XXX: files without text/plain shouldn't be indexed at all, we # should take care of this in the trigger if klass.getprops().has_key('type'): nodeids = [nodeid for nodeid in nodeids if klass.get(nodeid, 'type') == 'text/plain'] # XXX: We haven't implemented property-level search, so I'm just faking # it here with a property named 'XXX'. We still need to fix the other # backends and indexer_common.Indexer.search to only want to unpack two # values. return [(classname, nodeid, 'XXX') for nodeid in nodeids] # These only exist to satisfy the interface that's expected from indexers. def force_reindex(self): pass def add_text(self, identifier, text, mime_type=None): pass def close(self): pass class FileClass(hyperdb.FileClass, Class): '''This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. However, this implementation just stores it in the hyperdb. ''' def __init__(self, db, classname, **properties): '''The newly-created class automatically includes the "content" property., ''' properties['content'] = hyperdb.String(indexme='yes') Class.__init__(self, db, classname, **properties) default_mime_type = 'text/plain' def create(self, **propvalues): # figure the mime type if self.getprops().has_key('type') and not propvalues.get('type'): propvalues['type'] = self.default_mime_type return Class.create(self, **propvalues) def export_files(self, dirname, nodeid): dest = self.exportFilename(dirname, nodeid) ensureParentsExist(dest) fp = open(dest, "w") fp.write(self.get(nodeid, "content", default='')) fp.close() def import_files(self, dirname, nodeid): source = self.exportFilename(dirname, nodeid) fp = open(source, "r") # Use Database.setnode instead of self.set or self.set_inner here, as # Database.setnode doesn't update the "activity" or "actor" properties. self.db.setnode(self.classname, nodeid, values={'content': fp.read()}) fp.close() code-963003d2ffdd/roundup/backends/blobfiles.py0000644000000000000000000001354310344271362017523 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: blobfiles.py,v 1.19 2005-06-08 03:35:18 anthonybaxter Exp $ '''This module exports file storage for roundup backends. Files are stored into a directory hierarchy. ''' __docformat__ = 'restructuredtext' import os def files_in_dir(dir): if not os.path.exists(dir): return 0 num_files = 0 for dir_entry in os.listdir(dir): full_filename = os.path.join(dir,dir_entry) if os.path.isfile(full_filename): num_files = num_files + 1 elif os.path.isdir(full_filename): num_files = num_files + files_in_dir(full_filename) return num_files class FileStorage: """Store files in some directory structure""" def subdirFilename(self, classname, nodeid, property=None): """Determine what the filename and subdir for nodeid + classname is.""" if property: name = '%s%s.%s'%(classname, nodeid, property) else: # roundupdb.FileClass never specified the property name, so don't # include it name = '%s%s'%(classname, nodeid) # have a separate subdir for every thousand messages subdir = str(int(nodeid) / 1000) return os.path.join(subdir, name) def filename(self, classname, nodeid, property=None, create=0): '''Determine what the filename for the given node and optionally property is. Try a variety of different filenames - the file could be in the usual place, or it could be in a temp file pre-commit *or* it could be in an old-style, backwards-compatible flat directory. ''' filename = os.path.join(self.dir, 'files', classname, self.subdirFilename(classname, nodeid, property)) if create or os.path.exists(filename): return filename # try .tmp filename = filename + '.tmp' if os.path.exists(filename): return filename # ok, try flat (very old-style) if property: filename = os.path.join(self.dir, 'files', '%s%s.%s'%(classname, nodeid, property)) else: filename = os.path.join(self.dir, 'files', '%s%s'%(classname, nodeid)) if os.path.exists(filename): return filename # file just ain't there raise IOError('content file for %s not found'%filename) def storefile(self, classname, nodeid, property, content): '''Store the content of the file in the database. The property may be None, in which case the filename does not indicate which property is being saved. ''' # determine the name of the file to write to name = self.filename(classname, nodeid, property, create=1) # make sure the file storage dir exists if not os.path.exists(os.path.dirname(name)): os.makedirs(os.path.dirname(name)) # save to a temp file name = name + '.tmp' # make sure we don't register the rename action more than once if not os.path.exists(name): # save off the rename action self.transactions.append((self.doStoreFile, (classname, nodeid, property))) open(name, 'wb').write(content) def getfile(self, classname, nodeid, property): '''Get the content of the file in the database. ''' filename = self.filename(classname, nodeid, property) f = open(filename, 'rb') try: # snarf the contents and make sure we close the file return f.read() finally: f.close() def numfiles(self): '''Get number of files in storage, even across subdirectories. ''' files_dir = os.path.join(self.dir, 'files') return files_in_dir(files_dir) def doStoreFile(self, classname, nodeid, property, **databases): '''Store the file as part of a transaction commit. ''' # determine the name of the file to write to name = self.filename(classname, nodeid, property) # the file is currently ".tmp" - move it to its real name to commit if name.endswith('.tmp'): # creation dstname = os.path.splitext(name)[0] else: # edit operation dstname = name name = name + '.tmp' # content is being updated (and some platforms, eg. win32, won't # let us rename over the top of the old file) if os.path.exists(dstname): os.remove(dstname) os.rename(name, dstname) # return the classname, nodeid so we reindex this content return (classname, nodeid) def rollbackStoreFile(self, classname, nodeid, property, **databases): '''Remove the temp file as a part of a rollback ''' # determine the name of the file to delete name = self.filename(classname, nodeid, property) if not name.endswith('.tmp'): name += '.tmp' os.remove(name) # vim: set filetype=python ts=4 sw=4 et si code-963003d2ffdd/roundup/backends/indexer_common.py0000644000000000000000000000631010344271362020562 0ustar 00000000000000#$Id: indexer_common.py,v 1.4 2005-01-08 16:16:59 jlgijsbers Exp $ import re from roundup import hyperdb stopwords = [ "A", "AND", "ARE", "AS", "AT", "BE", "BUT", "BY", "FOR", "IF", "IN", "INTO", "IS", "IT", "NO", "NOT", "OF", "ON", "OR", "SUCH", "THAT", "THE", "THEIR", "THEN", "THERE", "THESE", "THEY", "THIS", "TO", "WAS", "WILL", "WITH" ] is_stopword = {} for word in stopwords: is_stopword[word] = None is_stopword = is_stopword.has_key def _isLink(propclass): return (isinstance(propclass, hyperdb.Link) or isinstance(propclass, hyperdb.Multilink)) class Indexer: def getHits(self, search_terms, klass): return self.find(search_terms) def search(self, search_terms, klass, ignore={}): '''Display search results looking for [search, terms] associated with the hyperdb Class "klass". Ignore hits on {class: property}. "dre" is a helper, not an argument. ''' # do the index lookup hits = self.getHits(search_terms, klass) if not hits: return {} designator_propname = {} for nm, propclass in klass.getprops().items(): if _isLink(propclass): designator_propname[propclass.classname] = nm # build a dictionary of nodes and their associated messages # and files nodeids = {} # this is the answer propspec = {} # used to do the klass.find for propname in designator_propname.values(): propspec[propname] = {} # used as a set (value doesn't matter) for classname, nodeid, property in hits: # skip this result if we don't care about this class/property if ignore.has_key((classname, property)): continue # if it's a property on klass, it's easy if classname == klass.classname: if not nodeids.has_key(nodeid): nodeids[nodeid] = {} continue # make sure the class is a linked one, otherwise ignore if not designator_propname.has_key(classname): continue # it's a linked class - set up to do the klass.find linkprop = designator_propname[classname] # eg, msg -> messages propspec[linkprop][nodeid] = 1 # retain only the meaningful entries for propname, idset in propspec.items(): if not idset: del propspec[propname] # klass.find tells me the klass nodeids the linked nodes relate to for resid in klass.find(**propspec): resid = str(resid) if not nodeids.has_key(id): nodeids[resid] = {} node_dict = nodeids[resid] # now figure out where it came from for linkprop in propspec.keys(): for nodeid in klass.get(resid, linkprop): if propspec[linkprop].has_key(nodeid): # OK, this node[propname] has a winner if not node_dict.has_key(linkprop): node_dict[linkprop] = [nodeid] else: node_dict[linkprop].append(nodeid) return nodeids code-963003d2ffdd/roundup/backends/indexer_dbm.py0000644000000000000000000002404210344271362020036 0ustar 00000000000000# # This module is derived from the module described at: # http://gnosis.cx/publish/programming/charming_python_15.txt # # Author: David Mertz (mertz@gnosis.cx) # Thanks to: Pat Knight (p.knight@ktgroup.co.uk) # Gregory Popovitch (greg@gpy.com) # # The original module was released under this license, and remains under # it: # # This file is released to the public domain. I (dqm) would # appreciate it if you choose to keep derived works under terms # that promote freedom, but obviously am giving up any rights # to compel such. # #$Id: indexer_dbm.py,v 1.6 2005-04-28 00:21:42 richard Exp $ '''This module provides an indexer class, RoundupIndexer, that stores text indices in a roundup instance. This class makes searching the content of messages, string properties and text files possible. ''' __docformat__ = 'restructuredtext' import os, shutil, re, mimetypes, marshal, zlib, errno from roundup.hyperdb import Link, Multilink from roundup.backends.indexer_common import Indexer, is_stopword class Indexer(Indexer): '''Indexes information from roundup's hyperdb to allow efficient searching. Three structures are created by the indexer:: files {identifier: (fileid, wordcount)} words {word: {fileid: count}} fileids {fileid: identifier} where identifier is (classname, nodeid, propertyname) ''' def __init__(self, db): self.indexdb_path = os.path.join(db.config.DATABASE, 'indexes') self.indexdb = os.path.join(self.indexdb_path, 'index.db') self.reindex = 0 self.quiet = 9 self.changed = 0 # see if we need to reindex because of a change in code version = os.path.join(self.indexdb_path, 'version') if (not os.path.exists(self.indexdb_path) or not os.path.exists(version)): # for now the file itself is a flag self.force_reindex() elif os.path.exists(version): version = open(version).read() # check the value and reindex if it's not the latest if version.strip() != '1': self.force_reindex() def force_reindex(self): '''Force a reindex condition ''' if os.path.exists(self.indexdb_path): shutil.rmtree(self.indexdb_path) os.makedirs(self.indexdb_path) os.chmod(self.indexdb_path, 0775) open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n') self.reindex = 1 self.changed = 1 def should_reindex(self): '''Should we reindex? ''' return self.reindex def add_text(self, identifier, text, mime_type='text/plain'): '''Add some text associated with the (classname, nodeid, property) identifier. ''' # make sure the index is loaded self.load_index() # remove old entries for this identifier if self.files.has_key(identifier): self.purge_entry(identifier) # split into words words = self.splitter(text, mime_type) # Find new file index, and assign it to identifier # (_TOP uses trick of negative to avoid conflict with file index) self.files['_TOP'] = (self.files['_TOP'][0]-1, None) file_index = abs(self.files['_TOP'][0]) self.files[identifier] = (file_index, len(words)) self.fileids[file_index] = identifier # find the unique words filedict = {} for word in words: if is_stopword(word): continue if filedict.has_key(word): filedict[word] = filedict[word]+1 else: filedict[word] = 1 # now add to the totals for word in filedict.keys(): # each word has a dict of {identifier: count} if self.words.has_key(word): entry = self.words[word] else: # new word entry = {} self.words[word] = entry # make a reference to the file for this word entry[file_index] = filedict[word] # save needed self.changed = 1 def splitter(self, text, ftype): '''Split the contents of a text string into a list of 'words' ''' if ftype == 'text/plain': words = self.text_splitter(text) else: return [] return words def text_splitter(self, text): """Split text/plain string into a list of words """ # case insensitive text = str(text).upper() # Split the raw text, losing anything longer than 25 characters # since that'll be gibberish (encoded text or somesuch) or shorter # than 3 characters since those short words appear all over the # place return re.findall(r'\b\w{2,25}\b', text) # we override this to ignore not 2 < word < 25 and also to fix a bug - # the (fail) case. def find(self, wordlist): '''Locate files that match ALL the words in wordlist ''' if not hasattr(self, 'words'): self.load_index() self.load_index(wordlist=wordlist) entries = {} hits = None for word in wordlist: if not 2 < len(word) < 25: # word outside the bounds of what we index - ignore continue word = word.upper() entry = self.words.get(word) # For each word, get index entries[word] = entry # of matching files if not entry: # Nothing for this one word (fail) return {} if hits is None: hits = {} for k in entry.keys(): if not self.fileids.has_key(k): raise ValueError, 'Index is corrupted: re-generate it' hits[k] = self.fileids[k] else: # Eliminate hits for every non-match for fileid in hits.keys(): if not entry.has_key(fileid): del hits[fileid] if hits is None: return {} return hits.values() segments = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_-!" def load_index(self, reload=0, wordlist=None): # Unless reload is indicated, do not load twice if self.index_loaded() and not reload: return 0 # Ok, now let's actually load it db = {'WORDS': {}, 'FILES': {'_TOP':(0,None)}, 'FILEIDS': {}} # Identify the relevant word-dictionary segments if not wordlist: segments = self.segments else: segments = ['-','#'] for word in wordlist: segments.append(word[0].upper()) # Load the segments for segment in segments: try: f = open(self.indexdb + segment, 'rb') except IOError, error: # probably just nonexistent segment index file if error.errno != errno.ENOENT: raise else: pickle_str = zlib.decompress(f.read()) f.close() dbslice = marshal.loads(pickle_str) if dbslice.get('WORDS'): # if it has some words, add them for word, entry in dbslice['WORDS'].items(): db['WORDS'][word] = entry if dbslice.get('FILES'): # if it has some files, add them db['FILES'] = dbslice['FILES'] if dbslice.get('FILEIDS'): # if it has fileids, add them db['FILEIDS'] = dbslice['FILEIDS'] self.words = db['WORDS'] self.files = db['FILES'] self.fileids = db['FILEIDS'] self.changed = 0 def save_index(self): # only save if the index is loaded and changed if not self.index_loaded() or not self.changed: return # brutal space saver... delete all the small segments for segment in self.segments: try: os.remove(self.indexdb + segment) except OSError, error: # probably just nonexistent segment index file if error.errno != errno.ENOENT: raise # First write the much simpler filename/fileid dictionaries dbfil = {'WORDS':None, 'FILES':self.files, 'FILEIDS':self.fileids} open(self.indexdb+'-','wb').write(zlib.compress(marshal.dumps(dbfil))) # The hard part is splitting the word dictionary up, of course letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_" segdicts = {} # Need batch of empty dicts for segment in letters: segdicts[segment] = {} for word, entry in self.words.items(): # Split into segment dicts initchar = word[0].upper() segdicts[initchar][word] = entry # save for initchar in letters: db = {'WORDS':segdicts[initchar], 'FILES':None, 'FILEIDS':None} pickle_str = marshal.dumps(db) filename = self.indexdb + initchar pickle_fh = open(filename, 'wb') pickle_fh.write(zlib.compress(pickle_str)) os.chmod(filename, 0664) # save done self.changed = 0 def purge_entry(self, identifier): '''Remove a file from file index and word index ''' self.load_index() if not self.files.has_key(identifier): return file_index = self.files[identifier][0] del self.files[identifier] del self.fileids[file_index] # The much harder part, cleanup the word index for key, occurs in self.words.items(): if occurs.has_key(file_index): del occurs[file_index] # save needed self.changed = 1 def index_loaded(self): return (hasattr(self,'fileids') and hasattr(self,'files') and hasattr(self,'words')) def rollback(self): ''' load last saved index info. ''' self.load_index(reload=1) # vim: set filetype=python ts=4 sw=4 et si code-963003d2ffdd/roundup/backends/indexer_rdbms.py0000644000000000000000000001167310344271362020411 0ustar 00000000000000#$Id: indexer_rdbms.py,v 1.11 2005-09-28 05:42:23 richard Exp $ ''' This implements the full-text indexer over two RDBMS tables. The first is a mapping of words to occurance IDs. The second maps the IDs to (Class, propname, itemid) instances. ''' import re from indexer_common import Indexer, is_stopword class Indexer(Indexer): def __init__(self, db): self.db = db self.reindex = 0 def close(self): '''close the indexing database''' # just nuke the circular reference self.db = None def save_index(self): '''Save the changes to the index.''' # not necessary - the RDBMS connection will handle this for us pass def force_reindex(self): '''Force a reindexing of the database. This essentially empties the tables ids and index and sets a flag so that the databases are reindexed''' self.reindex = 1 def should_reindex(self): '''returns True if the indexes need to be rebuilt''' return self.reindex def add_text(self, identifier, text, mime_type='text/plain'): ''' "identifier" is (classname, itemid, property) ''' if mime_type != 'text/plain': return # first, find the id of the (classname, itemid, property) a = self.db.arg sql = 'select _textid from __textids where _class=%s and '\ '_itemid=%s and _prop=%s'%(a, a, a) self.db.cursor.execute(sql, identifier) r = self.db.cursor.fetchone() if not r: id = self.db.newid('__textids') sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\ ' values (%s, %s, %s, %s)'%(a, a, a, a) self.db.cursor.execute(sql, (id, ) + identifier) self.db.cursor.execute('select max(_textid) from __textids') id = self.db.cursor.fetchone()[0] else: id = int(r[0]) # clear out any existing indexed values sql = 'delete from __words where _textid=%s'%a self.db.cursor.execute(sql, (id, )) # ok, find all the words in the text text = unicode(text, "utf-8", "replace").upper() wordlist = [w.encode("utf-8", "replace") for w in re.findall(r'(?u)\b\w{2,25}\b', text)] words = {} for word in wordlist: if is_stopword(word): continue if len(word) > 25: continue words[word] = 1 words = words.keys() # for each word, add an entry in the db for word in words: # don't dupe sql = 'select * from __words where _word=%s and _textid=%s'%(a, a) self.db.cursor.execute(sql, (word, id)) if self.db.cursor.fetchall(): continue sql = 'insert into __words (_word, _textid) values (%s, %s)'%(a, a) self.db.cursor.execute(sql, (word, id)) def find(self, wordlist): '''look up all the words in the wordlist. If none are found return an empty dictionary * more rules here ''' if not wordlist: return {} l = [word.upper() for word in wordlist if 26 > len(word) > 2] if not l: return {} if self.db.implements_intersect: # simple AND search sql = 'select distinct(_textid) from __words where _word=%s'%self.db.arg sql = '\nINTERSECT\n'.join([sql]*len(l)) self.db.cursor.execute(sql, tuple(l)) r = self.db.cursor.fetchall() if not r: return {} a = ','.join([self.db.arg] * len(r)) sql = 'select _class, _itemid, _prop from __textids '\ 'where _textid in (%s)'%a self.db.cursor.execute(sql, tuple([int(id) for (id,) in r])) else: # A more complex version for MySQL since it doesn't implement INTERSECT # Construct SQL statement to join __words table to itself # multiple times. sql = """select distinct(__words1._textid) from __words as __words1 %s where __words1._word=%s %s""" join_tmpl = ' left join __words as __words%d using (_textid) \n' match_tmpl = ' and __words%d._word=%s \n' join_list = [] match_list = [] for n in xrange(len(l) - 1): join_list.append(join_tmpl % (n + 2)) match_list.append(match_tmpl % (n + 2, self.db.arg)) sql = sql%(' '.join(join_list), self.db.arg, ' '.join(match_list)) self.db.cursor.execute(sql, l) r = map(lambda x: x[0], self.db.cursor.fetchall()) if not r: return {} a = ','.join([self.db.arg] * len(r)) sql = 'select _class, _itemid, _prop from __textids '\ 'where _textid in (%s)'%a self.db.cursor.execute(sql, tuple(map(int, r))) return self.db.cursor.fetchall() code-963003d2ffdd/roundup/backends/indexer_xapian.py0000644000000000000000000000613710344271362020561 0ustar 00000000000000#$Id: indexer_xapian.py,v 1.1 2005-04-28 00:21:42 richard Exp $ ''' This implements the full-text indexer using the Xapian indexer. ''' import re, os import xapian from indexer_common import Indexer, is_stopword # TODO: we need to delete documents when a property is *reindexed* class Indexer(Indexer): def __init__(self, db): self.db_path = db.config.DATABASE self.reindex = 0 self.transaction_active = False def _get_database(self): index = os.path.join(self.db_path, 'text-index') return xapian.WritableDatabase(index, xapian.DB_CREATE_OR_OPEN) def save_index(self): '''Save the changes to the index.''' if not self.transaction_active: return # XXX: Xapian databases don't actually implement transactions yet database = self._get_database() database.commit_transaction() self.transaction_active = False def close(self): '''close the indexing database''' pass def force_reindex(self): '''Force a reindexing of the database. This essentially empties the tables ids and index and sets a flag so that the databases are reindexed''' self.reindex = 1 def should_reindex(self): '''returns True if the indexes need to be rebuilt''' return self.reindex def add_text(self, identifier, text, mime_type='text/plain'): ''' "identifier" is (classname, itemid, property) ''' if mime_type != 'text/plain': return # open the database and start a transaction if needed database = self._get_database() # XXX: Xapian databases don't actually implement transactions yet #if not self.transaction_active: #database.begin_transaction() #self.transaction_active = True # TODO: allow configuration of other languages stemmer = xapian.Stem("english") doc = xapian.Document() # Xapian doesn't actually seem to care what data is put in here, so # we use it to store the text identifier. doc.set_data('%s:%s:%s'%identifier) for match in re.finditer(r'\b\w{2,25}\b', text.upper()): word = match.group(0) if is_stopword(word): continue term = stemmer.stem_word(word) doc.add_posting(term, match.start(0)) database.add_document(doc) def find(self, wordlist): '''look up all the words in the wordlist. If none are found return an empty dictionary * more rules here ''' if not wordlist: return {} database = self._get_database() enquire = xapian.Enquire(database) stemmer = xapian.Stem("english") terms = [] for term in [word.upper() for word in wordlist if 26 > len(word) > 2]: terms.append(stemmer.stem_word(term.upper())) query = xapian.Query(xapian.Query.OP_AND, terms) enquire.set_query(query) matches = enquire.get_mset(0, 10) return [tuple(m[xapian.MSET_DOCUMENT].get_data().split(':')) for m in matches] code-963003d2ffdd/roundup/backends/locking.py0000644000000000000000000000340510344271362017204 0ustar 00000000000000#! /usr/bin/env python # Copyright (c) 2002 ekit.com Inc (http://www.ekit-inc.com/) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # $Id: locking.py,v 1.8 2004-02-11 23:55:09 richard Exp $ '''This module provides a generic interface to acquire and release exclusive access to a file. It should work on Unix and Windows. ''' __docformat__ = 'restructuredtext' import portalocker def acquire_lock(path, block=1): '''Acquire a lock for the given path ''' import portalocker file = open(path, 'w') if block: portalocker.lock(file, portalocker.LOCK_EX) else: portalocker.lock(file, portalocker.LOCK_EX|portalocker.LOCK_NB) return file def release_lock(file): '''Release our lock on the given path ''' portalocker.unlock(file) code-963003d2ffdd/roundup/backends/portalocker.py0000644000000000000000000001164510344271362020110 0ustar 00000000000000# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking. # Requires python 1.5.2 or better. # ID line added by richard for Roundup file tracking # $Id: portalocker.py,v 1.8 2004-02-11 23:55:09 richard Exp $ """Cross-platform (posix/nt) API for flock-style file locking. Synopsis:: import portalocker file = open("somefile", "r+") portalocker.lock(file, portalocker.LOCK_EX) file.seek(12) file.write("foo") file.close() If you know what you're doing, you may choose to:: portalocker.unlock(file) before closing the file, but why? Methods:: lock( file, flags ) unlock( file ) Constants:: LOCK_EX LOCK_SH LOCK_NB I learned the win32 technique for locking files from sample code provided by John Nielsen in the documentation that accompanies the win32 modules. :Author: Jonathan Feinberg :Version: Id: portalocker.py,v 1.3 2001/05/29 18:47:55 Administrator Exp **un-cvsified by richard so the version doesn't change** """ __docformat__ = 'restructuredtext' import os if os.name == 'nt': import win32con import win32file import pywintypes LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK LOCK_SH = 0 # the default LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # is there any reason not to reuse the following structure? __overlapped = pywintypes.OVERLAPPED() elif os.name == 'posix': import fcntl LOCK_EX = fcntl.LOCK_EX LOCK_SH = fcntl.LOCK_SH LOCK_NB = fcntl.LOCK_NB else: raise RuntimeError("PortaLocker only defined for nt and posix platforms") if os.name == 'nt': # eugh, we want 0xffff0000 here, but python 2.3 won't let us :( FFFF0000 = -65536 def lock(file, flags): hfile = win32file._get_osfhandle(file.fileno()) # LockFileEx is not supported on all Win32 platforms (Win95, Win98, WinME). # If it's not supported, win32file will raise an exception. # Try LockFileEx first, as it has more functionality and handles # blocking locks more efficiently. try: win32file.LockFileEx(hfile, flags, 0, FFFF0000, __overlapped) except win32file.error, e: import winerror # Propagate upwards all exceptions other than not-implemented. if e[0] != winerror.ERROR_CALL_NOT_IMPLEMENTED: raise e # LockFileEx is not supported. Use LockFile. # LockFile does not support shared locking -- always exclusive. # Care: the low/high length params are reversed compared to LockFileEx. if not flags & LOCK_EX: import warnings warnings.warn("PortaLocker does not support shared locking on Win9x", RuntimeWarning) # LockFile only supports immediate-fail locking. if flags & LOCK_NB: win32file.LockFile(hfile, 0, 0, FFFF0000, 0) else: # Emulate a blocking lock with a polling loop. import time while 1: # Attempt a lock. try: win32file.LockFile(hfile, 0, 0, FFFF0000, 0) break except win32file.error, e: # Propagate upwards all exceptions other than lock violation. if e[0] != winerror.ERROR_LOCK_VIOLATION: raise e # Sleep and poll again. time.sleep(0.1) # TODO: should this return the result of the lock? def unlock(file): hfile = win32file._get_osfhandle(file.fileno()) # UnlockFileEx is not supported on all Win32 platforms (Win95, Win98, WinME). # If it's not supported, win32file will raise an api_error exception. try: win32file.UnlockFileEx(hfile, 0, FFFF0000, __overlapped) except win32file.error, e: import winerror # Propagate upwards all exceptions other than not-implemented. if e[0] != winerror.ERROR_CALL_NOT_IMPLEMENTED: raise e # UnlockFileEx is not supported. Use UnlockFile. # Care: the low/high length params are reversed compared to UnLockFileEx. win32file.UnlockFile(hfile, 0, 0, FFFF0000, 0) elif os.name =='posix': def lock(file, flags): fcntl.flock(file.fileno(), flags) # TODO: should this return the result of the lock? def unlock(file): fcntl.flock(file.fileno(), fcntl.LOCK_UN) if __name__ == '__main__': from time import time, strftime, localtime import sys import portalocker log = open('log.txt', "a+") portalocker.lock(log, portalocker.LOCK_EX) timestamp = strftime("%m/%d/%Y %H:%M:%S\n", localtime(time())) log.write( timestamp ) print "Wrote lines. Hit enter to release lock." dummy = sys.stdin.readline() log.close() code-963003d2ffdd/roundup/backends/rdbms_common.py0000644000000000000000000030357210344271362020245 0ustar 00000000000000# $Id: rdbms_common.py,v 1.159 2005-10-07 05:35:04 richard Exp $ ''' Relational database (SQL) backend common code. Basics: - map roundup classes to relational tables - automatically detect schema changes and modify the table schemas appropriately (we store the "database version" of the schema in the database itself as the only row of the "schema" table) - multilinks (which represent a many-to-many relationship) are handled through intermediate tables - journals are stored adjunct to the per-class tables - table names and columns have "_" prepended so the names can't clash with restricted names (like "order") - retirement is determined by the __retired__ column being true Database-specific changes may generally be pushed out to the overridable sql_* methods, since everything else should be fairly generic. There's probably a bit of work to be done if a database is used that actually honors column typing, since the initial databases don't (sqlite stores everything as a string.) The schema of the hyperdb being mapped to the database is stored in the database itself as a repr()'ed dictionary of information about each Class that maps to a table. If that information differs from the hyperdb schema, then we update it. We also store in the schema dict a version which allows us to upgrade the database schema when necessary. See upgrade_db(). ''' __docformat__ = 'restructuredtext' # standard python modules import sys, os, time, re, errno, weakref, copy, logging # roundup modules from roundup import hyperdb, date, password, roundupdb, security from roundup.hyperdb import String, Password, Date, Interval, Link, \ Multilink, DatabaseError, Boolean, Number, Node from roundup.backends import locking # support from blobfiles import FileStorage try: # re-enable once Xapian is fixed from indexer_xapian import Indexer_disabled except ImportError: from indexer_rdbms import Indexer from sessions_rdbms import Sessions, OneTimeKeys from roundup.date import Range # number of rows to keep in memory ROW_CACHE_SIZE = 100 # dummy value meaning "argument not passed" _marker = [] def _num_cvt(num): num = str(num) try: return int(num) except: return float(num) def _bool_cvt(value): if value in ('TRUE', 'FALSE'): return {'TRUE': 1, 'FALSE': 0}[value] # assume it's a number returned from the db API return int(value) def connection_dict(config, dbnamestr=None): ''' Used by Postgresql and MySQL to detemine the keyword args for opening the database connection.''' d = { } if dbnamestr: d[dbnamestr] = config.RDBMS_NAME for name in ('host', 'port', 'password', 'user', 'read_default_group', 'read_default_file'): cvar = 'RDBMS_'+name.upper() if config[cvar] is not None: d[name] = config[cvar] return d class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' Wrapper around an SQL database that presents a hyperdb interface. - some functionality is specific to the actual SQL database, hence the sql_* methods that are NotImplemented - we keep a cache of the latest ROW_CACHE_SIZE row fetches. ''' def __init__(self, config, journaltag=None): ''' Open the database and load the schema from it. ''' self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.indexer = Indexer(self) self.security = security.Security(self) # additional transaction support for external files and the like self.transactions = [] # keep a cache of the N most recently retrieved rows of any kind # (classname, nodeid) = row self.cache = {} self.cache_lru = [] self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, 'filtering': 0} # database lock self.lockfile = None # open a connection to the database, creating the "conn" attribute self.open_connection() def clearCache(self): self.cache = {} self.cache_lru = [] def getSessionManager(self): return Sessions(self) def getOTKManager(self): return OneTimeKeys(self) def open_connection(self): ''' Open a connection to the database, creating it if necessary. Must call self.load_dbschema() ''' raise NotImplemented def sql(self, sql, args=None): ''' Execute the sql with the optional args. ''' if __debug__: logging.getLogger('hyperdb').debug('SQL %r %r'%(sql, args)) if args: self.cursor.execute(sql, args) else: self.cursor.execute(sql) def sql_fetchone(self): ''' Fetch a single row. If there's nothing to fetch, return None. ''' return self.cursor.fetchone() def sql_fetchall(self): ''' Fetch all rows. If there's nothing to fetch, return []. ''' return self.cursor.fetchall() def sql_stringquote(self, value): ''' Quote the string so it's safe to put in the 'sql quotes' ''' return re.sub("'", "''", str(value)) def init_dbschema(self): self.database_schema = { 'version': self.current_db_version, 'tables': {} } def load_dbschema(self): ''' Load the schema definition that the database currently implements ''' self.cursor.execute('select schema from schema') schema = self.cursor.fetchone() if schema: self.database_schema = eval(schema[0]) else: self.database_schema = {} def save_dbschema(self): ''' Save the schema definition that the database currently implements ''' s = repr(self.database_schema) self.sql('delete from schema') self.sql('insert into schema values (%s)', (s,)) def post_init(self): ''' Called once the schema initialisation has finished. We should now confirm that the schema defined by our "classes" attribute actually matches the schema in the database. ''' save = self.upgrade_db() # now detect changes in the schema tables = self.database_schema['tables'] for classname, spec in self.classes.items(): if tables.has_key(classname): dbspec = tables[classname] if self.update_class(spec, dbspec): tables[classname] = spec.schema() save = 1 else: self.create_class(spec) tables[classname] = spec.schema() save = 1 for classname, spec in tables.items(): if not self.classes.has_key(classname): self.drop_class(classname, tables[classname]) del tables[classname] save = 1 # update the database version of the schema if save: self.save_dbschema() # reindex the db if necessary if self.indexer.should_reindex(): self.reindex() # commit self.sql_commit() # update this number when we need to make changes to the SQL structure # of the backen database current_db_version = 4 def upgrade_db(self): ''' Update the SQL database to reflect changes in the backend code. Return boolean whether we need to save the schema. ''' version = self.database_schema.get('version', 1) if version == self.current_db_version: # nothing to do return 0 if version < 2: if __debug__: logging.getLogger('hyperdb').info('upgrade to version 2') # change the schema structure self.database_schema = {'tables': self.database_schema} # version 1 didn't have the actor column (note that in # MySQL this will also transition the tables to typed columns) self.add_new_columns_v2() # version 1 doesn't have the OTK, session and indexing in the # database self.create_version_2_tables() if version < 3: if __debug__: logging.getLogger('hyperdb').info('upgrade to version 3') self.fix_version_2_tables() if version < 4: self.fix_version_3_tables() self.database_schema['version'] = self.current_db_version return 1 def fix_version_3_tables(self): # drop the shorter VARCHAR OTK column and add a new TEXT one for name in ('otk', 'session'): self.sql('DELETE FROM %ss'%name) self.sql('ALTER TABLE %ss DROP %s_value'%(name, name)) self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name)) def fix_version_2_tables(self): '''Default (used by sqlite): NOOP''' pass def _convert_journal_tables(self): '''Get current journal table contents, drop the table and re-create''' c = self.cursor cols = ','.join('nodeid date tag action params'.split()) for klass in self.classes.values(): # slurp and drop sql = 'select %s from %s__journal order by date'%(cols, klass.classname) c.execute(sql) contents = c.fetchall() self.drop_journal_table_indexes(klass.classname) c.execute('drop table %s__journal'%klass.classname) # re-create and re-populate self.create_journal_table(klass) a = self.arg sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%( klass.classname, cols, a, a, a, a, a) for row in contents: # no data conversion needed self.cursor.execute(sql, row) def _convert_string_properties(self): '''Get current Class tables that contain String properties, and convert the VARCHAR columns to TEXT''' c = self.cursor for klass in self.classes.values(): # slurp and drop cols, mls = self.determine_columns(klass.properties.items()) scols = ','.join([i[0] for i in cols]) sql = 'select id,%s from _%s'%(scols, klass.classname) c.execute(sql) contents = c.fetchall() self.drop_class_table_indexes(klass.classname, klass.getkey()) c.execute('drop table _%s'%klass.classname) # re-create and re-populate self.create_class_table(klass, create_sequence=0) a = ','.join([self.arg for i in range(len(cols)+1)]) sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname, scols, a) for row in contents: l = [] for entry in row: # mysql will already be a string - psql needs "help" if entry is not None and not isinstance(entry, type('')): entry = str(entry) l.append(entry) self.cursor.execute(sql, l) def refresh_database(self): self.post_init() def reindex(self, classname=None): if classname: classes = [self.getclass(classname)] else: classes = self.classes.values() for klass in classes: for nodeid in klass.list(): klass.index(nodeid) self.indexer.save_index() hyperdb_to_sql_datatypes = { hyperdb.String : 'TEXT', hyperdb.Date : 'TIMESTAMP', hyperdb.Link : 'INTEGER', hyperdb.Interval : 'VARCHAR(255)', hyperdb.Password : 'VARCHAR(255)', hyperdb.Boolean : 'BOOLEAN', hyperdb.Number : 'REAL', } def determine_columns(self, properties): ''' Figure the column names and multilink properties from the spec "properties" is a list of (name, prop) where prop may be an instance of a hyperdb "type" _or_ a string repr of that type. ''' cols = [ ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]), ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]), ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]), ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]), ] mls = [] # add the multilinks separately for col, prop in properties: if isinstance(prop, Multilink): mls.append(col) continue if isinstance(prop, type('')): raise ValueError, "string property spec!" #and prop.find('Multilink') != -1: #mls.append(col) datatype = self.hyperdb_to_sql_datatypes[prop.__class__] cols.append(('_'+col, datatype)) # Intervals stored as two columns if isinstance(prop, Interval): cols.append(('__'+col+'_int__', 'BIGINT')) cols.sort() return cols, mls def update_class(self, spec, old_spec, force=0): ''' Determine the differences between the current spec and the database version of the spec, and update where necessary. If 'force' is true, update the database anyway. ''' new_has = spec.properties.has_key new_spec = spec.schema() new_spec[1].sort() old_spec[1].sort() if not force and new_spec == old_spec: # no changes return 0 logger = logging.getLogger('hyperdb') logger.info('update_class %s'%spec.classname) logger.debug('old_spec %r'%(old_spec,)) logger.debug('new_spec %r'%(new_spec,)) # detect key prop change for potential index change keyprop_changes = {} if new_spec[0] != old_spec[0]: keyprop_changes = {'remove': old_spec[0], 'add': new_spec[0]} # detect multilinks that have been removed, and drop their table old_has = {} for name, prop in old_spec[1]: old_has[name] = 1 if new_has(name): continue if prop.find('Multilink to') != -1: # first drop indexes. self.drop_multilink_table_indexes(spec.classname, name) # now the multilink table itself sql = 'drop table %s_%s'%(spec.classname, name) else: # if this is the key prop, drop the index first if old_spec[0] == prop: self.drop_class_table_key_index(spec.classname, name) del keyprop_changes['remove'] # drop the column sql = 'alter table _%s drop column _%s'%(spec.classname, name) self.sql(sql) old_has = old_has.has_key # if we didn't remove the key prop just then, but the key prop has # changed, we still need to remove the old index if keyprop_changes.has_key('remove'): self.drop_class_table_key_index(spec.classname, keyprop_changes['remove']) # add new columns for propname, prop in new_spec[1]: if old_has(propname): continue prop = spec.properties[propname] if isinstance(prop, Multilink): self.create_multilink_table(spec, propname) else: # add the column coltype = self.hyperdb_to_sql_datatypes[prop.__class__] sql = 'alter table _%s add column _%s %s'%( spec.classname, propname, coltype) self.sql(sql) # extra Interval column if isinstance(prop, Interval): sql = 'alter table _%s add column __%s_int__ BIGINT'%( spec.classname, propname) self.sql(sql) # if the new column is a key prop, we need an index! if new_spec[0] == propname: self.create_class_table_key_index(spec.classname, propname) del keyprop_changes['add'] # if we didn't add the key prop just then, but the key prop has # changed, we still need to add the new index if keyprop_changes.has_key('add'): self.create_class_table_key_index(spec.classname, keyprop_changes['add']) return 1 def determine_all_columns(self, spec): """Figure out the columns from the spec and also add internal columns """ cols, mls = self.determine_columns(spec.properties.items()) # add on our special columns cols.append(('id', 'INTEGER PRIMARY KEY')) cols.append(('__retired__', 'INTEGER DEFAULT 0')) return cols, mls def create_class_table(self, spec): '''Create the class table for the given Class "spec". Creates the indexes too.''' cols, mls = self.determine_all_columns(spec) # create the base table scols = ','.join(['%s %s'%x for x in cols]) sql = 'create table _%s (%s)'%(spec.classname, scols) self.sql(sql) self.create_class_table_indexes(spec) return cols, mls def create_class_table_indexes(self, spec): ''' create the class table for the given spec ''' # create __retired__ index index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%( spec.classname, spec.classname) self.sql(index_sql2) # create index for key property if spec.key: index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%( spec.classname, spec.key, spec.classname, spec.key) self.sql(index_sql3) # TODO: create indexes on (selected?) Link property columns, as # they're more likely to be used for lookup def drop_class_table_indexes(self, cn, key): # drop the old table indexes first l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn] if key: l.append('_%s_%s_idx'%(cn, key)) table_name = '_%s'%cn for index_name in l: if not self.sql_index_exists(table_name, index_name): continue index_sql = 'drop index '+index_name self.sql(index_sql) def create_class_table_key_index(self, cn, key): ''' create the class table for the given spec ''' sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key) self.sql(sql) def drop_class_table_key_index(self, cn, key): table_name = '_%s'%cn index_name = '_%s_%s_idx'%(cn, key) if not self.sql_index_exists(table_name, index_name): return sql = 'drop index '+index_name self.sql(sql) def create_journal_table(self, spec): ''' create the journal table for a class given the spec and already-determined cols ''' # journal table cols = ','.join(['%s varchar'%x for x in 'nodeid date tag action params'.split()]) sql = '''create table %s__journal ( nodeid integer, date %s, tag varchar(255), action varchar(255), params text)''' % (spec.classname, self.hyperdb_to_sql_datatypes[hyperdb.Date]) self.sql(sql) self.create_journal_table_indexes(spec) def create_journal_table_indexes(self, spec): # index on nodeid sql = 'create index %s_journ_idx on %s__journal(nodeid)'%( spec.classname, spec.classname) self.sql(sql) def drop_journal_table_indexes(self, classname): index_name = '%s_journ_idx'%classname if not self.sql_index_exists('%s__journal'%classname, index_name): return index_sql = 'drop index '+index_name self.sql(index_sql) def create_multilink_table(self, spec, ml): ''' Create a multilink table for the "ml" property of the class given by the spec ''' # create the table sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%( spec.classname, ml) self.sql(sql) self.create_multilink_table_indexes(spec, ml) def create_multilink_table_indexes(self, spec, ml): # create index on linkid index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%( spec.classname, ml, spec.classname, ml) self.sql(index_sql) # create index on nodeid index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%( spec.classname, ml, spec.classname, ml) self.sql(index_sql) def drop_multilink_table_indexes(self, classname, ml): l = [ '%s_%s_l_idx'%(classname, ml), '%s_%s_n_idx'%(classname, ml) ] table_name = '%s_%s'%(classname, ml) for index_name in l: if not self.sql_index_exists(table_name, index_name): continue index_sql = 'drop index %s'%index_name self.sql(index_sql) def create_class(self, spec): ''' Create a database table according to the given spec. ''' cols, mls = self.create_class_table(spec) self.create_journal_table(spec) # now create the multilink tables for ml in mls: self.create_multilink_table(spec, ml) def drop_class(self, cn, spec): ''' Drop the given table from the database. Drop the journal and multilink tables too. ''' properties = spec[1] # figure the multilinks mls = [] for propanme, prop in properties: if isinstance(prop, Multilink): mls.append(propname) # drop class table and indexes self.drop_class_table_indexes(cn, spec[0]) self.drop_class_table(cn) # drop journal table and indexes self.drop_journal_table_indexes(cn) sql = 'drop table %s__journal'%cn self.sql(sql) for ml in mls: # drop multilink table and indexes self.drop_multilink_table_indexes(cn, ml) sql = 'drop table %s_%s'%(spec.classname, ml) self.sql(sql) def drop_class_table(self, cn): sql = 'drop table _%s'%cn self.sql(sql) # # Classes # def __getattr__(self, classname): ''' A convenient way of calling self.getclass(classname). ''' if self.classes.has_key(classname): return self.classes[classname] raise AttributeError, classname def addclass(self, cl): ''' Add a Class to the hyperdatabase. ''' cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl # add default Edit and View permissions self.security.addPermission(name="Create", klass=cn, description="User is allowed to create "+cn) self.security.addPermission(name="Edit", klass=cn, description="User is allowed to edit "+cn) self.security.addPermission(name="View", klass=cn, description="User is allowed to access "+cn) def getclasses(self): ''' Return a list of the names of all existing classes. ''' l = self.classes.keys() l.sort() return l def getclass(self, classname): '''Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. ''' try: return self.classes[classname] except KeyError: raise KeyError, 'There is no class called "%s"'%classname def clear(self): '''Delete all database contents. Note: I don't commit here, which is different behaviour to the "nuke from orbit" behaviour in the dbs. ''' logging.getLogger('hyperdb').info('clear') for cn in self.classes.keys(): sql = 'delete from _%s'%cn self.sql(sql) # # Nodes # hyperdb_to_sql_value = { hyperdb.String : str, # fractional seconds by default hyperdb.Date : lambda x: x.formal(sep=' ', sec='%.3f'), hyperdb.Link : int, hyperdb.Interval : str, hyperdb.Password : str, hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE', hyperdb.Number : lambda x: x, hyperdb.Multilink : lambda x: x, # used in journal marshalling } def addnode(self, classname, nodeid, node): ''' Add the specified node to its class's db. ''' if __debug__: logging.getLogger('hyperdb').debug('addnode %s%s %r'%(classname, nodeid, node)) # determine the column definitions and multilink tables cl = self.classes[classname] cols, mls = self.determine_columns(cl.properties.items()) # we'll be supplied these props if we're doing an import values = node.copy() if not values.has_key('creator'): # add in the "calculated" properties (dupe so we don't affect # calling code's node assumptions) values['creation'] = values['activity'] = date.Date() values['actor'] = values['creator'] = self.getuid() cl = self.classes[classname] props = cl.getprops(protected=1) del props['id'] # default the non-multilink columns for col, prop in props.items(): if not values.has_key(col): if isinstance(prop, Multilink): values[col] = [] else: values[col] = None # clear this node out of the cache if it's in there key = (classname, nodeid) if self.cache.has_key(key): del self.cache[key] self.cache_lru.remove(key) # figure the values to insert vals = [] for col,dt in cols: # this is somewhat dodgy.... if col.endswith('_int__'): # XXX eugh, this test suxxors value = values[col[2:-6]] # this is an Interval special "int" column if value is not None: vals.append(value.as_seconds()) else: vals.append(value) continue prop = props[col[1:]] value = values[col[1:]] if value is not None: value = self.hyperdb_to_sql_value[prop.__class__](value) vals.append(value) vals.append(nodeid) vals = tuple(vals) # make sure the ordering is correct for column name -> column value s = ','.join([self.arg for x in cols]) + ',%s'%self.arg cols = ','.join([col for col,dt in cols]) + ',id' # perform the inserts sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s) self.sql(sql, vals) # insert the multilink rows for col in mls: t = '%s_%s'%(classname, col) for entry in node[col]: sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t, self.arg, self.arg) self.sql(sql, (entry, nodeid)) def setnode(self, classname, nodeid, values, multilink_changes={}): ''' Change the specified node. ''' if __debug__: logging.getLogger('hyperdb').debug('setnode %s%s %r' % (classname, nodeid, values)) # clear this node out of the cache if it's in there key = (classname, nodeid) if self.cache.has_key(key): del self.cache[key] self.cache_lru.remove(key) cl = self.classes[classname] props = cl.getprops() cols = [] mls = [] # add the multilinks separately for col in values.keys(): prop = props[col] if isinstance(prop, Multilink): mls.append(col) elif isinstance(prop, Interval): # Intervals store the seconds value too cols.append(col) # extra leading '_' added by code below cols.append('_' +col + '_int__') else: cols.append(col) cols.sort() # figure the values to insert vals = [] for col in cols: if col.endswith('_int__'): # XXX eugh, this test suxxors # Intervals store the seconds value too col = col[1:-6] prop = props[col] value = values[col] if value is None: vals.append(None) else: vals.append(value.as_seconds()) else: prop = props[col] value = values[col] if value is None: e = None else: e = self.hyperdb_to_sql_value[prop.__class__](value) vals.append(e) vals.append(int(nodeid)) vals = tuple(vals) # if there's any updates to regular columns, do them if cols: # make sure the ordering is correct for column name -> column value s = ','.join(['_%s=%s'%(x, self.arg) for x in cols]) cols = ','.join(cols) # perform the update sql = 'update _%s set %s where id=%s'%(classname, s, self.arg) self.sql(sql, vals) # we're probably coming from an import, not a change if not multilink_changes: for name in mls: prop = props[name] value = values[name] t = '%s_%s'%(classname, name) # clear out previous values for this node # XXX numeric ids self.sql('delete from %s where nodeid=%s'%(t, self.arg), (nodeid,)) # insert the values for this node for entry in values[name]: sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t, self.arg, self.arg) # XXX numeric ids self.sql(sql, (entry, nodeid)) # we have multilink changes to apply for col, (add, remove) in multilink_changes.items(): tn = '%s_%s'%(classname, col) if add: sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn, self.arg, self.arg) for addid in add: # XXX numeric ids self.sql(sql, (int(nodeid), int(addid))) if remove: sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn, self.arg, self.arg) for removeid in remove: # XXX numeric ids self.sql(sql, (int(nodeid), int(removeid))) sql_to_hyperdb_value = { hyperdb.String : str, hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')), # hyperdb.Link : int, # XXX numeric ids hyperdb.Link : str, hyperdb.Interval : date.Interval, hyperdb.Password : lambda x: password.Password(encrypted=x), hyperdb.Boolean : _bool_cvt, hyperdb.Number : _num_cvt, hyperdb.Multilink : lambda x: x, # used in journal marshalling } def getnode(self, classname, nodeid): ''' Get a node from the database. ''' # see if we have this node cached key = (classname, nodeid) if self.cache.has_key(key): # push us back to the top of the LRU self.cache_lru.remove(key) self.cache_lru.insert(0, key) if __debug__: self.stats['cache_hits'] += 1 # return the cached information return self.cache[key] if __debug__: self.stats['cache_misses'] += 1 start_t = time.time() # figure the columns we're fetching cl = self.classes[classname] cols, mls = self.determine_columns(cl.properties.items()) scols = ','.join([col for col,dt in cols]) # perform the basic property fetch sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg) self.sql(sql, (nodeid,)) values = self.sql_fetchone() if values is None: raise IndexError, 'no such %s node %s'%(classname, nodeid) # make up the node node = {} props = cl.getprops(protected=1) for col in range(len(cols)): name = cols[col][0][1:] if name.endswith('_int__'): # XXX eugh, this test suxxors # ignore the special Interval-as-seconds column continue value = values[col] if value is not None: value = self.sql_to_hyperdb_value[props[name].__class__](value) node[name] = value # now the multilinks for col in mls: # get the link ids sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col, self.arg) self.cursor.execute(sql, (nodeid,)) # extract the first column from the result # XXX numeric ids items = [int(x[0]) for x in self.cursor.fetchall()] items.sort () node[col] = [str(x) for x in items] # save off in the cache key = (classname, nodeid) self.cache[key] = node # update the LRU self.cache_lru.insert(0, key) if len(self.cache_lru) > ROW_CACHE_SIZE: del self.cache[self.cache_lru.pop()] if __debug__: self.stats['get_items'] += (time.time() - start_t) return node def destroynode(self, classname, nodeid): '''Remove a node from the database. Called exclusively by the destroy() method on Class. ''' logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid)) # make sure the node exists if not self.hasnode(classname, nodeid): raise IndexError, '%s has no node %s'%(classname, nodeid) # see if we have this node cached if self.cache.has_key((classname, nodeid)): del self.cache[(classname, nodeid)] # see if there's any obvious commit actions that we should get rid of for entry in self.transactions[:]: if entry[1][:2] == (classname, nodeid): self.transactions.remove(entry) # now do the SQL sql = 'delete from _%s where id=%s'%(classname, self.arg) self.sql(sql, (nodeid,)) # remove from multilnks cl = self.getclass(classname) x, mls = self.determine_columns(cl.properties.items()) for col in mls: # get the link ids sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg) self.sql(sql, (nodeid,)) # remove journal entries sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg) self.sql(sql, (nodeid,)) def hasnode(self, classname, nodeid): ''' Determine if the database has a given node. ''' sql = 'select count(*) from _%s where id=%s'%(classname, self.arg) self.sql(sql, (nodeid,)) return int(self.cursor.fetchone()[0]) def countnodes(self, classname): ''' Count the number of nodes that exist for a particular Class. ''' sql = 'select count(*) from _%s'%classname self.sql(sql) return self.cursor.fetchone()[0] def addjournal(self, classname, nodeid, action, params, creator=None, creation=None): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' # handle supply of the special journalling parameters (usually # supplied on importing an existing database) if creator: journaltag = creator else: journaltag = self.getuid() if creation: journaldate = creation else: journaldate = date.Date() # create the journal entry cols = 'nodeid,date,tag,action,params' if __debug__: logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%(classname, nodeid, journaldate, journaltag, action, params)) # make the journalled data marshallable if isinstance(params, type({})): self._journal_marshal(params, classname) params = repr(params) dc = self.hyperdb_to_sql_value[hyperdb.Date] journaldate = dc(journaldate) self.save_journal(classname, cols, nodeid, journaldate, journaltag, action, params) def setjournal(self, classname, nodeid, journal): '''Set the journal to the "journal" list.''' # clear out any existing entries self.sql('delete from %s__journal where nodeid=%s'%(classname, self.arg), (nodeid,)) # create the journal entry cols = 'nodeid,date,tag,action,params' dc = self.hyperdb_to_sql_value[hyperdb.Date] for nodeid, journaldate, journaltag, action, params in journal: if __debug__: logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%( classname, nodeid, journaldate, journaltag, action, params)) # make the journalled data marshallable if isinstance(params, type({})): self._journal_marshal(params, classname) params = repr(params) self.save_journal(classname, cols, nodeid, dc(journaldate), journaltag, action, params) def _journal_marshal(self, params, classname): '''Convert the journal params values into safely repr'able and eval'able values.''' properties = self.getclass(classname).getprops() for param, value in params.items(): if not value: continue property = properties[param] cvt = self.hyperdb_to_sql_value[property.__class__] if isinstance(property, Password): params[param] = cvt(value) elif isinstance(property, Date): params[param] = cvt(value) elif isinstance(property, Interval): params[param] = cvt(value) elif isinstance(property, Boolean): params[param] = cvt(value) def getjournal(self, classname, nodeid): ''' get the journal for id ''' # make sure the node exists if not self.hasnode(classname, nodeid): raise IndexError, '%s has no node %s'%(classname, nodeid) cols = ','.join('nodeid date tag action params'.split()) journal = self.load_journal(classname, cols, nodeid) # now unmarshal the data dc = self.sql_to_hyperdb_value[hyperdb.Date] res = [] properties = self.getclass(classname).getprops() for nodeid, date_stamp, user, action, params in journal: params = eval(params) if isinstance(params, type({})): for param, value in params.items(): if not value: continue property = properties.get(param, None) if property is None: # deleted property continue cvt = self.sql_to_hyperdb_value[property.__class__] if isinstance(property, Password): params[param] = cvt(value) elif isinstance(property, Date): params[param] = cvt(value) elif isinstance(property, Interval): params[param] = cvt(value) elif isinstance(property, Boolean): params[param] = cvt(value) # XXX numeric ids res.append((str(nodeid), dc(date_stamp), user, action, params)) return res def save_journal(self, classname, cols, nodeid, journaldate, journaltag, action, params): ''' Save the journal entry to the database ''' entry = (nodeid, journaldate, journaltag, action, params) # do the insert a = self.arg sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%( classname, cols, a, a, a, a, a) self.sql(sql, entry) def load_journal(self, classname, cols, nodeid): ''' Load the journal from the database ''' # now get the journal entries sql = 'select %s from %s__journal where nodeid=%s order by date'%( cols, classname, self.arg) self.sql(sql, (nodeid,)) return self.cursor.fetchall() def pack(self, pack_before): ''' Delete all journal entries except "create" before 'pack_before'. ''' date_stamp = self.hyperdb_to_sql_value[Date](pack_before) # do the delete for classname in self.classes.keys(): sql = "delete from %s__journal where date<%s and "\ "action<>'create'"%(classname, self.arg) self.sql(sql, (date_stamp,)) def sql_commit(self): ''' Actually commit to the database. ''' logging.getLogger('hyperdb').info('commit') self.conn.commit() # open a new cursor for subsequent work self.cursor = self.conn.cursor() def commit(self): ''' Commit the current transactions. Save all data changed since the database was opened or since the last commit() or rollback(). ''' # commit the database self.sql_commit() # now, do all the other transaction stuff for method, args in self.transactions: method(*args) # save the indexer self.indexer.save_index() # clear out the transactions self.transactions = [] def sql_rollback(self): self.conn.rollback() def rollback(self): ''' Reverse all actions from the current transaction. Undo all the changes made since the database was opened or the last commit() or rollback() was performed. ''' logging.getLogger('hyperdb').info('rollback') self.sql_rollback() # roll back "other" transaction stuff for method, args in self.transactions: # delete temporary files if method == self.doStoreFile: self.rollbackStoreFile(*args) self.transactions = [] # clear the cache self.clearCache() def sql_close(self): logging.getLogger('hyperdb').info('close') self.conn.close() def close(self): ''' Close off the connection. ''' self.indexer.close() self.sql_close() # # The base Class class # class Class(hyperdb.Class): ''' The handle to a particular class of nodes in a hyperdatabase. All methods except __repr__ and getnode must be implemented by a concrete backend Class. ''' def __init__(self, db, classname, **properties): '''Create a new class with a given name and property specification. 'classname' must not collide with the name of an existing class, or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. ''' for name in 'creation activity creator actor'.split(): if properties.has_key(name): raise ValueError, '"creation", "activity", "creator" and '\ '"actor" are reserved' self.classname = classname self.properties = properties self.db = weakref.proxy(db) # use a weak ref to avoid circularity self.key = '' # should we journal changes (default yes) self.do_journal = 1 # do the db-related init stuff db.addclass(self) self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []} self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []} def schema(self): ''' A dumpable version of the schema that we can store in the database ''' return (self.key, [(x, repr(y)) for x,y in self.properties.items()]) def enableJournalling(self): '''Turn journalling on for this class ''' self.do_journal = 1 def disableJournalling(self): '''Turn journalling off for this class ''' self.do_journal = 0 # Editing nodes: def create(self, **propvalues): ''' Create a new node of this class and return its id. The keyword arguments in 'propvalues' map property names to values. The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. ''' self.fireAuditors('create', None, propvalues) newid = self.create_inner(**propvalues) self.fireReactors('create', newid, None) return newid def create_inner(self, **propvalues): ''' Called by create, in-between the audit and react calls. ''' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' if propvalues.has_key('creator') or propvalues.has_key('actor') or \ propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creator", "actor", "creation" and '\ '"activity" are reserved' # new node's id newid = self.db.newid(self.classname) # validate propvalues num_re = re.compile('^\d+$') for key, value in propvalues.items(): if key == self.key: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # try to handle this property try: prop = self.properties[key] except KeyError: raise KeyError, '"%s" has no property "%s"'%(self.classname, key) if value is not None and isinstance(prop, Link): if type(value) != type(''): raise ValueError, 'link value must be String' link_class = self.properties[key].classname # if it isn't a number, it's a key if not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, link_class) elif not self.db.getclass(link_class).hasnode(value): raise IndexError, '%s has no node %s'%(link_class, value) # save off the value propvalues[key] = value # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, value, 'link', (self.classname, newid, key)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of ids'%key # clean up and validate the list of links link_class = self.properties[key].classname l = [] for entry in value: if type(entry) != type(''): raise ValueError, '"%s" multilink value (%r) '\ 'must contain Strings'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, entry, self.properties[key].classname) l.append(entry) value = l propvalues[key] = value # handle additions for nodeid in value: if not self.db.getclass(link_class).hasnode(nodeid): raise IndexError, '%s has no node %s'%(link_class, nodeid) # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) elif isinstance(prop, String): if type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%key if prop.indexme: self.db.indexer.add_text((self.classname, newid, key), value) elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%key elif isinstance(prop, Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'%key elif isinstance(prop, Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'%key elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%key elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%key # make sure there's data where there needs to be for key, prop in self.properties.items(): if propvalues.has_key(key): continue if key == self.key: raise ValueError, 'key property "%s" is required'%key if isinstance(prop, Multilink): propvalues[key] = [] else: propvalues[key] = None # done self.db.addnode(self.classname, newid, propvalues) if self.do_journal: self.db.addjournal(self.classname, newid, ''"create", {}) # XXX numeric ids return str(newid) def get(self, nodeid, propname, default=_marker, cache=1): '''Get the value of a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. 'cache' exists for backwards compatibility, and is not used. ''' if propname == 'id': return nodeid # get the node's dict d = self.db.getnode(self.classname, nodeid) if propname == 'creation': if d.has_key('creation'): return d['creation'] else: return date.Date() if propname == 'activity': if d.has_key('activity'): return d['activity'] else: return date.Date() if propname == 'creator': if d.has_key('creator'): return d['creator'] else: return self.db.getuid() if propname == 'actor': if d.has_key('actor'): return d['actor'] else: return self.db.getuid() # get the property (raises KeyErorr if invalid) prop = self.properties[propname] # XXX may it be that propname is valid property name # (above error is not raised) and not d.has_key(propname)??? if (not d.has_key(propname)) or (d[propname] is None): if default is _marker: if isinstance(prop, Multilink): return [] else: return None else: return default # don't pass our list to other code if isinstance(prop, Multilink): return d[propname][:] return d[propname] def set(self, nodeid, **propvalues): '''Modify a property on an existing node of this class. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. Each key in 'propvalues' must be the name of a property of this class or a KeyError is raised. All values in 'propvalues' must be acceptable types for their corresponding properties or a TypeError is raised. If the value of the key property is set, it must not collide with other key strings or a ValueError is raised. If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. ''' self.fireAuditors('set', nodeid, propvalues) oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) propvalues = self.set_inner(nodeid, **propvalues) self.fireReactors('set', nodeid, oldvalues) return propvalues def set_inner(self, nodeid, **propvalues): ''' Called by set, in-between the audit and react calls. ''' if not propvalues: return propvalues if propvalues.has_key('creation') or propvalues.has_key('creator') or \ propvalues.has_key('actor') or propvalues.has_key('activity'): raise KeyError, '"creation", "creator", "actor" and '\ '"activity" are reserved' if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' node = self.db.getnode(self.classname, nodeid) if self.is_retired(nodeid): raise IndexError, 'Requested item is retired' num_re = re.compile('^\d+$') # make a copy of the values dictionary - we'll modify the contents propvalues = propvalues.copy() # if the journal value is to be different, store it in here journalvalues = {} # remember the add/remove stuff for multilinks, making it easier # for the Database layer to do its stuff multilink_changes = {} for propname, value in propvalues.items(): # check to make sure we're not duplicating an existing key if propname == self.key and node[propname] != value: try: self.lookup(value) except KeyError: pass else: raise ValueError, 'node with key "%s" exists'%value # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. try: prop = self.properties[propname] except KeyError: raise KeyError, '"%s" has no property named "%s"'%( self.classname, propname) # if the value's the same as the existing value, no sense in # doing anything current = node.get(propname, None) if value == current: del propvalues[propname] continue journalvalues[propname] = current # do stuff based on the prop type if isinstance(prop, Link): link_class = prop.classname # if it isn't a number, it's a key if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( propname) if isinstance(value, type('')) and not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, value, prop.classname) if (value is not None and not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) if self.do_journal and prop.do_journal: # register the unlink with the old linked node if node[propname] is not None: self.db.addjournal(link_class, node[propname], ''"unlink", (self.classname, nodeid, propname)) # register the link with the newly linked node if value is not None: self.db.addjournal(link_class, value, ''"link", (self.classname, nodeid, propname)) elif isinstance(prop, Multilink): if type(value) != type([]): raise TypeError, 'new property "%s" not a list of'\ ' ids'%propname link_class = self.properties[propname].classname l = [] for entry in value: # if it isn't a number, it's a key if type(entry) != type(''): raise ValueError, 'new property "%s" link value ' \ 'must be a string'%propname if not num_re.match(entry): try: entry = self.db.classes[link_class].lookup(entry) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( propname, entry, self.properties[propname].classname) l.append(entry) value = l propvalues[propname] = value # figure the journal entry for this property add = [] remove = [] # handle removals if node.has_key(propname): l = node[propname] else: l = [] for id in l[:]: if id in value: continue # register the unlink with the old linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'unlink', (self.classname, nodeid, propname)) l.remove(id) remove.append(id) # handle additions for id in value: if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%(link_class, id) if id in l: continue # register the link with the newly linked node if self.do_journal and self.properties[propname].do_journal: self.db.addjournal(link_class, id, 'link', (self.classname, nodeid, propname)) l.append(id) add.append(id) # figure the journal entry l = [] if add: l.append(('+', add)) if remove: l.append(('-', remove)) multilink_changes[propname] = (add, remove) if l: journalvalues[propname] = tuple(l) elif isinstance(prop, String): if value is not None and type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%propname if prop.indexme: if value is None: value = '' self.db.indexer.add_text((self.classname, nodeid, propname), value) elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% propname propvalues[propname] = value elif value is not None and isinstance(prop, Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an '\ 'Interval'%propname propvalues[propname] = value elif value is not None and isinstance(prop, Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%propname elif value is not None and isinstance(prop, Boolean): try: int(value) except ValueError: raise TypeError, 'new property "%s" not boolean'%propname # nothing to do? if not propvalues: return propvalues # update the activity time propvalues['activity'] = date.Date() propvalues['actor'] = self.db.getuid() # do the set self.db.setnode(self.classname, nodeid, propvalues, multilink_changes) # remove the activity props now they're handled del propvalues['activity'] del propvalues['actor'] # journal the set if self.do_journal: self.db.addjournal(self.classname, nodeid, ''"set", journalvalues) return propvalues def retire(self, nodeid): '''Retire a node. The properties on the node remain available from the get() method, and the node's id is never reused. Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.fireAuditors('retire', nodeid, None) # use the arg for __retired__ to cope with any odd database type # conversion (hello, sqlite) sql = 'update _%s set __retired__=%s where id=%s'%(self.classname, self.db.arg, self.db.arg) self.db.sql(sql, (1, nodeid)) if self.do_journal: self.db.addjournal(self.classname, nodeid, ''"retired", None) self.fireReactors('retire', nodeid, None) def restore(self, nodeid): '''Restore a retired node. Make node available for all operations like it was before retirement. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' node = self.db.getnode(self.classname, nodeid) # check if key property was overrided key = self.getkey() try: id = self.lookup(node[key]) except KeyError: pass else: raise KeyError, "Key property (%s) of retired node clashes with \ existing one (%s)" % (key, node[key]) self.fireAuditors('restore', nodeid, None) # use the arg for __retired__ to cope with any odd database type # conversion (hello, sqlite) sql = 'update _%s set __retired__=%s where id=%s'%(self.classname, self.db.arg, self.db.arg) self.db.sql(sql, (0, nodeid)) if self.do_journal: self.db.addjournal(self.classname, nodeid, ''"restored", None) self.fireReactors('restore', nodeid, None) def is_retired(self, nodeid): '''Return true if the node is rerired ''' sql = 'select __retired__ from _%s where id=%s'%(self.classname, self.db.arg) self.db.sql(sql, (nodeid,)) return int(self.db.sql_fetchone()[0]) def destroy(self, nodeid): '''Destroy a node. WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted WARNING: use retire() instead WARNING: the properties of this node will not be available ever again WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. The node is completely removed from the hyperdb, including all journal entries. It will no longer be available, and will generally break code if there are any references to the node. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' self.db.destroynode(self.classname, nodeid) def history(self, nodeid): '''Retrieve the journal of edits on a particular node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. The returned list contains tuples of the form (nodeid, date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' return self.db.getjournal(self.classname, nodeid) # Locating nodes: def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' return self.db.hasnode(self.classname, nodeid) def setkey(self, propname): '''Select a String property of this class to be the key property. 'propname' must be the name of a String property of this class or None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. ''' prop = self.getprops()[propname] if not isinstance(prop, String): raise TypeError, 'key properties must be String' self.key = propname def getkey(self): '''Return the name of the key property for this class or None.''' return self.key def labelprop(self, default_to_id=0): '''Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: 1. key property 2. "name" property 3. "title" property 4. first property from the sorted property name list ''' k = self.getkey() if k: return k props = self.getprops() if props.has_key('name'): return 'name' elif props.has_key('title'): return 'title' if default_to_id: return 'id' props = props.keys() props.sort() return props[0] def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. If this class has no key property, a TypeError is raised. If the 'keyvalue' matches one of the values for the key property among the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. ''' if not self.key: raise TypeError, 'No key property set for class %s'%self.classname # use the arg to handle any odd database type conversion (hello, # sqlite) sql = "select id from _%s where _%s=%s and __retired__ <> %s"%( self.classname, self.key, self.db.arg, self.db.arg) self.db.sql(sql, (keyvalue, 1)) # see if there was a result that's not retired row = self.db.sql_fetchone() if not row: raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key, keyvalue, self.classname) # return the id # XXX numeric ids return str(row[0]) def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. 'propspec' consists of keyword args propname=nodeid or propname={nodeid:1, } 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Examples:: db.issue.find(messages='1') db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' # shortcut if not propspec: return [] # validate the args props = self.getprops() propspec = propspec.items() for propname, nodeids in propspec: # check the prop is OK prop = props[propname] if not isinstance(prop, Link) and not isinstance(prop, Multilink): raise TypeError, "'%s' not a Link/Multilink property"%propname # first, links a = self.db.arg allvalues = () sql = [] where = [] for prop, values in propspec: if not isinstance(props[prop], hyperdb.Link): continue if type(values) is type({}) and len(values) == 1: values = values.keys()[0] if type(values) is type(''): allvalues += (values,) where.append('_%s = %s'%(prop, a)) elif values is None: where.append('_%s is NULL'%prop) else: values = values.keys() s = '' if None in values: values.remove(None) s = '_%s is NULL or '%prop allvalues += tuple(values) s += '_%s in (%s)'%(prop, ','.join([a]*len(values))) where.append('(' + s +')') if where: allvalues = (1, ) + allvalues sql.append('''select id from _%s where __retired__ <> %s and %s'''%(self.classname, a, ' and '.join(where))) # now multilinks for prop, values in propspec: if not isinstance(props[prop], hyperdb.Multilink): continue if not values: continue allvalues += (1, ) if type(values) is type(''): allvalues += (values,) s = a else: allvalues += tuple(values.keys()) s = ','.join([a]*len(values)) tn = '%s_%s'%(self.classname, prop) sql.append('''select id from _%s, %s where __retired__ <> %s and id = %s.nodeid and %s.linkid in (%s)'''%(self.classname, tn, a, tn, tn, s)) if not sql: return [] sql = ' union '.join(sql) self.db.sql(sql, allvalues) # XXX numeric ids l = [str(x[0]) for x in self.db.sql_fetchall()] return l def stringFind(self, **requirements): '''Locate a particular node by matching a set of its String properties in a caseless search. If the property is not a String property, a TypeError is raised. The return is a list of the id of all nodes that match. ''' where = [] args = [] for propname in requirements.keys(): prop = self.properties[propname] if not isinstance(prop, String): raise TypeError, "'%s' not a String property"%propname where.append(propname) args.append(requirements[propname].lower()) # generate the where clause s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where]) sql = 'select id from _%s where %s and __retired__<>%s'%( self.classname, s, self.db.arg) args.append(1) self.db.sql(sql, tuple(args)) # XXX numeric ids l = [str(x[0]) for x in self.db.sql_fetchall()] return l def list(self): ''' Return a list of the ids of the active nodes in this class. ''' return self.getnodeids(retired=0) def getnodeids(self, retired=None): ''' Retrieve all the ids of the nodes for a particular Class. Set retired=None to get all nodes. Otherwise it'll get all the retired or non-retired nodes, depending on the flag. ''' # flip the sense of the 'retired' flag if we don't want all of them if retired is not None: if retired: args = (0, ) else: args = (1, ) sql = 'select id from _%s where __retired__ <> %s'%(self.classname, self.db.arg) else: args = () sql = 'select id from _%s'%self.classname self.db.sql(sql, args) # XXX numeric ids ids = [str(x[0]) for x in self.db.cursor.fetchall()] return ids def filter(self, search_matches, filterspec, sort=(None,None), group=(None,None)): '''Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec "filterspec" is {propname: value(s)} "sort" and "group" are (dir, prop) where dir is '+', '-' or None and prop is a prop name or None "search_matches" is {nodeid: marker} or None The filter must match all properties specificed - but if the property value to match is a list, any one of the values in the list may match for that property to match. ''' # we can't match anything if search_matches is empty if search_matches == {}: return [] if __debug__: start_t = time.time() cn = self.classname # vars to hold the components of the SQL statement frum = [] # FROM clauses loj = [] # LEFT OUTER JOIN clauses where = [] # WHERE clauses args = [] # *any* positional arguments a = self.db.arg # figure the WHERE clause from the filterspec props = self.getprops() mlfilt = 0 # are we joining with Multilink tables? for k, v in filterspec.items(): propclass = props[k] # now do other where clause stuff if isinstance(propclass, Multilink): mlfilt = 1 tn = '%s_%s'%(cn, k) if v in ('-1', ['-1']): # only match rows that have count(linkid)=0 in the # corresponding multilink table) where.append('_%s.id not in (select nodeid from %s)'%(cn, tn)) elif isinstance(v, type([])): frum.append(tn) s = ','.join([a for x in v]) where.append('_%s.id=%s.nodeid and %s.linkid in (%s)'%(cn, tn, tn, s)) args = args + v else: frum.append(tn) where.append('_%s.id=%s.nodeid and %s.linkid=%s'%(cn, tn, tn, a)) args.append(v) elif k == 'id': if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s.%s in (%s)'%(cn, k, s)) args = args + v else: where.append('_%s.%s=%s'%(cn, k, a)) args.append(v) elif isinstance(propclass, String): if not isinstance(v, type([])): v = [v] # Quote the bits in the string that need it and then embed # in a "substring" search. Note - need to quote the '%' so # they make it through the python layer happily v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v] # now add to the where clause where.append('(' +' and '.join(["_%s._%s LIKE '%s'"%(cn, k, s) for s in v]) +')') # note: args are embedded in the query string now elif isinstance(propclass, Link): if isinstance(v, type([])): d = {} for entry in v: if entry == '-1': entry = None d[entry] = entry l = [] if d.has_key(None) or not d: del d[None] l.append('_%s._%s is NULL'%(cn, k)) if d: v = d.keys() s = ','.join([a for x in v]) l.append('(_%s._%s in (%s))'%(cn, k, s)) args = args + v if l: where.append('(' + ' or '.join(l) +')') else: if v in ('-1', None): v = None where.append('_%s._%s is NULL'%(cn, k)) else: where.append('_%s._%s=%s'%(cn, k, a)) args.append(v) elif isinstance(propclass, Date): dc = self.db.hyperdb_to_sql_value[hyperdb.Date] if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s._%s in (%s)'%(cn, k, s)) args = args + [dc(date.Date(v)) for x in v] else: try: # Try to filter on range of dates date_rng = propclass.range_from_raw (v, self.db) if date_rng.from_value: where.append('_%s._%s >= %s'%(cn, k, a)) args.append(dc(date_rng.from_value)) if date_rng.to_value: where.append('_%s._%s <= %s'%(cn, k, a)) args.append(dc(date_rng.to_value)) except ValueError: # If range creation fails - ignore that search parameter pass elif isinstance(propclass, Interval): # filter using the ___int__ column if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s.__%s_int__ in (%s)'%(cn, k, s)) args = args + [date.Interval(x).as_seconds() for x in v] else: try: # Try to filter on range of intervals date_rng = Range(v, date.Interval) if date_rng.from_value: where.append('_%s.__%s_int__ >= %s'%(cn, k, a)) args.append(date_rng.from_value.as_seconds()) if date_rng.to_value: where.append('_%s.__%s_int__ <= %s'%(cn, k, a)) args.append(date_rng.to_value.as_seconds()) except ValueError: # If range creation fails - ignore that search parameter pass else: if isinstance(v, type([])): s = ','.join([a for x in v]) where.append('_%s._%s in (%s)'%(cn, k, s)) args = args + v else: where.append('_%s._%s=%s'%(cn, k, a)) args.append(v) # don't match retired nodes where.append('_%s.__retired__ <> 1'%cn) # add results of full text search if search_matches is not None: v = search_matches.keys() s = ','.join([a for x in v]) where.append('_%s.id in (%s)'%(cn, s)) args = args + v # sanity check: sorting *and* grouping on the same property? if group[1] == sort[1]: sort = (None, None) # "grouping" is just the first-order sorting in the SQL fetch orderby = [] ordercols = [] mlsort = [] for sortby in group, sort: sdir, prop = sortby if sdir and prop: if isinstance(props[prop], Multilink): mlsort.append(sortby) continue elif isinstance(props[prop], Interval): # use the int column for sorting o = '__'+prop+'_int__' ordercols.append(o) elif isinstance(props[prop], Link): # determine whether the linked Class has an order property lcn = props[prop].classname link = self.db.classes[lcn] o = '_%s._%s'%(cn, prop) if link.getprops().has_key('order'): tn = '_' + lcn loj.append('LEFT OUTER JOIN %s on %s=%s.id'%(tn, o, tn)) ordercols.append(tn + '._order') o = tn + '._order' else: ordercols.append(o) elif prop == 'id': o = '_%s.id'%cn else: o = '_%s._%s'%(cn, prop) ordercols.append(o) if sdir == '-': o += ' desc' orderby.append(o) # construct the SQL frum.append('_'+cn) frum = ','.join(frum) if where: where = ' where ' + (' and '.join(where)) else: where = '' if mlfilt: # we're joining tables on the id, so we will get dupes if we # don't distinct() cols = ['distinct(_%s.id)'%cn] else: cols = ['_%s.id'%cn] if orderby: cols = cols + ordercols order = ' order by %s'%(','.join(orderby)) else: order = '' cols = ','.join(cols) loj = ' '.join(loj) sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order) args = tuple(args) __traceback_info__ = (sql, args) self.db.sql(sql, args) l = self.db.sql_fetchall() # return the IDs (the first column) # XXX numeric ids l = [str(row[0]) for row in l] if not mlsort: if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return l # ergh. someone wants to sort by a multilink. r = [] for id in l: m = [] for ml in mlsort: m.append(self.get(id, ml[1])) r.append((id, m)) i = 0 for sortby in mlsort: def sortfun(a, b, dir=sortby[i], i=i): if dir == '-': return cmp(b[1][i], a[1][i]) else: return cmp(a[1][i], b[1][i]) r.sort(sortfun) i += 1 r = [i[0] for i in r] if __debug__: self.db.stats['filtering'] += (time.time() - start_t) return r def count(self): '''Get the number of nodes in this class. If the returned integer is 'numnodes', the ids of all the nodes in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. ''' return self.db.countnodes(self.classname) # Manipulating properties: def getprops(self, protected=1): '''Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - those which may not be modified. ''' d = self.properties.copy() if protected: d['id'] = String() d['creation'] = hyperdb.Date() d['activity'] = hyperdb.Date() d['creator'] = hyperdb.Link('user') d['actor'] = hyperdb.Link('user') return d def addprop(self, **properties): '''Add properties to this class. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. None of the keys in 'properties' may collide with the names of existing properties, or a ValueError is raised before any properties have been added. ''' for key in properties.keys(): if self.properties.has_key(key): raise ValueError, key self.properties.update(properties) def index(self, nodeid): '''Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, String) and propclass.indexme: self.db.indexer.add_text((self.classname, nodeid, prop), str(self.get(nodeid, prop))) # # Detector interface # def audit(self, event, detector): '''Register a detector ''' l = self.auditors[event] if detector not in l: self.auditors[event].append(detector) def fireAuditors(self, action, nodeid, newvalues): '''Fire all registered auditors. ''' for audit in self.auditors[action]: audit(self.db, self, nodeid, newvalues) def react(self, event, detector): '''Register a detector ''' l = self.reactors[event] if detector not in l: self.reactors[event].append(detector) def fireReactors(self, action, nodeid, oldvalues): '''Fire all registered reactors. ''' for react in self.reactors[action]: react(self.db, self, nodeid, oldvalues) # # import / export support # def export_list(self, propnames, nodeid): ''' Export a node - generate a list of CSV-able data in the order specified by propnames for the given node. ''' properties = self.getprops() l = [] for prop in propnames: proptype = properties[prop] value = self.get(nodeid, prop) # "marshal" data where needed if value is None: pass elif isinstance(proptype, hyperdb.Date): value = value.get_tuple() elif isinstance(proptype, hyperdb.Interval): value = value.get_tuple() elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) l.append(repr(self.is_retired(nodeid))) return l def import_list(self, propnames, proplist): ''' Import a node - all information including "id" is present and should not be sanity checked. Triggers are not triggered. The journal should be initialised using the "creator" and "created" information. Return the nodeid of the node imported. ''' if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' properties = self.getprops() # make the new node's property map d = {} retire = 0 if not "id" in propnames: newid = self.db.newid(self.classname) else: newid = eval(proplist[propnames.index("id")]) for i in range(len(propnames)): # Use eval to reverse the repr() used to output the CSV value = eval(proplist[i]) # Figure the property for this column propname = propnames[i] # "unmarshal" where necessary if propname == 'id': continue elif propname == 'is retired': # is the item retired? if int(value): retire = 1 continue elif value is None: d[propname] = None continue prop = properties[propname] if value is None: # don't set Nones continue elif isinstance(prop, hyperdb.Date): value = date.Date(value) elif isinstance(prop, hyperdb.Interval): value = date.Interval(value) elif isinstance(prop, hyperdb.Password): pwd = password.Password() pwd.unpack(value) value = pwd d[propname] = value if isinstance(prop, String): if type(value) != type('') and type(value) != type(u''): raise TypeError, \ 'new property "%(propname)s" not a string: %(value)r' \ % locals() if prop.indexme: self.db.indexer.add_text((self.classname, newid, propname), value) # get a new id if necessary if newid is None: newid = self.db.newid(self.classname) # insert new node or update existing? if not self.hasnode(newid): self.db.addnode(self.classname, newid, d) # insert else: self.db.setnode(self.classname, newid, d) # update # retire? if retire: # use the arg for __retired__ to cope with any odd database type # conversion (hello, sqlite) sql = 'update _%s set __retired__=%s where id=%s'%(self.classname, self.db.arg, self.db.arg) self.db.sql(sql, (1, newid)) return newid def export_journals(self): '''Export a class's journal - generate a list of lists of CSV-able data: nodeid, date, user, action, params No heading here - the columns are fixed. ''' properties = self.getprops() r = [] for nodeid in self.getnodeids(): for nodeid, date, user, action, params in self.history(nodeid): date = date.get_tuple() if action == 'set': export_data = {} for propname, value in params.items(): if not properties.has_key(propname): # property no longer in the schema continue prop = properties[propname] # make sure the params are eval()'able if value is None: pass elif isinstance(prop, Date): value = value.get_tuple() elif isinstance(prop, Interval): value = value.get_tuple() elif isinstance(prop, Password): value = str(value) export_data[propname] = value params = export_data l = [nodeid, date, user, action, params] r.append(map(repr, l)) return r def import_journals(self, entries): '''Import a class's journal. Uses setjournal() to set the journal for each item.''' properties = self.getprops() d = {} for l in entries: l = map(eval, l) nodeid, jdate, user, action, params = l r = d.setdefault(nodeid, []) if action == 'set': for propname, value in params.items(): prop = properties[propname] if value is None: pass elif isinstance(prop, Date): value = date.Date(value) elif isinstance(prop, Interval): value = date.Interval(value) elif isinstance(prop, Password): pwd = password.Password() pwd.unpack(value) value = pwd params[propname] = value r.append((nodeid, date.Date(jdate), user, action, params)) for nodeid, l in d.items(): self.db.setjournal(self.classname, nodeid, l) class FileClass(hyperdb.FileClass, Class): '''This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. The default MIME type of this data is defined by the "default_mime_type" class attribute, which may be overridden by each node if the class defines a "type" String property. ''' default_mime_type = 'text/plain' def create(self, **propvalues): ''' snaffle the file propvalue and store in a file ''' # we need to fire the auditors now, or the content property won't # be in propvalues for the auditors to play with self.fireAuditors('create', None, propvalues) # now remove the content property so it's not stored in the db content = propvalues['content'] del propvalues['content'] # do the database create newid = self.create_inner(**propvalues) # figure the mime type mime_type = propvalues.get('type', self.default_mime_type) # and index! self.db.indexer.add_text((self.classname, newid, 'content'), content, mime_type) # fire reactors self.fireReactors('create', newid, None) # store off the content as a file self.db.storefile(self.classname, newid, None, content) return newid def get(self, nodeid, propname, default=_marker, cache=1): ''' Trap the content propname and get it from the file 'cache' exists for backwards compatibility, and is not used. ''' poss_msg = 'Possibly a access right configuration problem.' if propname == 'content': try: return self.db.getfile(self.classname, nodeid, None) except IOError, (strerror): # BUG: by catching this we donot see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) if default is not _marker: return Class.get(self, nodeid, propname, default) else: return Class.get(self, nodeid, propname) def getprops(self, protected=1): '''In addition to the actual properties on the node, these methods provide the "content" property. If the "protected" flag is true, we include protected properties - those which may not be modified. Note that the content prop is indexed separately, hence no indexme. ''' d = Class.getprops(self, protected=protected).copy() d['content'] = hyperdb.String() return d def set(self, itemid, **propvalues): ''' Snarf the "content" propvalue and update it in a file ''' self.fireAuditors('set', itemid, propvalues) oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) # now remove the content property so it's not stored in the db content = None if propvalues.has_key('content'): content = propvalues['content'] del propvalues['content'] # do the database create propvalues = self.set_inner(itemid, **propvalues) # do content? if content: # store and index self.db.storefile(self.classname, itemid, None, content) mime_type = None if self.getprops().has_key('type'): mime_type = propvalues.get('type', self.get(itemid, 'type')) if not mime_type: mime_type = self.default_mime_type self.db.indexer.add_text((self.classname, itemid, 'content'), content, mime_type) propvalues['content'] = content # fire reactors self.fireReactors('set', itemid, oldvalues) return propvalues def index(self, nodeid): '''Add (or refresh) the node to search indexes. Pass on the content-type property for the content property. ''' Class.index(self, nodeid) mime_type = None if self.getprops().has_key('type'): mime_type = self.get(nodeid, 'type') if not mime_type: mime_type = self.default_mime_type self.db.indexer.add_text((self.classname, nodeid, 'content'), str(self.get(nodeid, 'content')), mime_type) # XXX deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): '''The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' dictionary attempts to specify any of these properties or a "creation", "creator", "activity" or "actor" property, a ValueError is raised. ''' if not properties.has_key('title'): properties['title'] = hyperdb.String(indexme='yes') if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): # note: journalling is turned off as it really just wastes # space. this behaviour may be overridden in an instance properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) # vim: set et sts=4 sw=4 : code-963003d2ffdd/roundup/backends/sessions_dbm.py0000644000000000000000000001067110344271362020251 0ustar 00000000000000#$Id: sessions_dbm.py,v 1.5 2004-03-31 23:08:38 richard Exp $ """This module defines a very basic store that's used by the CGI interface to store session and one-time-key information. Yes, it's called "sessions" - because originally it only defined a session class. It's now also used for One Time Key handling too. """ __docformat__ = 'restructuredtext' import anydbm, whichdb, os, marshal, time class BasicDatabase: ''' Provide a nice encapsulation of an anydbm store. Keys are id strings, values are automatically marshalled data. ''' _db_type = None def __init__(self, db): self.config = db.config self.dir = db.config.DATABASE # ensure files are group readable and writable os.umask(0002) def exists(self, infoid): db = self.opendb('c') try: return db.has_key(infoid) finally: db.close() def clear(self): path = os.path.join(self.dir, self.name) if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') def cache_db_type(self, path): ''' determine which DB wrote the class file, and cache it as an attribute of __class__ (to allow for subclassed DBs to be different sorts) ''' db_type = '' if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! db_type = 'dbm' self.__class__._db_type = db_type _marker = [] def get(self, infoid, value, default=_marker): db = self.opendb('c') try: if db.has_key(infoid): values = marshal.loads(db[infoid]) else: if default != self._marker: return default raise KeyError, 'No such %s "%s"'%(self.name, infoid) return values.get(value, None) finally: db.close() def getall(self, infoid): db = self.opendb('c') try: try: d = marshal.loads(db[infoid]) del d['__timestamp'] return d except KeyError: raise KeyError, 'No such %s "%s"'%(self.name, infoid) finally: db.close() def set(self, infoid, **newvalues): db = self.opendb('c') try: if db.has_key(infoid): values = marshal.loads(db[infoid]) else: values = {'__timestamp': time.time()} values.update(newvalues) db[infoid] = marshal.dumps(values) finally: db.close() def list(self): db = self.opendb('r') try: return db.keys() finally: db.close() def destroy(self, infoid): db = self.opendb('c') try: if db.has_key(infoid): del db[infoid] finally: db.close() def opendb(self, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' # figure the class db type path = os.path.join(os.getcwd(), self.dir, self.name) if self._db_type is None: self.cache_db_type(path) db_type = self._db_type # new database? let anydbm pick the best dbm if not db_type: return anydbm.open(path, 'c') # open the database with the correct module dbm = __import__(db_type) return dbm.open(path, mode) def commit(self): pass def close(self): pass def updateTimestamp(self, sessid): self.set(sessid, __timestamp=time.time()) def clean(self, now): """Age sessions, remove when they haven't been used for a week. """ week = 60*60*24*7 for sessid in self.list(): sess = self.get(sessid, '__timestamp', None) if sess is None: sess=time.time() self.updateTimestamp(sessid) interval = now - sess if interval > week: self.destroy(sessid) class Sessions(BasicDatabase): name = 'sessions' class OneTimeKeys(BasicDatabase): name = 'otks' code-963003d2ffdd/roundup/backends/sessions_rdbms.py0000644000000000000000000000616310344271362020617 0ustar 00000000000000#$Id: sessions_rdbms.py,v 1.3 2004-05-02 23:16:05 richard Exp $ """This module defines a very basic store that's used by the CGI interface to store session and one-time-key information. Yes, it's called "sessions" - because originally it only defined a session class. It's now also used for One Time Key handling too. """ __docformat__ = 'restructuredtext' import os, time class BasicDatabase: ''' Provide a nice encapsulation of an RDBMS table. Keys are id strings, values are automatically marshalled data. ''' def __init__(self, db): self.db = db self.cursor = self.db.cursor def clear(self): self.cursor.execute('delete from %ss'%self.name) def exists(self, infoid): n = self.name self.cursor.execute('select count(*) from %ss where %s_key=%s'%(n, n, self.db.arg), (infoid,)) return int(self.cursor.fetchone()[0]) _marker = [] def get(self, infoid, value, default=_marker): n = self.name self.cursor.execute('select %s_value from %ss where %s_key=%s'%(n, n, n, self.db.arg), (infoid,)) res = self.cursor.fetchone() if not res: if default != self._marker: return default raise KeyError, 'No such %s "%s"'%(self.name, infoid) values = eval(res[0]) return values.get(value, None) def getall(self, infoid): n = self.name self.cursor.execute('select %s_value from %ss where %s_key=%s'%(n, n, n, self.db.arg), (infoid,)) res = self.cursor.fetchone() if not res: raise KeyError, 'No such %s "%s"'%(self.name, infoid) return eval(res[0]) def set(self, infoid, **newvalues): c = self.cursor n = self.name a = self.db.arg c.execute('select %s_value from %ss where %s_key=%s'%(n, n, n, a), (infoid,)) res = c.fetchone() if res: values = eval(res[0]) else: values = {} values.update(newvalues) if res: sql = 'update %ss set %s_value=%s where %s_key=%s'%(n, n, a, n, a) args = (repr(values), infoid) else: sql = 'insert into %ss (%s_key, %s_time, %s_value) '\ 'values (%s, %s, %s)'%(n, n, n, n, a, a, a) args = (infoid, time.time(), repr(values)) c.execute(sql, args) def destroy(self, infoid): self.cursor.execute('delete from %ss where %s_key=%s'%(self.name, self.name, self.db.arg), (infoid,)) def updateTimestamp(self, infoid): self.cursor.execute('update %ss set %s_time=%s where %s_key=%s'%( self.name, self.name, self.db.arg, self.name, self.db.arg), (time.time(), infoid)) def clean(self, now): """Age sessions, remove when they haven't been used for a week. """ old = now - 60*60*24*7 self.cursor.execute('delete from %ss where %s_time < %s'%(self.name, self.name, self.db.arg), (old, )) class Sessions(BasicDatabase): name = 'session' class OneTimeKeys(BasicDatabase): name = 'otk' code-963003d2ffdd/roundup/backends/tsearch2_setup.py0000644000000000000000000005053210344271362020514 0ustar 00000000000000#$Id: tsearch2_setup.py,v 1.2 2005-01-08 11:25:23 jlgijsbers Exp $ # All the SQL in this module is taken from the tsearch2 module in the contrib # tree of PostgreSQL 7.4.6. PostgreSQL, and this code, has the following # license: # # PostgreSQL Data Base Management System # (formerly known as Postgres, then as Postgres95). # # Portions Copyright (c) 1996-2003, The PostgreSQL Global Development Group # # Portions Copyright (c) 1994, The Regents of the University of California # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose, without fee, and without a written agreement # is hereby granted, provided that the above copyright notice and this # paragraph and the following two paragraphs appear in all copies. # # IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING # LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS # DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS # ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO # PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. tsearch_sql = """ -- Adjust this setting to control where the objects get CREATEd. SET search_path = public; --dict conf CREATE TABLE pg_ts_dict ( dict_name text not null primary key, dict_init oid, dict_initoption text, dict_lexize oid not null, dict_comment text ) with oids; --dict interface CREATE FUNCTION lexize(oid, text) returns _text as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION lexize(text, text) returns _text as '$libdir/tsearch2', 'lexize_byname' language 'C' with (isstrict); CREATE FUNCTION lexize(text) returns _text as '$libdir/tsearch2', 'lexize_bycurrent' language 'C' with (isstrict); CREATE FUNCTION set_curdict(int) returns void as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION set_curdict(text) returns void as '$libdir/tsearch2', 'set_curdict_byname' language 'C' with (isstrict); --built-in dictionaries CREATE FUNCTION dex_init(text) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION dex_lexize(internal,internal,int4) returns internal as '$libdir/tsearch2' language 'C' with (isstrict); insert into pg_ts_dict select 'simple', (select oid from pg_proc where proname='dex_init'), null, (select oid from pg_proc where proname='dex_lexize'), 'Simple example of dictionary.' ; CREATE FUNCTION snb_en_init(text) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION snb_lexize(internal,internal,int4) returns internal as '$libdir/tsearch2' language 'C' with (isstrict); insert into pg_ts_dict select 'en_stem', (select oid from pg_proc where proname='snb_en_init'), '/usr/share/postgresql/contrib/english.stop', (select oid from pg_proc where proname='snb_lexize'), 'English Stemmer. Snowball.' ; CREATE FUNCTION snb_ru_init(text) returns internal as '$libdir/tsearch2' language 'C'; insert into pg_ts_dict select 'ru_stem', (select oid from pg_proc where proname='snb_ru_init'), '/usr/share/postgresql/contrib/russian.stop', (select oid from pg_proc where proname='snb_lexize'), 'Russian Stemmer. Snowball.' ; CREATE FUNCTION spell_init(text) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION spell_lexize(internal,internal,int4) returns internal as '$libdir/tsearch2' language 'C' with (isstrict); insert into pg_ts_dict select 'ispell_template', (select oid from pg_proc where proname='spell_init'), null, (select oid from pg_proc where proname='spell_lexize'), 'ISpell interface. Must have .dict and .aff files' ; CREATE FUNCTION syn_init(text) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION syn_lexize(internal,internal,int4) returns internal as '$libdir/tsearch2' language 'C' with (isstrict); insert into pg_ts_dict select 'synonym', (select oid from pg_proc where proname='syn_init'), null, (select oid from pg_proc where proname='syn_lexize'), 'Example of synonym dictionary' ; --dict conf CREATE TABLE pg_ts_parser ( prs_name text not null primary key, prs_start oid not null, prs_nexttoken oid not null, prs_end oid not null, prs_headline oid not null, prs_lextype oid not null, prs_comment text ) with oids; --sql-level interface CREATE TYPE tokentype as (tokid int4, alias text, descr text); CREATE FUNCTION token_type(int4) returns setof tokentype as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION token_type(text) returns setof tokentype as '$libdir/tsearch2', 'token_type_byname' language 'C' with (isstrict); CREATE FUNCTION token_type() returns setof tokentype as '$libdir/tsearch2', 'token_type_current' language 'C' with (isstrict); CREATE FUNCTION set_curprs(int) returns void as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION set_curprs(text) returns void as '$libdir/tsearch2', 'set_curprs_byname' language 'C' with (isstrict); CREATE TYPE tokenout as (tokid int4, token text); CREATE FUNCTION parse(oid,text) returns setof tokenout as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION parse(text,text) returns setof tokenout as '$libdir/tsearch2', 'parse_byname' language 'C' with (isstrict); CREATE FUNCTION parse(text) returns setof tokenout as '$libdir/tsearch2', 'parse_current' language 'C' with (isstrict); --default parser CREATE FUNCTION prsd_start(internal,int4) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION prsd_getlexeme(internal,internal,internal) returns int4 as '$libdir/tsearch2' language 'C'; CREATE FUNCTION prsd_end(internal) returns void as '$libdir/tsearch2' language 'C'; CREATE FUNCTION prsd_lextype(internal) returns internal as '$libdir/tsearch2' language 'C'; CREATE FUNCTION prsd_headline(internal,internal,internal) returns internal as '$libdir/tsearch2' language 'C'; insert into pg_ts_parser select 'default', (select oid from pg_proc where proname='prsd_start'), (select oid from pg_proc where proname='prsd_getlexeme'), (select oid from pg_proc where proname='prsd_end'), (select oid from pg_proc where proname='prsd_headline'), (select oid from pg_proc where proname='prsd_lextype'), 'Parser from OpenFTS v0.34' ; --tsearch config CREATE TABLE pg_ts_cfg ( ts_name text not null primary key, prs_name text not null, locale text ) with oids; CREATE TABLE pg_ts_cfgmap ( ts_name text not null, tok_alias text not null, dict_name text[], primary key (ts_name,tok_alias) ) with oids; CREATE FUNCTION set_curcfg(int) returns void as '$libdir/tsearch2' language 'C' with (isstrict); CREATE FUNCTION set_curcfg(text) returns void as '$libdir/tsearch2', 'set_curcfg_byname' language 'C' with (isstrict); CREATE FUNCTION show_curcfg() returns oid as '$libdir/tsearch2' language 'C' with (isstrict); insert into pg_ts_cfg values ('default', 'default','C'); insert into pg_ts_cfg values ('default_russian', 'default','ru_RU.KOI8-R'); insert into pg_ts_cfg values ('simple', 'default'); insert into pg_ts_cfgmap values ('default', 'lword', '{en_stem}'); insert into pg_ts_cfgmap values ('default', 'nlword', '{simple}'); insert into pg_ts_cfgmap values ('default', 'word', '{simple}'); insert into pg_ts_cfgmap values ('default', 'email', '{simple}'); insert into pg_ts_cfgmap values ('default', 'url', '{simple}'); insert into pg_ts_cfgmap values ('default', 'host', '{simple}'); insert into pg_ts_cfgmap values ('default', 'sfloat', '{simple}'); insert into pg_ts_cfgmap values ('default', 'version', '{simple}'); insert into pg_ts_cfgmap values ('default', 'part_hword', '{simple}'); insert into pg_ts_cfgmap values ('default', 'nlpart_hword', '{simple}'); insert into pg_ts_cfgmap values ('default', 'lpart_hword', '{en_stem}'); insert into pg_ts_cfgmap values ('default', 'hword', '{simple}'); insert into pg_ts_cfgmap values ('default', 'lhword', '{en_stem}'); insert into pg_ts_cfgmap values ('default', 'nlhword', '{simple}'); insert into pg_ts_cfgmap values ('default', 'uri', '{simple}'); insert into pg_ts_cfgmap values ('default', 'file', '{simple}'); insert into pg_ts_cfgmap values ('default', 'float', '{simple}'); insert into pg_ts_cfgmap values ('default', 'int', '{simple}'); insert into pg_ts_cfgmap values ('default', 'uint', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'lword', '{en_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'nlword', '{ru_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'word', '{ru_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'email', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'url', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'host', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'sfloat', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'version', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'part_hword', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'nlpart_hword', '{ru_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'lpart_hword', '{en_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'hword', '{ru_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'lhword', '{en_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'nlhword', '{ru_stem}'); insert into pg_ts_cfgmap values ('default_russian', 'uri', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'file', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'float', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'int', '{simple}'); insert into pg_ts_cfgmap values ('default_russian', 'uint', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'lword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'nlword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'word', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'email', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'url', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'host', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'sfloat', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'version', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'part_hword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'nlpart_hword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'lpart_hword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'hword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'lhword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'nlhword', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'uri', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'file', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'float', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'int', '{simple}'); insert into pg_ts_cfgmap values ('simple', 'uint', '{simple}'); --tsvector type CREATE FUNCTION tsvector_in(cstring) RETURNS tsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE FUNCTION tsvector_out(tsvector) RETURNS cstring AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE TYPE tsvector ( INTERNALLENGTH = -1, INPUT = tsvector_in, OUTPUT = tsvector_out, STORAGE = extended ); CREATE FUNCTION length(tsvector) RETURNS int4 AS '$libdir/tsearch2', 'tsvector_length' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION to_tsvector(oid, text) RETURNS tsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION to_tsvector(text, text) RETURNS tsvector AS '$libdir/tsearch2', 'to_tsvector_name' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION to_tsvector(text) RETURNS tsvector AS '$libdir/tsearch2', 'to_tsvector_current' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION strip(tsvector) RETURNS tsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION setweight(tsvector,"char") RETURNS tsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict,iscachable); CREATE FUNCTION concat(tsvector,tsvector) RETURNS tsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict,iscachable); CREATE OPERATOR || ( LEFTARG = tsvector, RIGHTARG = tsvector, PROCEDURE = concat ); --query type CREATE FUNCTION tsquery_in(cstring) RETURNS tsquery AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE FUNCTION tsquery_out(tsquery) RETURNS cstring AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE TYPE tsquery ( INTERNALLENGTH = -1, INPUT = tsquery_in, OUTPUT = tsquery_out ); CREATE FUNCTION querytree(tsquery) RETURNS text AS '$libdir/tsearch2', 'tsquerytree' LANGUAGE 'C' with (isstrict); CREATE FUNCTION to_tsquery(oid, text) RETURNS tsquery AS '$libdir/tsearch2' LANGUAGE 'c' with (isstrict,iscachable); CREATE FUNCTION to_tsquery(text, text) RETURNS tsquery AS '$libdir/tsearch2','to_tsquery_name' LANGUAGE 'c' with (isstrict,iscachable); CREATE FUNCTION to_tsquery(text) RETURNS tsquery AS '$libdir/tsearch2','to_tsquery_current' LANGUAGE 'c' with (isstrict,iscachable); --operations CREATE FUNCTION exectsq(tsvector, tsquery) RETURNS bool AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict, iscachable); COMMENT ON FUNCTION exectsq(tsvector, tsquery) IS 'boolean operation with text index'; CREATE FUNCTION rexectsq(tsquery, tsvector) RETURNS bool AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict, iscachable); COMMENT ON FUNCTION rexectsq(tsquery, tsvector) IS 'boolean operation with text index'; CREATE OPERATOR @@ ( LEFTARG = tsvector, RIGHTARG = tsquery, PROCEDURE = exectsq, COMMUTATOR = '@@', RESTRICT = contsel, JOIN = contjoinsel ); CREATE OPERATOR @@ ( LEFTARG = tsquery, RIGHTARG = tsvector, PROCEDURE = rexectsq, COMMUTATOR = '@@', RESTRICT = contsel, JOIN = contjoinsel ); --Trigger CREATE FUNCTION tsearch2() RETURNS trigger AS '$libdir/tsearch2' LANGUAGE 'C'; --Relevation CREATE FUNCTION rank(float4[], tsvector, tsquery) RETURNS float4 AS '$libdir/tsearch2' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank(float4[], tsvector, tsquery, int4) RETURNS float4 AS '$libdir/tsearch2' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank(tsvector, tsquery) RETURNS float4 AS '$libdir/tsearch2', 'rank_def' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank(tsvector, tsquery, int4) RETURNS float4 AS '$libdir/tsearch2', 'rank_def' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank_cd(int4, tsvector, tsquery) RETURNS float4 AS '$libdir/tsearch2' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank_cd(int4, tsvector, tsquery, int4) RETURNS float4 AS '$libdir/tsearch2' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank_cd(tsvector, tsquery) RETURNS float4 AS '$libdir/tsearch2', 'rank_cd_def' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION rank_cd(tsvector, tsquery, int4) RETURNS float4 AS '$libdir/tsearch2', 'rank_cd_def' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(oid, text, tsquery, text) RETURNS text AS '$libdir/tsearch2', 'headline' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(oid, text, tsquery) RETURNS text AS '$libdir/tsearch2', 'headline' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(text, text, tsquery, text) RETURNS text AS '$libdir/tsearch2', 'headline_byname' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(text, text, tsquery) RETURNS text AS '$libdir/tsearch2', 'headline_byname' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(text, tsquery, text) RETURNS text AS '$libdir/tsearch2', 'headline_current' LANGUAGE 'C' WITH (isstrict, iscachable); CREATE FUNCTION headline(text, tsquery) RETURNS text AS '$libdir/tsearch2', 'headline_current' LANGUAGE 'C' WITH (isstrict, iscachable); --GiST --GiST key type CREATE FUNCTION gtsvector_in(cstring) RETURNS gtsvector AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE FUNCTION gtsvector_out(gtsvector) RETURNS cstring AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE TYPE gtsvector ( INTERNALLENGTH = -1, INPUT = gtsvector_in, OUTPUT = gtsvector_out ); -- support FUNCTIONs CREATE FUNCTION gtsvector_consistent(gtsvector,internal,int4) RETURNS bool AS '$libdir/tsearch2' LANGUAGE 'C'; CREATE FUNCTION gtsvector_compress(internal) RETURNS internal AS '$libdir/tsearch2' LANGUAGE 'C'; CREATE FUNCTION gtsvector_decompress(internal) RETURNS internal AS '$libdir/tsearch2' LANGUAGE 'C'; CREATE FUNCTION gtsvector_penalty(internal,internal,internal) RETURNS internal AS '$libdir/tsearch2' LANGUAGE 'C' with (isstrict); CREATE FUNCTION gtsvector_picksplit(internal, internal) RETURNS internal AS '$libdir/tsearch2' LANGUAGE 'C'; CREATE FUNCTION gtsvector_union(bytea, internal) RETURNS _int4 AS '$libdir/tsearch2' LANGUAGE 'C'; CREATE FUNCTION gtsvector_same(gtsvector, gtsvector, internal) RETURNS internal AS '$libdir/tsearch2' LANGUAGE 'C'; -- CREATE the OPERATOR class CREATE OPERATOR CLASS gist_tsvector_ops DEFAULT FOR TYPE tsvector USING gist AS OPERATOR 1 @@ (tsvector, tsquery) RECHECK , FUNCTION 1 gtsvector_consistent (gtsvector, internal, int4), FUNCTION 2 gtsvector_union (bytea, internal), FUNCTION 3 gtsvector_compress (internal), FUNCTION 4 gtsvector_decompress (internal), FUNCTION 5 gtsvector_penalty (internal, internal, internal), FUNCTION 6 gtsvector_picksplit (internal, internal), FUNCTION 7 gtsvector_same (gtsvector, gtsvector, internal), STORAGE gtsvector; --stat info CREATE TYPE statinfo as (word text, ndoc int4, nentry int4); --CREATE FUNCTION tsstat_in(cstring) --RETURNS tsstat --AS '$libdir/tsearch2' --LANGUAGE 'C' with (isstrict); -- --CREATE FUNCTION tsstat_out(tsstat) --RETURNS cstring --AS '$libdir/tsearch2' --LANGUAGE 'C' with (isstrict); -- --CREATE TYPE tsstat ( -- INTERNALLENGTH = -1, -- INPUT = tsstat_in, -- OUTPUT = tsstat_out, -- STORAGE = plain --); -- --CREATE FUNCTION ts_accum(tsstat,tsvector) --RETURNS tsstat --AS '$libdir/tsearch2' --LANGUAGE 'C' with (isstrict); -- --CREATE FUNCTION ts_accum_finish(tsstat) -- returns setof statinfo -- as '$libdir/tsearch2' -- language 'C' -- with (isstrict); -- --CREATE AGGREGATE stat ( -- BASETYPE=tsvector, -- SFUNC=ts_accum, -- STYPE=tsstat, -- FINALFUNC = ts_accum_finish, -- initcond = '' --); CREATE FUNCTION stat(text) returns setof statinfo as '$libdir/tsearch2', 'ts_stat' language 'C' with (isstrict); --reset - just for debuging CREATE FUNCTION reset_tsearch() returns void as '$libdir/tsearch2' language 'C' with (isstrict); --get cover (debug for rank_cd) CREATE FUNCTION get_covers(tsvector,tsquery) returns text as '$libdir/tsearch2' language 'C' with (isstrict); --debug function create type tsdebug as ( ts_name text, tok_type text, description text, token text, dict_name text[], "tsvector" tsvector ); create function _get_parser_from_curcfg() returns text as ' select prs_name from pg_ts_cfg where oid = show_curcfg() ' language 'SQL' with(isstrict,iscachable); create function ts_debug(text) returns setof tsdebug as ' select m.ts_name, t.alias as tok_type, t.descr as description, p.token, m.dict_name, strip(to_tsvector(p.token)) as tsvector from parse( _get_parser_from_curcfg(), $1 ) as p, token_type() as t, pg_ts_cfgmap as m, pg_ts_cfg as c where t.tokid=p.tokid and t.alias = m.tok_alias and m.ts_name=c.ts_name and c.oid=show_curcfg() ' language 'SQL' with(isstrict); """ def setup(cursor): sql = '\n'.join([line for line in tsearch_sql.split('\n') if not line.startswith('--')]) for query in sql.split(';'): if query.strip(): cursor.execute(query)