code-6257d4e49d2a/roundup/backends/.cvsignore0000644000000000000000000000000607437056447017142 0ustar 00000000000000*.pyc code-6257d4e49d2a/roundup/backends/__init__.py0000644000000000000000000000617507437056447017270 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # # $Id: __init__.py,v 1.11 2002-02-16 08:39:42 richard Exp $ __all__ = [] try: import sys if not hasattr(sys, 'version_info') or sys.version_info < (2,1,2): import anydbm, dumbdbm # dumbdbm only works in python 2.1.2+ assert anydbm._defaultmod != dumbdbm del anydbm del dumbdbm import back_anydbm anydbm = back_anydbm __all__.append('anydbm') except AssertionError: pass except ImportError: pass try: import back_bsddb bsddb = back_bsddb __all__.append('bsddb') except ImportError: pass try: import back_bsddb3 bsddb3 = back_bsddb3 __all__.append('bsddb3') except ImportError: pass # # $Log: not supported by cvs2svn $ # Revision 1.10 2002/01/22 07:08:50 richard # I was certain I'd already done this (there's even a change note in # CHANGES)... # # Revision 1.9 2001/12/12 02:30:51 richard # I fixed the problems with people whose anydbm was using the dbm module at the # backend. It turns out the dbm module modifies the file name to append ".db" # and my check to determine if we're opening an existing or new db just # tested os.path.exists() on the filename. Well, no longer! We now perform a # much better check _and_ cope with the anydbm implementation module changing # too! # I also fixed the backends __init__ so only ImportError is squashed. # # Revision 1.8 2001/12/10 22:20:01 richard # Enabled transaction support in the bsddb backend. It uses the anydbm code # where possible, only replacing methods where the db is opened (it uses the # btree opener specifically.) # Also cleaned up some change note generation. # Made the backends package work with pydoc too. # # Revision 1.7 2001/12/10 00:57:38 richard # From CHANGES: # . Added the "display" command to the admin tool - displays a node's values # . #489760 ] [issue] only subject # . fixed the doc/index.html to include the quoting in the mail alias. # # Also: # . fixed roundup-admin so it works with transactions # . disabled the back_anydbm module if anydbm tries to use dumbdbm # # Revision 1.6 2001/08/07 00:24:42 richard # stupid typo # # Revision 1.5 2001/08/07 00:15:51 richard # Added the copyright/license notice to (nearly) all files at request of # Bizar Software. # # # # vim: set filetype=python ts=4 sw=4 et si code-6257d4e49d2a/roundup/backends/back_anydbm.py0000644000000000000000000005344207437056447017762 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_anydbm.py,v 1.30 2002-02-27 03:40:59 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several serious bugs, and is not available) ''' import whichdb, anydbm, os, marshal from roundup import hyperdb, date from blobfiles import FileStorage # # Now the database # class Database(FileStorage, hyperdb.Database): """A database for storing records containing flexible data types. Transaction stuff TODO: . check the timestamp of the class file and nuke the cache if it's modified. Do some sort of conflict checking on the dirty stuff. . perhaps detect write collisions (related to above)? """ def __init__(self, config, journaltag=None): """Open a hyperdatabase given a specifier to some storage. The 'storagelocator' is obtained from config.DATABASE. The meaning of 'storagelocator' depends on the particular implementation of the hyperdatabase. It could be a file name, a directory path, a socket descriptor for a connection to a database over the network, etc. The 'journaltag' is a token that will be attached to the journal entries for any edits done on the database. If 'journaltag' is None, the database is opened in read-only mode: the Class.create(), Class.set(), and Class.retire() methods are disabled. """ self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.cache = {} # cache of nodes loaded or created self.dirtynodes = {} # keep track of the dirty nodes by class self.newnodes = {} # keep track of the new nodes by class self.transactions = [] def __repr__(self): return ''%id(self) # # Classes # def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" if self.classes.has_key(classname): if hyperdb.DEBUG: print '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): if hyperdb.DEBUG: print 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl def getclasses(self): """Return a list of the names of all existing classes.""" if hyperdb.DEBUG: print 'getclasses', (self,) l = self.classes.keys() l.sort() return l def getclass(self, classname): """Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. """ if hyperdb.DEBUG: print 'getclass', (self, classname) return self.classes[classname] # # Class DBs # def clear(self): '''Delete all database contents ''' if hyperdb.DEBUG: print 'clear', (self,) for cn in self.classes.keys(): for dummy in 'nodes', 'journals': path = os.path.join(self.dir, 'journals.%s'%cn) if os.path.exists(path): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' if hyperdb.DEBUG: print 'getclassdb', (self, classname, mode) return self._opendb('nodes.%s'%classname, mode) def _opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if hyperdb.DEBUG: print '_opendb', (self, name, mode) # determine which DB wrote the class file db_type = '' path = os.path.join(os.getcwd(), self.dir, name) if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! db_type = 'dbm' # new database? let anydbm pick the best dbm if not db_type: if hyperdb.DEBUG: print "_opendb anydbm.open(%r, 'n')"%path return anydbm.open(path, 'n') # open the database with the correct module try: dbm = __import__(db_type) except ImportError: raise hyperdb.DatabaseError, \ "Couldn't open database - the required module '%s'"\ "is not available"%db_type if hyperdb.DEBUG: print "_opendb %r.open(%r, %r)"%(db_type, path, mode) return dbm.open(path, mode) # # Nodes # def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' if hyperdb.DEBUG: print 'addnode', (self, classname, nodeid, node) self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node self.savenode(classname, nodeid, node) def setnode(self, classname, nodeid, node): ''' change the specified node ''' if hyperdb.DEBUG: print 'setnode', (self, classname, nodeid, node) self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node self.cache[classname][nodeid] = node self.savenode(classname, nodeid, node) def savenode(self, classname, nodeid, node): ''' perform the saving of data specified by the set/addnode ''' if hyperdb.DEBUG: print 'savenode', (self, classname, nodeid, node) self.transactions.append((self._doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database ''' if hyperdb.DEBUG: print 'getnode', (self, classname, nodeid, db) if cache: # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): return cache[nodeid] # get from the database and save in the cache if db is None: db = self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, "no such %s %s"%(classname, nodeid) res = marshal.loads(db[nodeid]) if cache: cache[nodeid] = res return res def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' if hyperdb.DEBUG: print 'hasnode', (self, classname, nodeid, db) # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): return 1 # not in the cache - check the database if db is None: db = self.getclassdb(classname) res = db.has_key(nodeid) return res def countnodes(self, classname, db=None): if hyperdb.DEBUG: print 'countnodes', (self, classname, db) # include the new nodes not saved to the DB yet count = len(self.newnodes.get(classname, {})) # and count those in the DB if db is None: db = self.getclassdb(classname) count = count + len(db.keys()) return count def getnodeids(self, classname, db=None): if hyperdb.DEBUG: print 'getnodeids', (self, classname, db) # start off with the new nodes res = self.newnodes.get(classname, {}).keys() if db is None: db = self.getclassdb(classname) res = res + db.keys() return res # # Files - special node properties # inherited from FileStorage # # Journal # def addjournal(self, classname, nodeid, action, params): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' if hyperdb.DEBUG: print 'addjournal', (self, classname, nodeid, action, params) self.transactions.append((self._doSaveJournal, (classname, nodeid, action, params))) def getjournal(self, classname, nodeid): ''' get the journal for id ''' if hyperdb.DEBUG: print 'getjournal', (self, classname, nodeid) # attempt to open the journal - in some rare cases, the journal may # not exist try: db = self._opendb('journals.%s'%classname, 'r') except anydbm.error, error: if str(error) == "need 'c' or 'n' flag to open new db": return [] elif error.args[0] != 2: raise return [] journal = marshal.loads(db[nodeid]) res = [] for entry in journal: (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, user, action, params)) return res def pack(self, pack_before): ''' delete all journal entries before 'pack_before' ''' if hyperdb.DEBUG: print 'packjournal', (self, pack_before) pack_before = pack_before.get_tuple() classes = self.getclasses() # TODO: factor this out to method - we're already doing it in # _opendb. db_type = '' path = os.path.join(os.getcwd(), self.dir, classes[0]) if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: raise hyperdb.DatabaseError, "Couldn't identify database type" elif os.path.exists(path+'.db'): db_type = 'dbm' for classname in classes: db_name = 'journals.%s'%classname db = self._opendb(db_name, 'w') for key in db.keys(): journal = marshal.loads(db[key]) l = [] last_set_entry = None for entry in journal: (nodeid, date_stamp, self.journaltag, action, params) = entry if date_stamp > pack_before or action == 'create': l.append(entry) elif action == 'set': # grab the last set entry to keep information on # activity last_set_entry = entry if last_set_entry: date_stamp = last_set_entry[1] # if the last set entry was made after the pack date # then it is already in the list if date_stamp < pack_before: l.append(last_set_entry) db[key] = marshal.dumps(l) if db_type == 'gdbm': db.reorganize() db.close() # # Basic transaction support # def commit(self): ''' Commit the current transactions. ''' if hyperdb.DEBUG: print 'commit', (self,) # TODO: lock the DB # keep a handle to all the database files opened self.databases = {} # now, do all the transactions for method, args in self.transactions: method(*args) # now close all the database files for db in self.databases.values(): db.close() del self.databases # TODO: unlock the DB # all transactions committed, back to normal self.cache = {} self.dirtynodes = {} self.newnodes = {} self.transactions = [] def _doSaveNode(self, classname, nodeid, node): if hyperdb.DEBUG: print '_doSaveNode', (self, classname, nodeid, node) # get the database handle db_name = 'nodes.%s'%classname if self.databases.has_key(db_name): db = self.databases[db_name] else: db = self.databases[db_name] = self.getclassdb(classname, 'c') # now save the marshalled data db[nodeid] = marshal.dumps(node) def _doSaveJournal(self, classname, nodeid, action, params): entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) if hyperdb.DEBUG: print '_doSaveJournal', entry # get the database handle db_name = 'journals.%s'%classname if self.databases.has_key(db_name): db = self.databases[db_name] else: db = self.databases[db_name] = self._opendb(db_name, 'c') # now insert the journal entry if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) def _doStoreFile(self, name, **databases): # the file is currently ".tmp" - move it to its real name to commit os.rename(name+".tmp", name) def rollback(self): ''' Reverse all actions from the current transaction. ''' if hyperdb.DEBUG: print 'rollback', (self, ) for method, args in self.transactions: # delete temporary files if method == self._doStoreFile: if os.path.exists(args[0]+".tmp"): os.remove(args[0]+".tmp") self.cache = {} self.dirtynodes = {} self.newnodes = {} self.transactions = [] # #$Log: not supported by cvs2svn $ #Revision 1.29 2002/02/25 14:34:31 grubert # . use blobfiles in back_anydbm which is used in back_bsddb. # change test_db as dirlist does not work for subdirectories. # ATTENTION: blobfiles now creates subdirectories for files. # #Revision 1.28 2002/02/16 09:14:17 richard # . #514854 ] History: "User" is always ticket creator # #Revision 1.27 2002/01/22 07:21:13 richard #. fixed back_bsddb so it passed the journal tests # #... it didn't seem happy using the back_anydbm _open method, which is odd. #Yet another occurrance of whichdb not being able to recognise older bsddb #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the #process. # #Revision 1.26 2002/01/22 05:18:38 rochecompaan #last_set_entry was referenced before assignment # #Revision 1.25 2002/01/22 05:06:08 rochecompaan #We need to keep the last 'set' entry in the journal to preserve #information on 'activity' for nodes. # #Revision 1.24 2002/01/21 16:33:20 rochecompaan #You can now use the roundup-admin tool to pack the database # #Revision 1.23 2002/01/18 04:32:04 richard #Rollback was breaking because a message hadn't actually been written to the file. Needs #more investigation. # #Revision 1.22 2002/01/14 02:20:15 richard # . changed all config accesses so they access either the instance or the # config attriubute on the db. This means that all config is obtained from # instance_config instead of the mish-mash of classes. This will make # switching to a ConfigParser setup easier too, I hope. # #At a minimum, this makes migration a _little_ easier (a lot easier in the #0.5.0 switch, I hope!) # #Revision 1.21 2002/01/02 02:31:38 richard #Sorry for the huge checkin message - I was only intending to implement #496356 #but I found a number of places where things had been broken by transactions: # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename # for _all_ roundup-generated smtp messages to be sent to. # . the transaction cache had broken the roundupdb.Class set() reactors # . newly-created author users in the mailgw weren't being committed to the db # #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working #on when I found that stuff :): # . #496356 ] Use threading in messages # . detectors were being registered multiple times # . added tests for mailgw # . much better attaching of erroneous messages in the mail gateway # #Revision 1.20 2001/12/18 15:30:34 rochecompaan #Fixed bugs: # . Fixed file creation and retrieval in same transaction in anydbm # backend # . Cgi interface now renders new issue after issue creation # . Could not set issue status to resolved through cgi interface # . Mail gateway was changing status back to 'chatting' if status was # omitted as an argument # #Revision 1.19 2001/12/17 03:52:48 richard #Implemented file store rollback. As a bonus, the hyperdb is now capable of #storing more than one file per node - if a property name is supplied, #the file is called designator.property. #I decided not to migrate the existing files stored over to the new naming #scheme - the FileClass just doesn't specify the property name. # #Revision 1.18 2001/12/16 10:53:38 richard #take a copy of the node dict so that the subsequent set #operation doesn't modify the oldvalues structure # #Revision 1.17 2001/12/14 23:42:57 richard #yuck, a gdbm instance tests false :( #I've left the debugging code in - it should be removed one day if we're ever #_really_ anal about performace :) # #Revision 1.16 2001/12/12 03:23:14 richard #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb #incorrectly identifies a dbm file as a dbhash file on my system. This has #been submitted to the python bug tracker as issue #491888: #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470 # #Revision 1.15 2001/12/12 02:30:51 richard #I fixed the problems with people whose anydbm was using the dbm module at the #backend. It turns out the dbm module modifies the file name to append ".db" #and my check to determine if we're opening an existing or new db just #tested os.path.exists() on the filename. Well, no longer! We now perform a #much better check _and_ cope with the anydbm implementation module changing #too! #I also fixed the backends __init__ so only ImportError is squashed. # #Revision 1.14 2001/12/10 22:20:01 richard #Enabled transaction support in the bsddb backend. It uses the anydbm code #where possible, only replacing methods where the db is opened (it uses the #btree opener specifically.) #Also cleaned up some change note generation. #Made the backends package work with pydoc too. # #Revision 1.13 2001/12/02 05:06:16 richard #. We now use weakrefs in the Classes to keep the database reference, so # the close() method on the database is no longer needed. # I bumped the minimum python requirement up to 2.1 accordingly. #. #487480 ] roundup-server #. #487476 ] INSTALL.txt # #I also cleaned up the change message / post-edit stuff in the cgi client. #There's now a clearly marked "TODO: append the change note" where I believe #the change note should be added there. The "changes" list will obviously #have to be modified to be a dict of the changes, or somesuch. # #More testing needed. # #Revision 1.12 2001/12/01 07:17:50 richard #. We now have basic transaction support! Information is only written to # the database when the commit() method is called. Only the anydbm # backend is modified in this way - neither of the bsddb backends have been. # The mail, admin and cgi interfaces all use commit (except the admin tool # doesn't have a commit command, so interactive users can't commit...) #. Fixed login/registration forwarding the user to the right page (or not, # on a failure) # #Revision 1.11 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.10 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.9 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.8 2001/09/29 13:27:00 richard #CGI interfaces now spit up a top-level index of all the instances they can #serve. # #Revision 1.7 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.6 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.5 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.4 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.3 2001/07/25 01:23:07 richard #Added the Roundup spec to the new documentation directory. # #Revision 1.2 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # # code-6257d4e49d2a/roundup/backends/back_bsddb.py0000644000000000000000000001506207437056447017562 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_bsddb.py,v 1.16 2002-02-27 03:40:59 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' import bsddb, os, marshal from roundup import hyperdb, date # these classes are so similar, we just use the anydbm methods import back_anydbm # # Now the database # class Database(back_anydbm.Database): """A database for storing records containing flexible data types.""" # # Class DBs # def clear(self): for cn in self.classes.keys(): db = os.path.join(self.dir, 'nodes.%s'%cn) bsddb.btopen(db, 'n') db = os.path.join(self.dir, 'journals.%s'%cn) bsddb.btopen(db, 'n') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname) if os.path.exists(path): return bsddb.btopen(path, mode) else: return bsddb.btopen(path, 'n') def _opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if hyperdb.DEBUG: print self, '_opendb', (self, name, mode) # determine which DB wrote the class file path = os.path.join(os.getcwd(), self.dir, name) if not os.path.exists(path): if hyperdb.DEBUG: print "_opendb bsddb.open(%r, 'n')"%path return bsddb.btopen(path, 'n') # open the database with the correct module if hyperdb.DEBUG: print "_opendb bsddb.open(%r, %r)"%(path, mode) return bsddb.btopen(path, mode) # # Journal # def getjournal(self, classname, nodeid): ''' get the journal for id ''' # attempt to open the journal - in some rare cases, the journal may # not exist try: db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb.error, error: if error.args[0] != 2: raise return [] # mor handling of bad journals if not db.has_key(nodeid): return [] journal = marshal.loads(db[nodeid]) res = [] for entry in journal: (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, user, action, params)) db.close() return res def _doSaveJournal(self, classname, nodeid, action, params): entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) db.close() # #$Log: not supported by cvs2svn $ #Revision 1.15 2002/02/16 09:15:33 richard #forgot to patch bsddb backend too # #Revision 1.14 2002/01/22 07:21:13 richard #. fixed back_bsddb so it passed the journal tests # #... it didn't seem happy using the back_anydbm _open method, which is odd. #Yet another occurrance of whichdb not being able to recognise older bsddb #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the #process. # #Revision 1.13 2001/12/10 22:20:01 richard #Enabled transaction support in the bsddb backend. It uses the anydbm code #where possible, only replacing methods where the db is opened (it uses the #btree opener specifically.) #Also cleaned up some change note generation. #Made the backends package work with pydoc too. # #Revision 1.12 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.11 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.10 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.9 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.8 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.7 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.6 2001/07/30 02:36:23 richard #Handle non-existence of db files in the other backends (code from anydbm). # #Revision 1.5 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.4 2001/07/23 08:25:33 richard #more handling of bad journals # #Revision 1.3 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # #Revision 1.2 2001/07/23 07:56:05 richard #Storing only marshallable data in the db - no nasty pickled class references. # #Revision 1.1 2001/07/23 07:22:13 richard #*sigh* some databases have _foo.so as their underlying implementation. #This time for sure, Rocky. # #Revision 1.1 2001/07/23 07:15:57 richard #Moved the backends into the backends package. Anydbm hasn't been tested at all. # #Revision 1.1 2001/07/23 06:23:41 richard #moved hyper_bsddb.py to the new backends package as bsddb.py # #Revision 1.2 2001/07/22 12:09:32 richard #Final commit of Grande Splite # #Revision 1.1 2001/07/22 11:58:35 richard #More Grande Splite # code-6257d4e49d2a/roundup/backends/back_bsddb3.py0000644000000000000000000002160407437056447017644 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: back_bsddb3.py,v 1.11 2002-01-14 02:20:15 richard Exp $ import bsddb3, os, marshal from roundup import hyperdb, date, password # # Now the database # class Database(hyperdb.Database): """A database for storing records containing flexible data types.""" def __init__(self, config, journaltag=None): """Open a hyperdatabase given a specifier to some storage. The 'storagelocator' is obtained from config.DATABASE. The meaning of 'storagelocator' depends on the particular implementation of the hyperdatabase. It could be a file name, a directory path, a socket descriptor for a connection to a database over the network, etc. The 'journaltag' is a token that will be attached to the journal entries for any edits done on the database. If 'journaltag' is None, the database is opened in read-only mode: the Class.create(), Class.set(), and Class.retire() methods are disabled. """ self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} # # Classes # def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" return self.classes[classname] def addclass(self, cl): cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl def getclasses(self): """Return a list of the names of all existing classes.""" l = self.classes.keys() l.sort() return l def getclass(self, classname): """Get the Class object representing a particular class. If 'classname' is not a valid class name, a KeyError is raised. """ return self.classes[classname] # # Class DBs # def clear(self): for cn in self.classes.keys(): db = os.path.join(self.dir, 'nodes.%s'%cn) bsddb3.btopen(db, 'c') db = os.path.join(self.dir, 'journals.%s'%cn) bsddb3.btopen(db, 'c') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for multiple actions ''' path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname) if os.path.exists(path): return bsddb3.btopen(path, mode) else: return bsddb3.btopen(path, 'c') # # Nodes # def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' db = self.getclassdb(classname, 'c') # now save the marshalled data db[nodeid] = marshal.dumps(node) db.close() setnode = addnode def getnode(self, classname, nodeid, cldb=None): ''' add the specified node to its class's db ''' db = cldb or self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, nodeid res = marshal.loads(db[nodeid]) if not cldb: db.close() return res def hasnode(self, classname, nodeid, cldb=None): ''' add the specified node to its class's db ''' db = cldb or self.getclassdb(classname) res = db.has_key(nodeid) if not cldb: db.close() return res def countnodes(self, classname, cldb=None): db = cldb or self.getclassdb(classname) return len(db.keys()) if not cldb: db.close() return res def getnodeids(self, classname, cldb=None): db = cldb or self.getclassdb(classname) res = db.keys() if not cldb: db.close() return res # # Journal # def addjournal(self, classname, nodeid, action, params): ''' Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(db[nodeid]) l.append(entry) else: l = [entry] db[nodeid] = marshal.dumps(l) db.close() def getjournal(self, classname, nodeid): ''' get the journal for id ''' # attempt to open the journal - in some rare cases, the journal may # not exist try: db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb3.error, error: if error.args[0] != 2: raise return [] # mor handling of bad journals if not db.has_key(nodeid): return [] journal = marshal.loads(db[nodeid]) res = [] for entry in journal: (nodeid, date_stamp, self.journaltag, action, params) = entry date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, self.journaltag, action, params)) db.close() return res def close(self): ''' Close the Database - we must release the circular refs so that we can be del'ed and the underlying bsddb connections closed cleanly. ''' self.classes = {} # # Basic transaction support # # TODO: well, write these methods (and then use them in other code) def register_action(self): ''' Register an action to the transaction undo log ''' def commit(self): ''' Commit the current transaction, start a new one ''' def rollback(self): ''' Reverse all actions from the current transaction ''' # #$Log: not supported by cvs2svn $ #Revision 1.10 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # #Revision 1.9 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the #journal call, and removes duplication of code from the backends. The #backend code now only sees strings. # #Revision 1.8 2001/10/09 07:25:59 richard #Added the Password property type. See "pydoc roundup.password" for #implementation details. Have updated some of the documentation too. # #Revision 1.7 2001/08/12 06:32:36 richard #using isinstance(blah, Foo) now instead of isFooType # #Revision 1.6 2001/08/07 00:24:42 richard #stupid typo # #Revision 1.5 2001/08/07 00:15:51 richard #Added the copyright/license notice to (nearly) all files at request of #Bizar Software. # #Revision 1.4 2001/08/03 02:45:47 anthonybaxter #'n' -> 'c' for create. # #Revision 1.3 2001/07/30 02:36:23 richard #Handle non-existence of db files in the other backends (code from anydbm). # #Revision 1.2 2001/07/30 01:41:36 richard #Makes schema changes mucho easier. # #Revision 1.1 2001/07/24 04:26:03 anthonybaxter #bsddb3 implementation. For now, it's the bsddb implementation with a "3" #added in crayon. # #Revision 1.4 2001/07/23 08:25:33 richard #more handling of bad journals # #Revision 1.3 2001/07/23 08:20:44 richard #Moved over to using marshal in the bsddb and anydbm backends. #roundup-admin now has a "freshen" command that'll load/save all nodes (not # retired - mod hyperdb.Class.list() so it lists retired nodes) # #Revision 1.2 2001/07/23 07:56:05 richard #Storing only marshallable data in the db - no nasty pickled class references. # #Revision 1.1 2001/07/23 07:22:13 richard #*sigh* some databases have _foo.so as their underlying implementation. #This time for sure, Rocky. # #Revision 1.1 2001/07/23 07:15:57 richard #Moved the backends into the backends package. Anydbm hasn't been tested at all. # #Revision 1.1 2001/07/23 06:23:41 richard #moved hyper_bsddb.py to the new backends package as bsddb.py # #Revision 1.2 2001/07/22 12:09:32 richard #Final commit of Grande Splite # #Revision 1.1 2001/07/22 11:58:35 richard #More Grande Splite # code-6257d4e49d2a/roundup/backends/blobfiles.py0000644000000000000000000000777707437056447017503 0ustar 00000000000000# # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # #$Id: blobfiles.py,v 1.2 2002-02-27 03:40:59 richard Exp $ ''' This module exports file storage for roundup backends. Files are stored into a directory hierarchy. ''' import os, os.path def files_in_dir(dir): if not os.path.exists(dir): return 0 num_files = 0 for dir_entry in os.listdir(dir): full_filename = os.path.join(dir,dir_entry) if os.path.isfile(full_filename): num_files = num_files + 1 elif os.path.isdir(full_filename): num_files = num_files + files_in_dir(full_filename) return num_files class FileStorage: """Store files in some directory structure""" # TODO: maybe set "files" # def __init__(self): # pass def filename(self, classname, nodeid, property=None): '''Determine what the filename for the given node and optionally property is. ''' if property: name = '%s%s.%s'%(classname, nodeid, property) else: # roundupdb.FileClass never specified the property name, so don't # include it name = '%s%s'%(classname, nodeid) # have a separate subdir for every thousand messages subdir = str(int(nodeid) / 1000) return os.path.join(self.dir, 'files', classname, subdir, name) def filename_flat(self, classname, nodeid, property=None): '''Determine what the filename for the given node and optionally property is. ''' if property: return os.path.join(self.dir, 'files', '%s%s.%s'%(classname, nodeid, property)) else: # roundupdb.FileClass never specified the property name, so don't # include it return os.path.join(self.dir, 'files', '%s%s'%(classname, nodeid)) def storefile(self, classname, nodeid, property, content): '''Store the content of the file in the database. The property may be None, in which case the filename does not indicate which property is being saved. ''' name = self.filename(classname, nodeid, property) if not os.path.exists(os.path.dirname(name)): os.makedirs(os.path.dirname(name)) open(name + '.tmp', 'wb').write(content) self.transactions.append((self._doStoreFile, (name, ))) def getfile(self, classname, nodeid, property): '''Get the content of the file in the database. ''' filename = self.filename(classname, nodeid, property) try: return open(filename, 'rb').read() except: try: return open(filename+'.tmp', 'rb').read() except: # fallback to flat file storage filename = self.filename_flat(classname, nodeid, property) return open(filename, 'rb').read() def numfiles(self): '''Get number of files in storage, even across subdirectories. ''' files_dir = os.path.join(self.dir, 'files') return files_in_dir(files_dir) def _doStoreFile(self, name, **databases): '''Must be implemented by subclass ''' raise NotImplementedError