comparison roundup/hyperdb.py @ 4063:625915ce35b8

Uniformly use """...""" instead of '''...''' for comments.
author Stefan Seefeld <stefan@seefeld.name>
date Fri, 20 Feb 2009 16:03:03 +0000
parents 7ad0918ee8bd
children eddb82d0964c
comparison
equal deleted inserted replaced
4062:6ea417bafd9c 4063:625915ce35b8
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" 14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, 15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 # 17 #
18 # $Id: hyperdb.py,v 1.132 2008-08-18 06:21:53 richard Exp $
19 18
20 """Hyperdatabase implementation, especially field types. 19 """Hyperdatabase implementation, especially field types.
21 """ 20 """
22 __docformat__ = 'restructuredtext' 21 __docformat__ = 'restructuredtext'
23 22
135 134
136 class _Pointer(_Type): 135 class _Pointer(_Type):
137 """An object designating a Pointer property that links or multilinks 136 """An object designating a Pointer property that links or multilinks
138 to a node in a specified class.""" 137 to a node in a specified class."""
139 def __init__(self, classname, do_journal='yes', required=False): 138 def __init__(self, classname, do_journal='yes', required=False):
140 ''' Default is to journal link and unlink events 139 """ Default is to journal link and unlink events
141 ''' 140 """
142 super(_Pointer, self).__init__(required) 141 super(_Pointer, self).__init__(required)
143 self.classname = classname 142 self.classname = classname
144 self.do_journal = do_journal == 'yes' 143 self.do_journal = do_journal == 'yes'
145 def __repr__(self): 144 def __repr__(self):
146 """more useful for dumps. But beware: This is also used in schema 145 """more useful for dumps. But beware: This is also used in schema
271 # Support for splitting designators 270 # Support for splitting designators
272 # 271 #
273 class DesignatorError(ValueError): 272 class DesignatorError(ValueError):
274 pass 273 pass
275 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')): 274 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
276 ''' Take a foo123 and return ('foo', 123) 275 """ Take a foo123 and return ('foo', 123)
277 ''' 276 """
278 m = dre.match(designator) 277 m = dre.match(designator)
279 if m is None: 278 if m is None:
280 raise DesignatorError, _('"%s" not a node designator')%designator 279 raise DesignatorError, _('"%s" not a node designator')%designator
281 return m.group(1), m.group(2) 280 return m.group(1), m.group(2)
282 281
283 class Proptree(object): 282 class Proptree(object):
284 ''' Simple tree data structure for optimizing searching of 283 """ Simple tree data structure for optimizing searching of
285 properties. Each node in the tree represents a roundup Class 284 properties. Each node in the tree represents a roundup Class
286 Property that has to be navigated for finding the given search 285 Property that has to be navigated for finding the given search
287 or sort properties. The sort_type attribute is used for 286 or sort properties. The sort_type attribute is used for
288 distinguishing nodes in the tree used for sorting or searching: If 287 distinguishing nodes in the tree used for sorting or searching: If
289 it is 0 for a node, that node is not used for sorting. If it is 1, 288 it is 0 for a node, that node is not used for sorting. If it is 1,
291 for sorting only. 290 for sorting only.
292 291
293 The Proptree is also used for transitively searching attributes for 292 The Proptree is also used for transitively searching attributes for
294 backends that do not support transitive search (e.g. anydbm). The 293 backends that do not support transitive search (e.g. anydbm). The
295 _val attribute with set_val is used for this. 294 _val attribute with set_val is used for this.
296 ''' 295 """
297 296
298 def __init__(self, db, cls, name, props, parent = None): 297 def __init__(self, db, cls, name, props, parent = None):
299 self.db = db 298 self.db = db
300 self.name = name 299 self.name = name
301 self.props = props 300 self.props = props
563 562
564 # 563 #
565 # the base Database class 564 # the base Database class
566 # 565 #
567 class DatabaseError(ValueError): 566 class DatabaseError(ValueError):
568 '''Error to be raised when there is some problem in the database code 567 """Error to be raised when there is some problem in the database code
569 ''' 568 """
570 pass 569 pass
571 class Database: 570 class Database:
572 '''A database for storing records containing flexible data types. 571 """A database for storing records containing flexible data types.
573 572
574 This class defines a hyperdatabase storage layer, which the Classes use to 573 This class defines a hyperdatabase storage layer, which the Classes use to
575 store their data. 574 store their data.
576 575
577 576
590 Implementation 589 Implementation
591 -------------- 590 --------------
592 591
593 All methods except __repr__ must be implemented by a concrete backend Database. 592 All methods except __repr__ must be implemented by a concrete backend Database.
594 593
595 ''' 594 """
596 595
597 # flag to set on retired entries 596 # flag to set on retired entries
598 RETIRED_FLAG = '__hyperdb_retired' 597 RETIRED_FLAG = '__hyperdb_retired'
599 598
600 BACKEND_MISSING_STRING = None 599 BACKEND_MISSING_STRING = None
632 def __getattr__(self, classname): 631 def __getattr__(self, classname):
633 """A convenient way of calling self.getclass(classname).""" 632 """A convenient way of calling self.getclass(classname)."""
634 raise NotImplementedError 633 raise NotImplementedError
635 634
636 def addclass(self, cl): 635 def addclass(self, cl):
637 '''Add a Class to the hyperdatabase. 636 """Add a Class to the hyperdatabase.
638 ''' 637 """
639 raise NotImplementedError 638 raise NotImplementedError
640 639
641 def getclasses(self): 640 def getclasses(self):
642 """Return a list of the names of all existing classes.""" 641 """Return a list of the names of all existing classes."""
643 raise NotImplementedError 642 raise NotImplementedError
648 If 'classname' is not a valid class name, a KeyError is raised. 647 If 'classname' is not a valid class name, a KeyError is raised.
649 """ 648 """
650 raise NotImplementedError 649 raise NotImplementedError
651 650
652 def clear(self): 651 def clear(self):
653 '''Delete all database contents. 652 """Delete all database contents.
654 ''' 653 """
655 raise NotImplementedError 654 raise NotImplementedError
656 655
657 def getclassdb(self, classname, mode='r'): 656 def getclassdb(self, classname, mode='r'):
658 '''Obtain a connection to the class db that will be used for 657 """Obtain a connection to the class db that will be used for
659 multiple actions. 658 multiple actions.
660 ''' 659 """
661 raise NotImplementedError 660 raise NotImplementedError
662 661
663 def addnode(self, classname, nodeid, node): 662 def addnode(self, classname, nodeid, node):
664 """Add the specified node to its class's db. 663 """Add the specified node to its class's db.
665 """ 664 """
666 raise NotImplementedError 665 raise NotImplementedError
667 666
668 def serialise(self, classname, node): 667 def serialise(self, classname, node):
669 '''Copy the node contents, converting non-marshallable data into 668 """Copy the node contents, converting non-marshallable data into
670 marshallable data. 669 marshallable data.
671 ''' 670 """
672 return node 671 return node
673 672
674 def setnode(self, classname, nodeid, node): 673 def setnode(self, classname, nodeid, node):
675 '''Change the specified node. 674 """Change the specified node.
676 ''' 675 """
677 raise NotImplementedError 676 raise NotImplementedError
678 677
679 def unserialise(self, classname, node): 678 def unserialise(self, classname, node):
680 '''Decode the marshalled node data 679 """Decode the marshalled node data
681 ''' 680 """
682 return node 681 return node
683 682
684 def getnode(self, classname, nodeid): 683 def getnode(self, classname, nodeid):
685 '''Get a node from the database. 684 """Get a node from the database.
686 685
687 'cache' exists for backwards compatibility, and is not used. 686 'cache' exists for backwards compatibility, and is not used.
688 ''' 687 """
689 raise NotImplementedError 688 raise NotImplementedError
690 689
691 def hasnode(self, classname, nodeid): 690 def hasnode(self, classname, nodeid):
692 '''Determine if the database has a given node. 691 """Determine if the database has a given node.
693 ''' 692 """
694 raise NotImplementedError 693 raise NotImplementedError
695 694
696 def countnodes(self, classname): 695 def countnodes(self, classname):
697 '''Count the number of nodes that exist for a particular Class. 696 """Count the number of nodes that exist for a particular Class.
698 ''' 697 """
699 raise NotImplementedError 698 raise NotImplementedError
700 699
701 def storefile(self, classname, nodeid, property, content): 700 def storefile(self, classname, nodeid, property, content):
702 '''Store the content of the file in the database. 701 """Store the content of the file in the database.
703 702
704 The property may be None, in which case the filename does not 703 The property may be None, in which case the filename does not
705 indicate which property is being saved. 704 indicate which property is being saved.
706 ''' 705 """
707 raise NotImplementedError 706 raise NotImplementedError
708 707
709 def getfile(self, classname, nodeid, property): 708 def getfile(self, classname, nodeid, property):
710 '''Get the content of the file in the database. 709 """Get the content of the file in the database.
711 ''' 710 """
712 raise NotImplementedError 711 raise NotImplementedError
713 712
714 def addjournal(self, classname, nodeid, action, params): 713 def addjournal(self, classname, nodeid, action, params):
715 ''' Journal the Action 714 """ Journal the Action
716 'action' may be: 715 'action' may be:
717 716
718 'create' or 'set' -- 'params' is a dictionary of property values 717 'create' or 'set' -- 'params' is a dictionary of property values
719 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 718 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
720 'retire' -- 'params' is None 719 'retire' -- 'params' is None
721 ''' 720 """
722 raise NotImplementedError 721 raise NotImplementedError
723 722
724 def getjournal(self, classname, nodeid): 723 def getjournal(self, classname, nodeid):
725 ''' get the journal for id 724 """ get the journal for id
726 ''' 725 """
727 raise NotImplementedError 726 raise NotImplementedError
728 727
729 def pack(self, pack_before): 728 def pack(self, pack_before):
730 ''' pack the database 729 """ pack the database
731 ''' 730 """
732 raise NotImplementedError 731 raise NotImplementedError
733 732
734 def commit(self): 733 def commit(self):
735 ''' Commit the current transactions. 734 """ Commit the current transactions.
736 735
737 Save all data changed since the database was opened or since the 736 Save all data changed since the database was opened or since the
738 last commit() or rollback(). 737 last commit() or rollback().
739 738
740 fail_ok indicates that the commit is allowed to fail. This is used 739 fail_ok indicates that the commit is allowed to fail. This is used
741 in the web interface when committing cleaning of the session 740 in the web interface when committing cleaning of the session
742 database. We don't care if there's a concurrency issue there. 741 database. We don't care if there's a concurrency issue there.
743 742
744 The only backend this seems to affect is postgres. 743 The only backend this seems to affect is postgres.
745 ''' 744 """
746 raise NotImplementedError 745 raise NotImplementedError
747 746
748 def rollback(self): 747 def rollback(self):
749 ''' Reverse all actions from the current transaction. 748 """ Reverse all actions from the current transaction.
750 749
751 Undo all the changes made since the database was opened or the last 750 Undo all the changes made since the database was opened or the last
752 commit() or rollback() was performed. 751 commit() or rollback() was performed.
753 ''' 752 """
754 raise NotImplementedError 753 raise NotImplementedError
755 754
756 def close(self): 755 def close(self):
757 """Close the database. 756 """Close the database.
758 757
796 actions = "create set retire restore".split() 795 actions = "create set retire restore".split()
797 self.auditors = dict([(a, PrioList()) for a in actions]) 796 self.auditors = dict([(a, PrioList()) for a in actions])
798 self.reactors = dict([(a, PrioList()) for a in actions]) 797 self.reactors = dict([(a, PrioList()) for a in actions])
799 798
800 def __repr__(self): 799 def __repr__(self):
801 '''Slightly more useful representation 800 """Slightly more useful representation
802 ''' 801 """
803 return '<hyperdb.Class "%s">'%self.classname 802 return '<hyperdb.Class "%s">'%self.classname
804 803
805 # Editing nodes: 804 # Editing nodes:
806 805
807 def create(self, **propvalues): 806 def create(self, **propvalues):
835 """ 834 """
836 raise NotImplementedError 835 raise NotImplementedError
837 836
838 # not in spec 837 # not in spec
839 def getnode(self, nodeid): 838 def getnode(self, nodeid):
840 ''' Return a convenience wrapper for the node. 839 """ Return a convenience wrapper for the node.
841 840
842 'nodeid' must be the id of an existing node of this class or an 841 'nodeid' must be the id of an existing node of this class or an
843 IndexError is raised. 842 IndexError is raised.
844 843
845 'cache' exists for backwards compatibility, and is not used. 844 'cache' exists for backwards compatibility, and is not used.
846 ''' 845 """
847 return Node(self, nodeid) 846 return Node(self, nodeid)
848 847
849 def getnodeids(self, retired=None): 848 def getnodeids(self, retired=None):
850 '''Retrieve all the ids of the nodes for a particular Class. 849 """Retrieve all the ids of the nodes for a particular Class.
851 ''' 850 """
852 raise NotImplementedError 851 raise NotImplementedError
853 852
854 def set(self, nodeid, **propvalues): 853 def set(self, nodeid, **propvalues):
855 """Modify a property on an existing node of this class. 854 """Modify a property on an existing node of this class.
856 855
881 methods, and other nodes may reuse the values of their key properties. 880 methods, and other nodes may reuse the values of their key properties.
882 """ 881 """
883 raise NotImplementedError 882 raise NotImplementedError
884 883
885 def restore(self, nodeid): 884 def restore(self, nodeid):
886 '''Restpre a retired node. 885 """Restpre a retired node.
887 886
888 Make node available for all operations like it was before retirement. 887 Make node available for all operations like it was before retirement.
889 ''' 888 """
890 raise NotImplementedError 889 raise NotImplementedError
891 890
892 def is_retired(self, nodeid): 891 def is_retired(self, nodeid):
893 '''Return true if the node is rerired 892 """Return true if the node is rerired
894 ''' 893 """
895 raise NotImplementedError 894 raise NotImplementedError
896 895
897 def destroy(self, nodeid): 896 def destroy(self, nodeid):
898 """Destroy a node. 897 """Destroy a node.
899 898
930 """ 929 """
931 raise NotImplementedError 930 raise NotImplementedError
932 931
933 # Locating nodes: 932 # Locating nodes:
934 def hasnode(self, nodeid): 933 def hasnode(self, nodeid):
935 '''Determine if the given nodeid actually exists 934 """Determine if the given nodeid actually exists
936 ''' 935 """
937 raise NotImplementedError 936 raise NotImplementedError
938 937
939 def setkey(self, propname): 938 def setkey(self, propname):
940 """Select a String property of this class to be the key property. 939 """Select a String property of this class to be the key property.
941 940
1228 propnames.sort() 1227 propnames.sort()
1229 return propnames 1228 return propnames
1230 1229
1231 1230
1232 class HyperdbValueError(ValueError): 1231 class HyperdbValueError(ValueError):
1233 ''' Error converting a raw value into a Hyperdb value ''' 1232 """ Error converting a raw value into a Hyperdb value """
1234 pass 1233 pass
1235 1234
1236 def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')): 1235 def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')):
1237 ''' Convert the link value (may be id or key value) to an id value. ''' 1236 """ Convert the link value (may be id or key value) to an id value. """
1238 linkcl = db.classes[prop.classname] 1237 linkcl = db.classes[prop.classname]
1239 if not idre.match(value): 1238 if not idre.match(value):
1240 if linkcl.getkey(): 1239 if linkcl.getkey():
1241 try: 1240 try:
1242 value = linkcl.lookup(value) 1241 value = linkcl.lookup(value)
1257 """ 1256 """
1258 text = text.replace('\r\n', '\n') 1257 text = text.replace('\r\n', '\n')
1259 return text.replace('\r', '\n') 1258 return text.replace('\r', '\n')
1260 1259
1261 def rawToHyperdb(db, klass, itemid, propname, value, **kw): 1260 def rawToHyperdb(db, klass, itemid, propname, value, **kw):
1262 ''' Convert the raw (user-input) value to a hyperdb-storable value. The 1261 """ Convert the raw (user-input) value to a hyperdb-storable value. The
1263 value is for the "propname" property on itemid (may be None for a 1262 value is for the "propname" property on itemid (may be None for a
1264 new item) of "klass" in "db". 1263 new item) of "klass" in "db".
1265 1264
1266 The value is usually a string, but in the case of multilink inputs 1265 The value is usually a string, but in the case of multilink inputs
1267 it may be either a list of strings or a string with comma-separated 1266 it may be either a list of strings or a string with comma-separated
1268 values. 1267 values.
1269 ''' 1268 """
1270 properties = klass.getprops() 1269 properties = klass.getprops()
1271 1270
1272 # ensure it's a valid property name 1271 # ensure it's a valid property name
1273 propname = propname.strip() 1272 propname = propname.strip()
1274 try: 1273 try:
1286 propname=propname, itemid=itemid, **kw) 1285 propname=propname, itemid=itemid, **kw)
1287 1286
1288 return value 1287 return value
1289 1288
1290 class FileClass: 1289 class FileClass:
1291 ''' A class that requires the "content" property and stores it on 1290 """ A class that requires the "content" property and stores it on
1292 disk. 1291 disk.
1293 ''' 1292 """
1294 default_mime_type = 'text/plain' 1293 default_mime_type = 'text/plain'
1295 1294
1296 def __init__(self, db, classname, **properties): 1295 def __init__(self, db, classname, **properties):
1297 '''The newly-created class automatically includes the "content" 1296 """The newly-created class automatically includes the "content"
1298 property. 1297 property.
1299 ''' 1298 """
1300 if not properties.has_key('content'): 1299 if not properties.has_key('content'):
1301 properties['content'] = String(indexme='yes') 1300 properties['content'] = String(indexme='yes')
1302 1301
1303 def export_propnames(self): 1302 def export_propnames(self):
1304 ''' Don't export the "content" property 1303 """ Don't export the "content" property
1305 ''' 1304 """
1306 propnames = self.getprops().keys() 1305 propnames = self.getprops().keys()
1307 propnames.remove('content') 1306 propnames.remove('content')
1308 propnames.sort() 1307 propnames.sort()
1309 return propnames 1308 return propnames
1310 1309
1311 def exportFilename(self, dirname, nodeid): 1310 def exportFilename(self, dirname, nodeid):
1312 subdir_filename = self.db.subdirFilename(self.classname, nodeid) 1311 subdir_filename = self.db.subdirFilename(self.classname, nodeid)
1313 return os.path.join(dirname, self.classname+'-files', subdir_filename) 1312 return os.path.join(dirname, self.classname+'-files', subdir_filename)
1314 1313
1315 def export_files(self, dirname, nodeid): 1314 def export_files(self, dirname, nodeid):
1316 ''' Export the "content" property as a file, not csv column 1315 """ Export the "content" property as a file, not csv column
1317 ''' 1316 """
1318 source = self.db.filename(self.classname, nodeid) 1317 source = self.db.filename(self.classname, nodeid)
1319 1318
1320 dest = self.exportFilename(dirname, nodeid) 1319 dest = self.exportFilename(dirname, nodeid)
1321 ensureParentsExist(dest) 1320 ensureParentsExist(dest)
1322 shutil.copyfile(source, dest) 1321 shutil.copyfile(source, dest)
1323 1322
1324 def import_files(self, dirname, nodeid): 1323 def import_files(self, dirname, nodeid):
1325 ''' Import the "content" property as a file 1324 """ Import the "content" property as a file
1326 ''' 1325 """
1327 source = self.exportFilename(dirname, nodeid) 1326 source = self.exportFilename(dirname, nodeid)
1328 1327
1329 dest = self.db.filename(self.classname, nodeid, create=1) 1328 dest = self.db.filename(self.classname, nodeid, create=1)
1330 ensureParentsExist(dest) 1329 ensureParentsExist(dest)
1331 shutil.copyfile(source, dest) 1330 shutil.copyfile(source, dest)
1339 if props['content'].indexme: 1338 if props['content'].indexme:
1340 self.db.indexer.add_text((self.classname, nodeid, 'content'), 1339 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1341 self.get(nodeid, 'content'), mime_type) 1340 self.get(nodeid, 'content'), mime_type)
1342 1341
1343 class Node: 1342 class Node:
1344 ''' A convenience wrapper for the given node 1343 """ A convenience wrapper for the given node
1345 ''' 1344 """
1346 def __init__(self, cl, nodeid, cache=1): 1345 def __init__(self, cl, nodeid, cache=1):
1347 self.__dict__['cl'] = cl 1346 self.__dict__['cl'] = cl
1348 self.__dict__['nodeid'] = nodeid 1347 self.__dict__['nodeid'] = nodeid
1349 def keys(self, protected=1): 1348 def keys(self, protected=1):
1350 return self.cl.getprops(protected=protected).keys() 1349 return self.cl.getprops(protected=protected).keys()
1390 def retire(self): 1389 def retire(self):
1391 return self.cl.retire(self.nodeid) 1390 return self.cl.retire(self.nodeid)
1392 1391
1393 1392
1394 def Choice(name, db, *options): 1393 def Choice(name, db, *options):
1395 '''Quick helper to create a simple class with choices 1394 """Quick helper to create a simple class with choices
1396 ''' 1395 """
1397 cl = Class(db, name, name=String(), order=String()) 1396 cl = Class(db, name, name=String(), order=String())
1398 for i in range(len(options)): 1397 for i in range(len(options)):
1399 cl.create(name=options[i], order=i) 1398 cl.create(name=options[i], order=i)
1400 return Link(name) 1399 return Link(name)
1401 1400

Roundup Issue Tracker: http://roundup-tracker.org/