Mercurial > p > roundup > code
changeset 8304:24549122f9b1
Factor common code to hyperdb/roundupdb
There was some common copied code in backends/back_anydbm.py and
backends/rdbms_common.py. This is now moved to hyperdb.py and
roundupdb.py, respectively (the FileClass lives in hyperdb.py while the
IssueClass is in roundupdb.py)
| author | Ralf Schlatterbeck <rsc@runtux.com> |
|---|---|
| date | Sat, 01 Mar 2025 13:08:09 +0100 |
| parents | 45ec660eb7f7 |
| children | a81a3cd067fa |
| files | roundup/backends/back_anydbm.py roundup/backends/rdbms_common.py roundup/hyperdb.py roundup/roundupdb.py |
| diffstat | 4 files changed, 193 insertions(+), 330 deletions(-) [+] |
line wrap: on
line diff
--- a/roundup/backends/back_anydbm.py Thu Feb 27 10:05:38 2025 +0100 +++ b/roundup/backends/back_anydbm.py Sat Mar 01 13:08:09 2025 +0100 @@ -31,7 +31,7 @@ import time from roundup.anypy.dbm_ import anydbm, whichdb -from roundup.anypy.strings import b2s, bs2b, repr_export, eval_import, is_us +from roundup.anypy.strings import b2s, repr_export, eval_import, is_us from roundup import hyperdb, date, password, roundupdb, security, support from roundup.mlink_expr import Expression, ExpressionError @@ -48,8 +48,6 @@ from roundup.backends.indexer_common import get_indexer -from hashlib import md5 - def db_exists(config): # check for the user db @@ -2221,168 +2219,19 @@ class FileClass(hyperdb.FileClass, Class): - """This class defines a large chunk of data. To support this, it has a - mandatory String property "content" which is typically saved off - externally to the hyperdb. - - The default MIME type of this data is defined by the - "default_mime_type" class attribute, which may be overridden by each - node if the class defines a "type" String property. - """ + # Use for explicit upcalls in generic code, for py2 compat we cannot + # use super() without making everything a new-style class. + subclass = Class def __init__(self, db, classname, **properties): - """The newly-created class automatically includes the "content" - and "type" properties. - """ - if 'content' not in properties: - properties['content'] = hyperdb.String(indexme='yes') - if 'type' not in properties: - properties['type'] = hyperdb.String() + self._update_properties(properties) Class.__init__(self, db, classname, **properties) - def create(self, **propvalues): - """ Snarf the "content" propvalue and store in a file - """ - # we need to fire the auditors now, or the content property won't - # be in propvalues for the auditors to play with - self.fireAuditors('create', None, propvalues) - - # now remove the content property so it's not stored in the db - content = propvalues['content'] - del propvalues['content'] - - # make sure we have a MIME type - mime_type = propvalues.get('type', self.default_mime_type) - - # do the database create - newid = self.create_inner(**propvalues) - - # store off the content as a file - self.db.storefile(self.classname, newid, None, bs2b(content)) - - # fire reactors - self.fireReactors('create', newid, None) - - return newid - - def get(self, nodeid, propname, default=_marker, cache=1): - """ Trap the content propname and get it from the file - - 'cache' exists for backwards compatibility, and is not used. - """ - poss_msg = 'Possibly an access right configuration problem.' - if propname == 'content': - try: - return b2s(self.db.getfile(self.classname, nodeid, None)) - except IOError as strerror: - # XXX by catching this we don't see an error in the log. - return 'ERROR reading file: %s%s\n%s\n%s' % ( - self.classname, nodeid, poss_msg, strerror) - except UnicodeDecodeError as e: - # if content is not text (e.g. jpeg file) we get - # unicode error trying to convert to string in python 3. - # trap it and supply an error message. Include md5sum - # of content as this string is included in the etag - # calculation of the object. - return ('%s%s is not text, retrieve using ' - 'binary_content property. mdsum: %s') % ( - self.classname, nodeid, - md5(self.db.getfile( - self.classname, nodeid, None)).hexdigest()) # nosec - bandit md5 use ok - elif propname == 'binary_content': - return self.db.getfile(self.classname, nodeid, None) - - if default is not _marker: - return Class.get(self, nodeid, propname, default) - else: - return Class.get(self, nodeid, propname) - - def set(self, itemid, **propvalues): - """ Snarf the "content" propvalue and update it in a file - """ - self.fireAuditors('set', itemid, propvalues) - - # create the oldvalues dict - fill in any missing values - oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) - for name, prop in self.getprops(protected=0).items(): - if name in oldvalues: - continue - if isinstance(prop, hyperdb.Multilink): - oldvalues[name] = [] - else: - oldvalues[name] = None - - # now remove the content property so it's not stored in the db - content = None - if 'content' in propvalues: - content = propvalues['content'] - del propvalues['content'] - - # do the database update - propvalues = self.set_inner(itemid, **propvalues) - - # do content? - if content: - # store and possibly index - self.db.storefile(self.classname, itemid, None, bs2b(content)) - if self.properties['content'].indexme: - index_content = content - if bytes != str and isinstance(content, bytes): - index_content = content.decode('utf-8', errors='ignore') - mime_type = self.get(itemid, 'type', self.default_mime_type) - self.db.indexer.add_text((self.classname, itemid, 'content'), - index_content, mime_type) - propvalues['content'] = content - - # fire reactors - self.fireReactors('set', itemid, oldvalues) - return propvalues - - def index(self, nodeid): - """ Add (or refresh) the node to search indexes. - - Use the content-type property for the content property. - """ - # find all the String properties that have indexme - for prop, propclass in self.getprops().items(): - if prop == 'content' and propclass.indexme: - mime_type = self.get(nodeid, 'type', self.default_mime_type) - index_content = self.get(nodeid, 'binary_content') - if bytes != str and isinstance(index_content, bytes): - index_content = index_content.decode('utf-8', - errors='ignore') - self.db.indexer.add_text((self.classname, nodeid, 'content'), - index_content, mime_type) - elif isinstance(propclass, hyperdb.String) and propclass.indexme: - # index them under (classname, nodeid, property) - try: - value = str(self.get(nodeid, prop)) - except IndexError: - # node has been destroyed - continue - self.db.indexer.add_text((self.classname, nodeid, prop), value) - - -# deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): - # Overridden methods: + # Use for explicit upcalls in generic code, for py2 compat we cannot + # use super() without making everything a new-style class. + subclass = Class def __init__(self, db, classname, **properties): - """The newly-created class automatically includes the "messages", - "files", "nosy", and "superseder" properties. If the 'properties' - dictionary attempts to specify any of these properties or a - "creation" or "activity" property, a ValueError is raised. - """ - if 'title' not in properties: - properties['title'] = hyperdb.String(indexme='yes') - if 'messages' not in properties: - properties['messages'] = hyperdb.Multilink("msg") - if 'files' not in properties: - properties['files'] = hyperdb.Multilink("file") - if 'nosy' not in properties: - # note: journalling is turned off as it really just wastes - # space. this behaviour may be overridden in an instance - properties['nosy'] = hyperdb.Multilink("user", do_journal="no") - if 'superseder' not in properties: - properties['superseder'] = hyperdb.Multilink(classname) + self._update_properties(classname, properties) Class.__init__(self, db, classname, **properties) # vim: set et sts=4 sw=4 :
--- a/roundup/backends/rdbms_common.py Thu Feb 27 10:05:38 2025 +0100 +++ b/roundup/backends/rdbms_common.py Sat Mar 01 13:08:09 2025 +0100 @@ -59,11 +59,9 @@ import re import time -from hashlib import md5 - # roundup modules from roundup import hyperdb, date, password, roundupdb, security, support -from roundup.anypy.strings import b2s, bs2b, us2s, repr_export, eval_import +from roundup.anypy.strings import us2s, repr_export, eval_import from roundup.backends.blobfiles import FileStorage from roundup.backends.indexer_common import get_indexer from roundup.backends.indexer_common import Indexer as CommonIndexer @@ -3411,170 +3409,19 @@ class FileClass(hyperdb.FileClass, Class): - """This class defines a large chunk of data. To support this, it has a - mandatory String property "content" which is typically saved off - externally to the hyperdb. - - The default MIME type of this data is defined by the - "default_mime_type" class attribute, which may be overridden by each - node if the class defines a "type" String property. - """ + # Use for explicit upcalls in generic code, for py2 compat we cannot + # use super() without making everything a new-style class. + subclass = Class def __init__(self, db, classname, **properties): - """The newly-created class automatically includes the "content" - and "type" properties. - """ - if 'content' not in properties: - properties['content'] = hyperdb.String(indexme='yes') - if 'type' not in properties: - properties['type'] = hyperdb.String() + self._update_properties(properties) Class.__init__(self, db, classname, **properties) - def create(self, **propvalues): - """ snaffle the file propvalue and store in a file - """ - # we need to fire the auditors now, or the content property won't - # be in propvalues for the auditors to play with - self.fireAuditors('create', None, propvalues) - - # now remove the content property so it's not stored in the db - content = propvalues['content'] - del propvalues['content'] - - # do the database create - newid = self.create_inner(**propvalues) - - # figure the mime type - mime_type = propvalues.get('type', self.default_mime_type) - - # and index! - if self.properties['content'].indexme: - index_content = content - if bytes != str and isinstance(content, bytes): - index_content = content.decode('utf-8', errors='ignore') - self.db.indexer.add_text((self.classname, newid, 'content'), - index_content, mime_type) - - # store off the content as a file - self.db.storefile(self.classname, newid, None, bs2b(content)) - - # fire reactors - self.fireReactors('create', newid, None) - - return newid - - def get(self, nodeid, propname, default=_marker, cache=1): - """ Trap the content propname and get it from the file - - 'cache' exists for backwards compatibility, and is not used. - """ - poss_msg = 'Possibly a access right configuration problem.' - if propname == 'content': - try: - return b2s(self.db.getfile(self.classname, nodeid, None)) - except IOError as strerror: - # BUG: by catching this we donot see an error in the log. - return 'ERROR reading file: %s%s\n%s\n%s' % ( - self.classname, nodeid, poss_msg, strerror) - except UnicodeDecodeError: - # if content is not text (e.g. jpeg file) we get - # unicode error trying to convert to string in python 3. - # trap it and supply an error message. Include md5sum - # of content as this string is included in the etag - # calculation of the object. - return ('%s%s is not text, retrieve using ' - 'binary_content property. mdsum: %s') % ( - self.classname, nodeid, - md5(self.db.getfile( - self.classname, - nodeid, - None)).hexdigest()) # nosec - bandit md5 use ok - elif propname == 'binary_content': - return self.db.getfile(self.classname, nodeid, None) - - if default is not _marker: - return Class.get(self, nodeid, propname, default) - else: - return Class.get(self, nodeid, propname) - - def set(self, itemid, **propvalues): - """ Snarf the "content" propvalue and update it in a file - """ - self.fireAuditors('set', itemid, propvalues) - oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) - - # now remove the content property so it's not stored in the db - content = None - if 'content' in propvalues: - content = propvalues['content'] - del propvalues['content'] - - # do the database create - propvalues = self.set_inner(itemid, **propvalues) - - # do content? - if content: - # store and possibly index - self.db.storefile(self.classname, itemid, None, bs2b(content)) - if self.properties['content'].indexme: - mime_type = self.get(itemid, 'type', self.default_mime_type) - index_content = content - if bytes != str and isinstance(content, bytes): - index_content = content.decode('utf-8', errors='ignore') - self.db.indexer.add_text((self.classname, itemid, 'content'), - index_content, mime_type) - propvalues['content'] = content - - # fire reactors - self.fireReactors('set', itemid, oldvalues) - return propvalues - - def index(self, nodeid): - """ Add (or refresh) the node to search indexes. - - Use the content-type property for the content property. - """ - # find all the String properties that have indexme - for prop, propclass in self.getprops().items(): - if prop == 'content' and propclass.indexme: - mime_type = self.get(nodeid, 'type', self.default_mime_type) - index_content = self.get(nodeid, 'binary_content') - if bytes != str and isinstance(index_content, bytes): - index_content = index_content.decode('utf-8', - errors='ignore') - self.db.indexer.add_text((self.classname, nodeid, 'content'), - index_content, mime_type) - elif isinstance(propclass, hyperdb.String) and propclass.indexme: - # index them under (classname, nodeid, property) - try: - value = str(self.get(nodeid, prop)) - except IndexError: - # node has been destroyed - continue - self.db.indexer.add_text((self.classname, nodeid, prop), value) - - -# XXX deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): - # Overridden methods: + # Use for explicit upcalls in generic code, for py2 compat we cannot + # use super() without making everything a new-style class. + subclass = Class def __init__(self, db, classname, **properties): - """The newly-created class automatically includes the "messages", - "files", "nosy", and "superseder" properties. If the 'properties' - dictionary attempts to specify any of these properties or a - "creation", "creator", "activity" or "actor" property, a ValueError - is raised. - """ - if 'title' not in properties: - properties['title'] = hyperdb.String(indexme='yes') - if 'messages' not in properties: - properties['messages'] = hyperdb.Multilink("msg") - if 'files' not in properties: - properties['files'] = hyperdb.Multilink("file") - if 'nosy' not in properties: - # note: journalling is turned off as it really just wastes - # space. this behaviour may be overridden in an instance - properties['nosy'] = hyperdb.Multilink("user", do_journal="no") - if 'superseder' not in properties: - properties['superseder'] = hyperdb.Multilink(classname) + self._update_properties(classname, properties) Class.__init__(self, db, classname, **properties) # vim: set et sts=4 sw=4 :
--- a/roundup/hyperdb.py Thu Feb 27 10:05:38 2025 +0100 +++ b/roundup/hyperdb.py Sat Mar 01 13:08:09 2025 +0100 @@ -21,6 +21,7 @@ __docformat__ = 'restructuredtext' # standard python modules +import copy import logging import os import re @@ -29,6 +30,8 @@ import traceback import weakref +from hashlib import md5 + # roundup modules from . import date, password from .support import ensureParentsExist, PrioList @@ -36,10 +39,13 @@ from roundup.i18n import _ from roundup.cgi.exceptions import DetectorError from roundup.anypy.cmp_ import NoneAndDictComparable -from roundup.anypy.strings import eval_import +from roundup.anypy.strings import b2s, bs2b, eval_import logger = logging.getLogger('roundup.hyperdb') +# marker used for an unspecified keyword argument +_marker = [] + # # Types @@ -1229,8 +1235,6 @@ """ raise NotImplementedError - _marker = [] - def get(self, nodeid, propname, default=_marker, cache=1): """Get the value of a property on an existing node of this class. @@ -2098,17 +2102,59 @@ class FileClass: - """ A class that requires the "content" property and stores it on - disk. + """ This class defines a large chunk of data. To support this, it + has a mandatory String property "content" which is saved off + externally to the hyperdb. + + The default MIME type of this data is defined by the + "default_mime_type" class attribute, which may be overridden by + each node if the class defines a "type" String property. """ default_mime_type = 'text/plain' - def __init__(self, db, classname, **properties): + def _update_properties(self, properties): """The newly-created class automatically includes the "content" - property. + and "type" properties. This method must be called by __init__. """ if 'content' not in properties: properties['content'] = String(indexme='yes') + if 'type' not in properties: + properties['type'] = String() + + def create(self, **propvalues): + """ snaffle the file propvalue and store in a file + """ + # we need to fire the auditors now, or the content property won't + # be in propvalues for the auditors to play with + self.fireAuditors('create', None, propvalues) + + # now remove the content property so it's not stored in the db + content = propvalues['content'] + del propvalues['content'] + + # do the database create + newid = self.create_inner(**propvalues) + + # figure the mime type + mime_type = propvalues.get('type', self.default_mime_type) + + # optionally index + # This wasn't done for the anydbm backend (but the 'set' method + # *did* update the index) so this is probably a bug-fix for anydbm + if self.properties['content'].indexme: + index_content = content + if bytes != str and isinstance(content, bytes): + index_content = content.decode('utf-8', errors='ignore') + self.db.indexer.add_text((self.classname, newid, 'content'), + index_content, mime_type) + + # store off the content as a file + self.db.storefile(self.classname, newid, None, bs2b(content)) + + # fire reactors + self.fireReactors('create', newid, None) + + return newid def export_propnames(self): """ Don't export the "content" property @@ -2136,6 +2182,40 @@ ensureParentsExist(dest) shutil.copyfile(source, dest) + def get(self, nodeid, propname, default=_marker, cache=1): + """ Trap the content propname and get it from the file + + 'cache' exists for backwards compatibility, and is not used. + """ + poss_msg = 'Possibly an access right configuration problem.' + if propname == 'content': + try: + return b2s(self.db.getfile(self.classname, nodeid, None)) + except IOError as strerror: + # BUG: by catching this we don't see an error in the log. + return 'ERROR reading file: %s%s\n%s\n%s' % ( + self.classname, nodeid, poss_msg, strerror) + except UnicodeDecodeError: + # if content is not text (e.g. jpeg file) we get + # unicode error trying to convert to string in python 3. + # trap it and supply an error message. Include md5sum + # of content as this string is included in the etag + # calculation of the object. + return ('%s%s is not text, retrieve using ' + 'binary_content property. mdsum: %s') % ( + self.classname, nodeid, + md5(self.db.getfile( + self.classname, + nodeid, + None)).hexdigest()) # nosec - bandit md5 use ok + elif propname == 'binary_content': + return self.db.getfile(self.classname, nodeid, None) + + if default is not _marker: + return self.subclass.get(self, nodeid, propname, default) + else: + return self.subclass.get(self, nodeid, propname) + def import_files(self, dirname, nodeid): """ Import the "content" property as a file """ @@ -2161,6 +2241,72 @@ self.db.indexer.add_text((self.classname, nodeid, 'content'), index_content, mime_type) + def index(self, nodeid): + """ Add (or refresh) the node to search indexes. + + Use the content-type property for the content property. + """ + # find all the String properties that have indexme + for prop, propclass in self.getprops().items(): + if prop == 'content' and propclass.indexme: + mime_type = self.get(nodeid, 'type', self.default_mime_type) + index_content = self.get(nodeid, 'binary_content') + if bytes != str and isinstance(index_content, bytes): + index_content = index_content.decode('utf-8', + errors='ignore') + self.db.indexer.add_text((self.classname, nodeid, 'content'), + index_content, mime_type) + elif isinstance(propclass, String) and propclass.indexme: + # index them under (classname, nodeid, property) + try: + value = str(self.get(nodeid, prop)) + except IndexError: + # node has been destroyed + continue + self.db.indexer.add_text((self.classname, nodeid, prop), value) + + def set(self, itemid, **propvalues): + """ Snarf the "content" propvalue and update it in a file + """ + self.fireAuditors('set', itemid, propvalues) + + # create the oldvalues dict - fill in any missing values + oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) + # The following is redundant for rdbms backends but needed for anydbm + # the performance impact is so low we that we don't factor this. + for name, prop in self.getprops(protected=0).items(): + if name in oldvalues: + continue + if isinstance(prop, Multilink): + oldvalues[name] = [] + else: + oldvalues[name] = None + + # now remove the content property so it's not stored in the db + content = None + if 'content' in propvalues: + content = propvalues['content'] + del propvalues['content'] + + # do the database update + propvalues = self.set_inner(itemid, **propvalues) + + # do content? + if content: + # store and possibly index + self.db.storefile(self.classname, itemid, None, bs2b(content)) + if self.properties['content'].indexme: + index_content = content + if bytes != str and isinstance(content, bytes): + index_content = content.decode('utf-8', errors='ignore') + mime_type = self.get(itemid, 'type', self.default_mime_type) + self.db.indexer.add_text((self.classname, itemid, 'content'), + index_content, mime_type) + propvalues['content'] = content + + # fire reactors + self.fireReactors('set', itemid, oldvalues) + return propvalues class Node: """ A convenience wrapper for the given node
--- a/roundup/roundupdb.py Thu Feb 27 10:05:38 2025 +0100 +++ b/roundup/roundupdb.py Sat Mar 01 13:08:09 2025 +0100 @@ -197,7 +197,7 @@ pass -# deviation from spec - was called IssueClass +# deviation from spec - was called ItemClass class IssueClass: """This class is intended to be mixed-in with a hyperdb backend implementation. The backend should provide a mechanism that @@ -228,6 +228,27 @@ ''"actor", ''"activity", ''"creator", ''"creation", ) + def _update_properties(self, classname, properties): + """The newly-created class automatically includes the "messages", + "files", "nosy", and "superseder" properties. If the 'properties' + dictionary attempts to specify any of these properties or a + "creation", "creator", "activity" or "actor" property, a ValueError + is raised. This method must be called by __init__. + + """ + if 'title' not in properties: + properties['title'] = hyperdb.String(indexme='yes') + if 'messages' not in properties: + properties['messages'] = hyperdb.Multilink("msg") + if 'files' not in properties: + properties['files'] = hyperdb.Multilink("file") + if 'nosy' not in properties: + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") + if 'superseder' not in properties: + properties['superseder'] = hyperdb.Multilink(classname) + # New methods: def addmessage(self, issueid, summary, text): """Add a message to an issue's mail spool.