Mercurial Repository: p/roundup/code: roundup/hyperdb.py comparison

comparison roundup/hyperdb.py @ 8304:24549122f9b1

Factor common code to hyperdb/roundupdb There was some common copied code in backends/back_anydbm.py and backends/rdbms_common.py. This is now moved to hyperdb.py and roundupdb.py, respectively (the FileClass lives in hyperdb.py while the IssueClass is in roundupdb.py)

author	Ralf Schlatterbeck <rsc@runtux.com>
date	Sat, 01 Mar 2025 13:08:09 +0100
parents	b99e76e76496
children	a81a3cd067fa

comparison

equal deleted inserted replaced

-:45ec660eb7f7
+:24549122f9b1
 """Hyperdatabase implementation, especially field types.
 """
 __docformat__ = 'restructuredtext'
 # standard python modules
+import copy
 import logging
 import os
 import re
 import shutil
 import sys
 import traceback
 import weakref
+from hashlib import md5
 # roundup modules
 from . import date, password
 from .support import ensureParentsExist, PrioList
 from roundup.mlink_expr import Expression
 from roundup.i18n import _
 from roundup.cgi.exceptions import DetectorError
 from roundup.anypy.cmp_ import NoneAndDictComparable
-from roundup.anypy.strings import eval_import
+from roundup.anypy.strings import b2s, bs2b, eval_import
 logger = logging.getLogger('roundup.hyperdb')
+# marker used for an unspecified keyword argument
+_marker = []
 #
 # Types
 #
 If an id in a link or multilink property does not refer to a valid
 node, an IndexError is raised.
 """
 raise NotImplementedError
-_marker = []
 def get(self, nodeid, propname, default=_marker, cache=1):
 """Get the value of a property on an existing node of this class.
 'nodeid' must be the id of an existing node of this class or an
 return value
 class FileClass:
-""" A class that requires the "content" property and stores it on
+""" This class defines a large chunk of data. To support this, it
-disk.
+has a mandatory String property "content" which is saved off
+externally to the hyperdb.
+The default MIME type of this data is defined by the
+"default_mime_type" class attribute, which may be overridden by
+each node if the class defines a "type" String property.
 """
 default_mime_type = 'text/plain'
-def __init__(self, db, classname, **properties):
+def _update_properties(self, properties):
 """The newly-created class automatically includes the "content"
-property.
+and "type" properties. This method must be called by __init__.
 """
 if 'content' not in properties:
 properties['content'] = String(indexme='yes')
+if 'type' not in properties:
+properties['type'] = String()
+def create(self, **propvalues):
+""" snaffle the file propvalue and store in a file
+"""
+# we need to fire the auditors now, or the content property won't
+# be in propvalues for the auditors to play with
+self.fireAuditors('create', None, propvalues)
+# now remove the content property so it's not stored in the db
+content = propvalues['content']
+del propvalues['content']
+# do the database create
+newid = self.create_inner(**propvalues)
+# figure the mime type
+mime_type = propvalues.get('type', self.default_mime_type)
+# optionally index
+# This wasn't done for the anydbm backend (but the 'set' method
+# *did* update the index) so this is probably a bug-fix for anydbm
+if self.properties['content'].indexme:
+index_content = content
+if bytes != str and isinstance(content, bytes):
+index_content = content.decode('utf-8', errors='ignore')
+self.db.indexer.add_text((self.classname, newid, 'content'),
+index_content, mime_type)
+# store off the content as a file
+self.db.storefile(self.classname, newid, None, bs2b(content))
+# fire reactors
+self.fireReactors('create', newid, None)
+return newid
 def export_propnames(self):
 """ Don't export the "content" property
 """
 propnames = list(self.getprops().keys())
 source = self.db.filename(self.classname, nodeid)
 dest = self.exportFilename(dirname, nodeid)
 ensureParentsExist(dest)
 shutil.copyfile(source, dest)
+def get(self, nodeid, propname, default=_marker, cache=1):
+""" Trap the content propname and get it from the file
+'cache' exists for backwards compatibility, and is not used.
+"""
+poss_msg = 'Possibly an access right configuration problem.'
+if propname == 'content':
+try:
+return b2s(self.db.getfile(self.classname, nodeid, None))
+except IOError as strerror:
+# BUG: by catching this we don't see an error in the log.
+return 'ERROR reading file: %s%s\n%s\n%s' % (
+self.classname, nodeid, poss_msg, strerror)
+except UnicodeDecodeError:
+# if content is not text (e.g. jpeg file) we get
+# unicode error trying to convert to string in python 3.
+# trap it and supply an error message. Include md5sum
+# of content as this string is included in the etag
+# calculation of the object.
+return ('%s%s is not text, retrieve using '
+'binary_content property. mdsum: %s') % (
+self.classname, nodeid,
+md5(self.db.getfile(
+self.classname,
+nodeid,
+None)).hexdigest())  # nosec - bandit md5 use ok
+elif propname == 'binary_content':
+return self.db.getfile(self.classname, nodeid, None)
+if default is not _marker:
+return self.subclass.get(self, nodeid, propname, default)
+else:
+return self.subclass.get(self, nodeid, propname)
 def import_files(self, dirname, nodeid):
 """ Import the "content" property as a file
 """
 source = self.exportFilename(dirname, nodeid)
 # other types. So if mime type of file is correct, we
 # call add_text on content.
 self.db.indexer.add_text((self.classname, nodeid, 'content'),
 index_content, mime_type)
+def index(self, nodeid):
+""" Add (or refresh) the node to search indexes.
+Use the content-type property for the content property.
+"""
+# find all the String properties that have indexme
+for prop, propclass in self.getprops().items():
+if prop == 'content' and propclass.indexme:
+mime_type = self.get(nodeid, 'type', self.default_mime_type)
+index_content = self.get(nodeid, 'binary_content')
+if bytes != str and isinstance(index_content, bytes):
+index_content = index_content.decode('utf-8',
+errors='ignore')
+self.db.indexer.add_text((self.classname, nodeid, 'content'),
+index_content, mime_type)
+elif isinstance(propclass, String) and propclass.indexme:
+# index them under (classname, nodeid, property)
+try:
+value = str(self.get(nodeid, prop))
+except IndexError:
+# node has been destroyed
+continue
+self.db.indexer.add_text((self.classname, nodeid, prop), value)
+def set(self, itemid, **propvalues):
+""" Snarf the "content" propvalue and update it in a file
+"""
+self.fireAuditors('set', itemid, propvalues)
+# create the oldvalues dict - fill in any missing values
+oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
+# The following is redundant for rdbms backends but needed for anydbm
+# the performance impact is so low we that we don't factor this.
+for name, prop in self.getprops(protected=0).items():
+if name in oldvalues:
+continue
+if isinstance(prop, Multilink):
+oldvalues[name] = []
+else:
+oldvalues[name] = None
+# now remove the content property so it's not stored in the db
+content = None
+if 'content' in propvalues:
+content = propvalues['content']
+del propvalues['content']
+# do the database update
+propvalues = self.set_inner(itemid, **propvalues)
+# do content?
+if content:
+# store and possibly index
+self.db.storefile(self.classname, itemid, None, bs2b(content))
+if self.properties['content'].indexme:
+index_content = content
+if bytes != str and isinstance(content, bytes):
+index_content = content.decode('utf-8', errors='ignore')
+mime_type = self.get(itemid, 'type', self.default_mime_type)
+self.db.indexer.add_text((self.classname, itemid, 'content'),
+index_content, mime_type)
+propvalues['content'] = content
+# fire reactors
+self.fireReactors('set', itemid, oldvalues)
+return propvalues
 class Node:
 """ A convenience wrapper for the given node
 """
 def __init__(self, cl, nodeid, cache=1):

Mercurial > p > roundup > code

comparison roundup/hyperdb.py @ 8304:24549122f9b1