Mercurial Repository: p/roundup/code: roundup/hyperdb.py comparison

comparison roundup/hyperdb.py @ 4472:34dce76bb202

Multilink fixes and optimizations: - Optimisation: Late evaluation of Multilinks (only in rdbms backends): previously we materialized each multilink in a Node -- this creates an SQL query for each multilink (e.g. 'files' and 'messages' for each line in the issue index display) -- even if the multilinks aren't displayed. Now we compute multilinks only if they're accessed (and keep them cached). - Add a filter_iter similar to the existing filter call. This feature is considered experimental. This is currently not used in the web-interface but passes all tests for the filter call except sorting by Multilinks (which isn't supported by SQL and isn't a sane concept anyway). When using filter_iter instead of filter this saves a *lot* of SQL queries: Filter returns only the IDs of Nodes in the database, the additional content of a Node has to be fetched in a separate SQL call. The new filter_iter also returns the IDs of Nodes (one by one, it's an iterator) but pre-seeds the cache with the content of the Node. The information needed for seeding the cache is retrieved in the same SQL query as the ids.

author	Ralf Schlatterbeck <schlatterbeck@users.sourceforge.net>
date	Mon, 21 Mar 2011 20:44:39 +0000
parents	9b619dcb030a
children	1613754d2646

comparison

equal deleted inserted replaced

-:4f353d71d716
+:34dce76bb202
 class Proptree(object):
 """ Simple tree data structure for optimizing searching of
 properties. Each node in the tree represents a roundup Class
 Property that has to be navigated for finding the given search
-or sort properties. The sort_type attribute is used for
+or sort properties. The need_for attribute is used for
-distinguishing nodes in the tree used for sorting or searching: If
+distinguishing nodes in the tree used for sorting, searching or
-it is 0 for a node, that node is not used for sorting. If it is 1,
+retrieval: The attribute is a dictionary containing one or several
-it is used for both, sorting and searching. If it is 2 it is used
+of the values 'sort', 'search', 'retrieve'.
-for sorting only.
 The Proptree is also used for transitively searching attributes for
 backends that do not support transitive search (e.g. anydbm). The
 _val attribute with set_val is used for this.
 """
-def __init__(self, db, cls, name, props, parent = None):
+def __init__(self, db, cls, name, props, parent=None, retr=False):
 self.db = db
 self.name = name
 self.props = props
 self.parent = parent
 self._val = None
 self.classname = None
 self.uniqname = None
 self.children = []
 self.sortattr = []
 self.propdict = {}
-self.sort_type = 0
+self.need_for = {'search' : True}
 self.sort_direction = None
 self.sort_ids = None
 self.sort_ids_needed = False
 self.sort_result = None
 self.attr_sort_done = False
 self.tree_sort_done = False
 self.propclass = None
 self.orderby = []
+self.sql_idx = None # index of retrieved column in sql result
 if parent:
 self.root = parent.root
 self.depth = parent.depth + 1
 else:
 self.root = self
 self.seqno = 1
 self.depth = 0
-self.sort_type = 1
+self.need_for['sort'] = True
 self.id = self.root.seqno
 self.root.seqno += 1
 if self.cls:
 self.classname = self.cls.classname
 self.uniqname = '%s%s' % (self.cls.classname, self.id)
 if not self.parent:
 self.uniqname = self.cls.classname
+if retr:
-def append(self, name, sort_type = 0):
+self.append_retr_props()
+def append(self, name, need_for='search', retr=False):
 """Append a property to self.children. Will create a new
 propclass for the child.
 """
 if name in self.propdict:
 pt = self.propdict[name]
-if sort_type and not pt.sort_type:
+pt.need_for[need_for] = True
-pt.sort_type = 1
+if retr and isinstance(pt.propclass, Link):
+pt.append_retr_props()
 return pt
 propclass = self.props[name]
 cls = None
 props = None
 if isinstance(propclass, (Link, Multilink)):
 cls = self.db.getclass(propclass.classname)
 props = cls.getprops()
 child = self.__class__(self.db, cls, name, props, parent = self)
-child.sort_type = sort_type
+child.need_for = {need_for : True}
 child.propclass = propclass
 self.children.append(child)
 self.propdict[name] = child
+if retr and isinstance(child.propclass, Link):
+child.append_retr_props()
 return child
+def append_retr_props(self):
+"""Append properties for retrieval."""
+for name, prop in self.cls.getprops(protected=1).iteritems():
+if isinstance(prop, Multilink):
+continue
+self.append(name, need_for='retrieve')
 def compute_sort_done(self, mlseen=False):
 """ Recursively check if attribute is needed for sorting
-(self.sort_type > 0) or all children have tree_sort_done set and
+('sort' in self.need_for) or all children have tree_sort_done set and
 sort_ids_needed unset: set self.tree_sort_done if one of the conditions
 holds. Also remove sort_ids_needed recursively once having seen a
 Multilink.
 """
 if isinstance (self.propclass, Multilink):
 self.tree_sort_done = True
 for p in self.children:
 p.compute_sort_done(mlseen)
 if not p.tree_sort_done:
 self.tree_sort_done = False
-if not self.sort_type:
+if 'sort' not in self.need_for:
 self.tree_sort_done = True
 if mlseen:
 self.tree_sort_done = False
 def ancestors(self):
 Once all properties are non-transitive, the search generates a
 simple _filter call which does the real work
 """
 filterspec = {}
 for p in self.children:
-if p.sort_type < 2:
+if 'search' in p.need_for:
 if p.children:
 p.search(sort = False)
 filterspec[p.name] = p.val
 self.val = self.cls._filter(search_matches, filterspec, sort and self)
 return self.val
 """ All children needed for sorting. If intermediate is True,
 intermediate nodes (not being a sort attribute) are returned,
 too.
 """
 return [p for p in self.children
-if p.sort_type > 0 and (intermediate or p.sort_direction)]
+if 'sort' in p.need_for and (intermediate or p.sort_direction)]
 def __iter__(self):
 """ Yield nodes in depth-first order -- visited nodes first """
 for p in self.children:
 yield p
 dir_idx.append (idx)
 directions.append (sa.sort_direction)
 curdir = sa.sort_direction
 idx += 1
 sortattr.append (val)
-#print >> sys.stderr, "\nsortattr", sortattr
 sortattr = zip (*sortattr)
 for dir, i in reversed(zip(directions, dir_idx)):
 rev = dir == '-'
 sortattr = sorted (sortattr, key = lambda x:x[i:idx], reverse = rev)
 idx = i
 """For some backends this implements the non-transitive
 search, for more information see the filter method.
 """
 raise NotImplementedError
-def _proptree(self, filterspec, sortattr=[]):
+def _proptree(self, filterspec, sortattr=[], retr=False):
 """Build a tree of all transitive properties in the given
 filterspec.
-"""
+If we retrieve (retr is True) linked items we don't follow
-proptree = Proptree(self.db, self, '', self.getprops())
+across multilinks. We also don't follow if the searched value
+can contain NULL values.
+"""
+proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
 for key, v in filterspec.iteritems():
 keys = key.split('.')
 p = proptree
+mlseen = False
 for k in keys:
-p = p.append(k)
+if isinstance (p.propclass, Multilink):
+mlseen = True
+isnull = v == '-1' or v is None
+nullin = isinstance(v, type([])) and ('-1' in v or None in v)
+r = retr and not mlseen and not isnull and not nullin
+p = p.append(k, retr=r)
 p.val = v
 multilinks = {}
 for s in sortattr:
 keys = s[1].split('.')
 p = proptree
+mlseen = False
 for k in keys:
-p = p.append(k, sort_type = 2)
+if isinstance (p.propclass, Multilink):
+mlseen = True
+r = retr and not mlseen
+p = p.append(k, need_for='sort', retr=r)
 if isinstance (p.propclass, Multilink):
 multilinks[p] = True
 if p.cls:
-p = p.append(p.cls.orderprop(), sort_type = 2)
+p = p.append(p.cls.orderprop(), need_for='sort')
 if p.sort_direction: # if an orderprop is also specified explicitly
 continue
 p.sort_direction = s[0]
 proptree.sortattr.append (p)
 for p in multilinks.iterkeys():
 Implementation note:
 This implements a non-optimized version of Transitive search
 using _filter implemented in a backend class. A more efficient
 version can be implemented in the individual backends -- e.g.,
-an SQL backen will want to create a single SQL statement and
+an SQL backend will want to create a single SQL statement and
 override the filter method instead of implementing _filter.
 """
 sortattr = self._sortattr(sort = sort, group = group)
 proptree = self._proptree(filterspec, sortattr)
 proptree.search(search_matches)
 return proptree.sort()
+# non-optimized filter_iter, a backend may chose to implement a
+# better version that provides a real iterator that pre-fills the
+# cache for each id returned. Note that the filter_iter doesn't
+# promise to correctly sort by multilink (which isn't sane to do
+# anyway).
+filter_iter = filter
 def count(self):
 """Get the number of nodes in this class.
 If the returned integer is 'numnodes', the ids of all the nodes

Mercurial > p > roundup > code

comparison roundup/hyperdb.py @ 4472:34dce76bb202