Mercurial Repository: p/roundup/code: roundup/hyperdb.py comparison

comparison roundup/hyperdb.py @ 6500:30358e334232 issue2550923_computed_property

merge trunk changes into this branch

author	John Rouillard <rouilj@ieee.org>
date	Thu, 16 Sep 2021 14:30:56 -0400
parents	1e5ed659e8ca 8e06194ff0b0
children	e1588ae185dc

comparison

equal deleted inserted replaced

-:1a15089c2e49
+:30358e334232
 import logging
 # roundup modules
 from . import date, password
 from .support import ensureParentsExist, PrioList
+from roundup.mlink_expr import Expression
 from roundup.i18n import _
 from roundup.cgi.exceptions import DetectorError
 from roundup.anypy.cmp_ import NoneAndDictComparable
 from roundup.anypy.strings import eval_import
 return len(db.issue.get(nodeid, 'messages'))
 class String(_Type):
 """An object designating a String property."""
-def __init__(self, indexme='no', required=False, default_value="",
+def __init__(self, indexme='no', required=False, default_value=None,
 quiet=False):
 super(String, self).__init__(required, default_value, quiet)
 self.indexme = indexme == 'yes'
 def from_raw(self, value, propname='', **kw):
 is a dictionary containing one or several of the values 'sort',
 'search', 'retrieve'.
 The Proptree is also used for transitively searching attributes for
 backends that do not support transitive search (e.g. anydbm). The
-_val attribute with set_val is used for this.
+val attribute with set_val is used for this.
 """
 def __init__(self, db, cls, name, props, parent=None, retr=False):
 self.db = db
 self.name = name
 self.props = props
 self.parent = parent
-self._val = None
+self.val = None
 self.has_values = False
+self.has_result = False
 self.cls = cls
 self.classname = None
 self.uniqname = None
 self.children = []
 self.sortattr = []
 propclass for the child.
 """
 if name in self.propdict:
 pt = self.propdict[name]
 pt.need_for[need_for] = True
-if retr and isinstance(pt.propclass, Link):
+# For now we do not recursively retrieve Link properties
-pt.append_retr_props()
+#if retr and isinstance(pt.propclass, Link):
+#    pt.append_retr_props()
 return pt
 propclass = self.props[name]
 cls = None
 props = None
 if isinstance(propclass, (Link, Multilink)):
 child.need_retired = True
 else:
 child.need_child_retired = True
 self.children.append(child)
 self.propdict[name] = child
-if retr and isinstance(child.propclass, Link):
+# For now we do not recursively retrieve Link properties
-child.append_retr_props()
+#if retr and isinstance(child.propclass, Link):
+#    child.append_retr_props()
 return child
 def append_retr_props(self):
 """Append properties for retrieval."""
 for name, prop in self.cls.getprops(protected=1).items():
 """
 filterspec = {}
 exact_match_spec = {}
 for p in self.children:
 if 'search' in p.need_for:
-if p.children:
+x = [c for c in p.children if 'search' in c.need_for]
+if x:
 p.search(sort=False)
 if getattr(p.propclass,'rev_property',None):
 pn = p.propclass.rev_property.name
 cl = p.propclass.rev_property.cls
 if not isinstance(p.val, type([])):
 p.val = [p.val]
-if p.val == ['-1'] :
+nval = [int(i) for i in p.val]
-s1 = set(self.cls.getnodeids(retired=False))
+pval = [str(i) for i in nval if i >= 0]
-s2 = set()
+items = set()
+if not nval or min(nval) >= -1:
+if -1 in nval:
+s1 = set(self.cls.getnodeids(retired=False))
+s2 = set()
+for id in cl.getnodeids(retired=False):
+node = cl.getnode(id)
+if node[pn]:
+if isinstance(node[pn], type([])):
+s2.update(node[pn])
+else:
+s2.add(node[pn])
+items |= s1.difference(s2)
+if isinstance(p.propclass.rev_property, Link):
+items |= set(cl.get(x, pn) for x in pval
+if not cl.is_retired(x))
+else:
+items |= set().union(*(cl.get(x, pn) for x in pval
+if not cl.is_retired(x)))
+else:
+# Expression: materialize rev multilinks and run
+# expression on them
+expr = Expression(nval)
+by_id = {}
+for id in self.cls.getnodeids(retired=False):
+by_id[id] = set()
+items = set()
 for id in cl.getnodeids(retired=False):
 node = cl.getnode(id)
 if node[pn]:
-if isinstance(node [pn], type([])):
+v = node[pn]
-s2.update(node [pn])
+if not isinstance(v, type([])):
-else:
+v = [v]
-s2.add(node [pn])
+for x in v:
-items = s1.difference(s2)
+if x not in by_id:
-elif isinstance(p.propclass.rev_property, Link):
+continue
-items = set(cl.get(x, pn) for x in p.val
+by_id[x].add(id)
-if not cl.is_retired(x))
+for k in by_id:
-else:
+if expr.evaluate(by_id[k]):
-items = set().union(*(cl.get(x, pn) for x in p.val
+items.add(k)
-if not cl.is_retired(x)))
-filterspec[p.name] = list(sorted(items))
+# The subquery has found nothing. So it doesn't make
+# sense to search further.
+if not items:
+self.set_val([], force=True)
+return self.val
+filterspec[p.name] = list(sorted(items, key=int))
 elif isinstance(p.val, type([])):
 exact = []
 subst = []
 for v in p.val:
 if isinstance(v, Exact_Match):
 if exact:
 exact_match_spec[p.name] = exact
 if subst:
 filterspec[p.name] = subst
 elif not exact: # don't set if we have exact criteria
-filterspec[p.name] =[ '-1' ] # no match was found
+if p.has_result:
+# A subquery already has found nothing. So
+# it doesn't make sense to search further.
+self.set_val([], force=True)
+return self.val
+else:
+filterspec[p.name] = ['-1'] # no match was found
 else:
 assert not isinstance(p.val, Exact_Match)
 filterspec[p.name] = p.val
-self.val = self.cls._filter(search_matches, filterspec, sort and self,
+self.set_val(self.cls._filter(search_matches, filterspec, sort and self,
 retired=retired,
-exact_match_spec=exact_match_spec)
+exact_match_spec=exact_match_spec))
 return self.val
 def sort(self, ids=None):
 """ Sort ids by the order information stored in self. With
 optimisations: Some order attributes may be precomputed (by the
 pt.sort_ids = None
 for pt in self.sortattr:
 pt.sort_result = None
 return ids
-def _set_val(self, val):
+def set_val(self, val, force=False, result=True):
-""" Check if self._val is already defined. If yes, we compute the
+""" Check if self.val is already defined (it is not None and
+has_values is True). If yes, we compute the
 intersection of the old and the new value(s)
 Note: If self is a Leaf node we need to compute a
 union: Normally we intersect (logical and) different
 subqueries into a Link or Multilink property. But for
 leaves we might have a part of a query in a filterspec and
 in an exact_match_spec. These have to be all there, the
 generated search will ensure a logical and of all tests for
 equality/substring search.
 """
+if force:
+assert val == []
+assert result
+self.val = val
+self.has_values = True
+self.has_result = True
+return
 if self.has_values:
-v = self._val
+v = self.val
-if not isinstance(self._val, type([])):
+if not isinstance(self.val, type([])):
-v = [self._val]
+v = [self.val]
 vals = set(v)
 if not isinstance(val, type([])):
 val = [val]
+if self.has_result:
+assert result
 # if cls is None we're a leaf
 if self.cls:
 vals.intersection_update(val)
 else:
 vals.update(val)
-self._val = [v for v in vals]
+self.val = list(vals)
 else:
-self._val = val
+# If a subquery found nothing we don't care if there is an
+# expression
+if not self.has_values or not val:
+self.val = val
+if result:
+self.has_result = True
+else:
+if not result:
+assert not self.cls
+vals.update(val)
+self.val = list(vals)
+else:
+assert self.cls
+is_expression = \
+self.val and min(int(i) for i in self.val) < -1
+if is_expression:
+# Tag on the ORed values with an AND
+l = val
+for i in range(len(val)-1):
+l.append('-4')
+l.append('-3')
+self.val = self.val + l
+else:
+vals.intersection_update(val)
+self.val = list(vals)
+self.has_result = True
 self.has_values = True
-val = property(lambda self: self._val, _set_val)
 def _sort(self, val):
 """Finally sort by the given sortattr.sort_result. Note that we
 do not sort by attrs having attr_sort_done set. The caller is
 responsible for setting attr_sort_done only for trailing
 def _proptree(self, filterspec, exact_match_spec={}, sortattr=[],
 retr=False):
 """Build a tree of all transitive properties in the given
 exact_match_spec/filterspec.
 If we retrieve (retr is True) linked items we don't follow
-across multilinks. We also don't follow if the searched value
+across multilinks or links.
-can contain NULL values.
 """
 proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
 for exact, spec in enumerate((filterspec, exact_match_spec)):
 for key, v in spec.items():
 keys = key.split('.')
 if exact:
 if isinstance(v, type([])):
 vv = []
 for x in v:
 vv.append(Exact_Match(x))
-p.val = vv
+p.set_val(vv, result=False)
 else:
-p.val = [Exact_Match(v)]
+p.set_val([Exact_Match(v)], result=False)
 else:
-p.val = v
+p.set_val(v, result=False)
 multilinks = {}
 for s in sortattr:
 keys = s[1].split('.')
 p = proptree
 mlseen = False
 This also means that for strings in exact_match_spec it doesn't
 make sense to specify multiple values because those cannot all
 be matched exactly.
+For Link and Multilink properties the special ID value '-1'
+matches empty Link or Multilink fields. For Multilinks a postfix
+expression syntax using negative ID numbers (as strings) as
+operators is supported. Each non-negative number (or '-1') is
+pushed on an operand stack. A negative number pops the required
+number of arguments from the stack, applies the operator, and
+pushes the result. The following operators are supported:
+- -2 stands for 'NOT' and takes one argument
+- -3 stands for 'AND' and takes two arguments
+- -4 stands for 'OR' and takes two arguments
+Note that this special handling of ID arguments is applied only
+when a negative number smaller than -1 is encountered as an ID
+in the filter call. Otherwise the implicit OR default applies.
+Examples of using Multilink expressions would be
+- '1', '2', '-4', '3', '4', '-4', '-3'
+would search for IDs (1 or 2) and (3 or 4)
+- '-1' '-2' would search for all non-empty Multilinks
 The propname in filterspec and prop in a sort/group spec may be
 transitive, i.e., it may contain properties of the form
 link.link.link.name, e.g. you can search for all issues where a
 message was added by a certain user in the last week with a
 filterspec of
 cl = Class(db, name, name=String(), order=String())
 for i in range(len(options)):
 cl.create(name=options[i], order=i)
 return Link(name)
-# vim: set filetype=python sts=4 sw=4 et si :

Mercurial > p > roundup > code

comparison roundup/hyperdb.py @ 6500:30358e334232 issue2550923_computed_property