Mercurial > p > roundup > code
diff roundup/backends/rdbms_common.py @ 6332:6a6b4651be1f
Use server-side cursor for postgres in some cases
In filter, filter_iter, and _materialize_multilinks, use named cursor
with postgresql. This turns of client-side cursor handling and avoids
*large* roundup process (or wsgi process) in case of large results.
Fixes issue2551114.
| author | Ralf Schlatterbeck <rsc@runtux.com> |
|---|---|
| date | Thu, 04 Mar 2021 12:55:21 +0100 |
| parents | 6834bb5473da |
| children | 99455aeec1ae |
line wrap: on
line diff
--- a/roundup/backends/rdbms_common.py Tue Mar 02 12:43:19 2021 -0500 +++ b/roundup/backends/rdbms_common.py Thu Mar 04 12:55:21 2021 +0100 @@ -1146,14 +1146,14 @@ joi = ', %s' % tn2 w = ' and %s.%s=%s.id and %s.__retired__=0'%(tn, lid, tn2, tn2) + cursor = self.sql_new_cursor(name='_materialize_multilink') sql = 'select %s from %s%s where %s=%s%s' %(lid, tn, joi, nid, self.arg, w) - self.sql(sql, (nodeid,)) - # extract the first column from the result - # XXX numeric ids - items = [int(x[0]) for x in self.cursor.fetchall()] - items.sort() - node[propname] = [str(x) for x in items] + self.sql(sql, (nodeid,), cursor) + # Reduce this to only the first row (the ID), this can save a + # lot of space for large query results (not using fetchall) + node[propname] = [str(x) for x in sorted(int(r[0]) for r in cursor)] + cursor.close() def _materialize_multilinks(self, classname, nodeid, node, props=None): """ get all Multilinks of a node (lazy eval may have skipped this) @@ -1461,6 +1461,14 @@ # open a new cursor for subsequent work self.cursor = self.conn.cursor() + def sql_new_cursor(self, conn=None, *args, **kw): + """ Create new cursor, this may need additional parameters for + performance optimization for different backends. + """ + if conn is None: + conn = self.conn + return conn.cursor() + def commit(self): """ Commit the current transactions. @@ -2854,18 +2862,26 @@ return [] proptree, sql, args = sq - self.db.sql(sql, args) - l = self.db.sql_fetchall() - + cursor = self.db.sql_new_cursor(name='filter') + self.db.sql(sql, args, cursor) + # Reduce this to only the first row (the ID), this can save a + # lot of space for large query results (not using fetchall) + # We cannot do this if sorting by multilink + if proptree.tree_sort_done: + l = [str(row[0]) for row in cursor] + else: + l = cursor.fetchall() + cursor.close() + + # Multilink sorting # Compute values needed for sorting in proptree.sort - for p in proptree: - if hasattr(p, 'auxcol'): - p.sort_ids = [row[p.auxcol] for row in l] - p.sort_result = p._sort_repr (p.propclass.sort_repr, p.sort_ids) - # return the IDs (the first column) - # XXX numeric ids - l = [str(row[0]) for row in l] - l = proptree.sort (l) + if not proptree.tree_sort_done: + for p in proptree: + if hasattr(p, 'auxcol'): + p.sort_ids = [row[p.auxcol] for row in l] + p.sort_result = p._sort_repr \ + (p.propclass.sort_repr, p.sort_ids) + l = proptree.sort ([str(row[0]) for row in l]) if __debug__: self.db.stats['filtering'] += (time.time() - start_t) @@ -2890,7 +2906,7 @@ if sq is None: return proptree, sql, args = sq - cursor = self.db.conn.cursor() + cursor = self.db.sql_new_cursor(name='filter_iter') self.db.sql(sql, args, cursor) classes = {} for p in proptree: @@ -2922,6 +2938,7 @@ node[propname] = value self.db._cache_save(key, node) yield str(row[0]) + cursor.close() def filter_sql(self, sql): """Return a list of the ids of the items in this class that match
