diff roundup/backends/rdbms_common.py @ 6332:6a6b4651be1f

Use server-side cursor for postgres in some cases In filter, filter_iter, and _materialize_multilinks, use named cursor with postgresql. This turns of client-side cursor handling and avoids *large* roundup process (or wsgi process) in case of large results. Fixes issue2551114.
author Ralf Schlatterbeck <rsc@runtux.com>
date Thu, 04 Mar 2021 12:55:21 +0100
parents 6834bb5473da
children 99455aeec1ae
line wrap: on
line diff
--- a/roundup/backends/rdbms_common.py	Tue Mar 02 12:43:19 2021 -0500
+++ b/roundup/backends/rdbms_common.py	Thu Mar 04 12:55:21 2021 +0100
@@ -1146,14 +1146,14 @@
                     joi = ', %s' % tn2
                     w = ' and %s.%s=%s.id and %s.__retired__=0'%(tn, lid,
                         tn2, tn2)
+            cursor = self.sql_new_cursor(name='_materialize_multilink')
             sql = 'select %s from %s%s where %s=%s%s' %(lid, tn, joi, nid,
                 self.arg, w)
-            self.sql(sql, (nodeid,))
-            # extract the first column from the result
-            # XXX numeric ids
-            items = [int(x[0]) for x in self.cursor.fetchall()]
-            items.sort()
-            node[propname] = [str(x) for x in items]
+            self.sql(sql, (nodeid,), cursor)
+            # Reduce this to only the first row (the ID), this can save a
+            # lot of space for large query results (not using fetchall)
+            node[propname] = [str(x) for x in sorted(int(r[0]) for r in cursor)]
+            cursor.close()
 
     def _materialize_multilinks(self, classname, nodeid, node, props=None):
         """ get all Multilinks of a node (lazy eval may have skipped this)
@@ -1461,6 +1461,14 @@
         # open a new cursor for subsequent work
         self.cursor = self.conn.cursor()
 
+    def sql_new_cursor(self, conn=None, *args, **kw):
+        """ Create new cursor, this may need additional parameters for
+            performance optimization for different backends.
+        """
+        if conn is None:
+            conn = self.conn
+        return conn.cursor()
+
     def commit(self):
         """ Commit the current transactions.
 
@@ -2854,18 +2862,26 @@
             return []
         proptree, sql, args = sq
 
-        self.db.sql(sql, args)
-        l = self.db.sql_fetchall()
-
+        cursor = self.db.sql_new_cursor(name='filter')
+        self.db.sql(sql, args, cursor)
+        # Reduce this to only the first row (the ID), this can save a
+        # lot of space for large query results (not using fetchall)
+        # We cannot do this if sorting by multilink
+        if proptree.tree_sort_done:
+            l = [str(row[0]) for row in cursor]
+        else:
+            l = cursor.fetchall()
+        cursor.close()
+
+        # Multilink sorting
         # Compute values needed for sorting in proptree.sort
-        for p in proptree:
-            if hasattr(p, 'auxcol'):
-                p.sort_ids = [row[p.auxcol] for row in l]
-                p.sort_result = p._sort_repr (p.propclass.sort_repr, p.sort_ids)
-        # return the IDs (the first column)
-        # XXX numeric ids
-        l = [str(row[0]) for row in l]
-        l = proptree.sort (l)
+        if not proptree.tree_sort_done:
+            for p in proptree:
+                if hasattr(p, 'auxcol'):
+                    p.sort_ids = [row[p.auxcol] for row in l]
+                    p.sort_result = p._sort_repr \
+                        (p.propclass.sort_repr, p.sort_ids)
+            l = proptree.sort ([str(row[0]) for row in l])
 
         if __debug__:
             self.db.stats['filtering'] += (time.time() - start_t)
@@ -2890,7 +2906,7 @@
         if sq is None:
             return
         proptree, sql, args = sq
-        cursor = self.db.conn.cursor()
+        cursor = self.db.sql_new_cursor(name='filter_iter')
         self.db.sql(sql, args, cursor)
         classes = {}
         for p in proptree:
@@ -2922,6 +2938,7 @@
                     node[propname] = value
                 self.db._cache_save(key, node)
             yield str(row[0])
+        cursor.close()
 
     def filter_sql(self, sql):
         """Return a list of the ids of the items in this class that match

Roundup Issue Tracker: http://roundup-tracker.org/