Mercurial > p > roundup > code
changeset 4429:d75e92fbfcca
- lower memory footprint for (journal-) import
.-- only for rdbms backends, other backends shouldn't have that much
data anyway.
| author | Ralf Schlatterbeck <schlatterbeck@users.sourceforge.net> |
|---|---|
| date | Thu, 07 Oct 2010 12:02:12 +0000 |
| parents | e04f4c060887 |
| children | f2f2904fe6ce |
| files | CHANGES.txt roundup/backends/rdbms_common.py |
| diffstat | 2 files changed, 26 insertions(+), 6 deletions(-) [+] |
line wrap: on
line diff
--- a/CHANGES.txt Thu Oct 07 02:31:58 2010 +0000 +++ b/CHANGES.txt Thu Oct 07 12:02:12 2010 +0000 @@ -44,6 +44,8 @@ Thanks to Benni Bärmann for reporting. - Allow search_popup macro to work with all db classes, issue2550567 (thanks John Kristensen) +- lower memory footprint for (journal-) import -- only for rdbms + backends, other backends shouldn't have that much data anyway. 2010-07-12 1.4.15
--- a/roundup/backends/rdbms_common.py Thu Oct 07 02:31:58 2010 +0000 +++ b/roundup/backends/rdbms_common.py Thu Oct 07 12:02:12 2010 +0000 @@ -2644,12 +2644,31 @@ def import_journals(self, entries): """Import a class's journal. - Uses setjournal() to set the journal for each item.""" + Uses setjournal() to set the journal for each item. + Strategy for import: Sort first by id, then import journals for + each id, this way the memory footprint is a lot smaller than the + initial implementation which stored everything in a big hash by + id and then proceeded to import journals for each id.""" properties = self.getprops() - d = {} + a = [] for l in entries: + # first element in sorted list is the (numeric) id + # in python2.4 and up we would use sorted with a key... + a.append ((int (l [0].strip ("'")), l)) + a.sort () + + + last = 0 + r = [] + for n, l in a: nodeid, jdate, user, action, params = map(eval, l) - r = d.setdefault(nodeid, []) + assert (str(n) == nodeid) + if n != last: + if r: + self.db.setjournal(self.classname, nodeid, r) + last = n + r = [] + if action == 'set': for propname, value in params.iteritems(): prop = properties[propname] @@ -2668,9 +2687,8 @@ # old tracker with data stored in the create! params = {} r.append((nodeid, date.Date(jdate), user, action, params)) - - for nodeid, l in d.iteritems(): - self.db.setjournal(self.classname, nodeid, l) + if r: + self.db.setjournal(self.classname, nodeid, r) class FileClass(hyperdb.FileClass, Class): """This class defines a large chunk of data. To support this, it has a
