Mercurial > p > roundup > code
comparison roundup/backends/rdbms_common.py @ 4429:d75e92fbfcca
- lower memory footprint for (journal-) import
.-- only for rdbms backends, other backends shouldn't have that much
data anyway.
| author | Ralf Schlatterbeck <schlatterbeck@users.sourceforge.net> |
|---|---|
| date | Thu, 07 Oct 2010 12:02:12 +0000 |
| parents | 67bef70ab9b9 |
| children | f2f2904fe6ce |
comparison
equal
deleted
inserted
replaced
| 4428:e04f4c060887 | 4429:d75e92fbfcca |
|---|---|
| 2642 return r | 2642 return r |
| 2643 | 2643 |
| 2644 def import_journals(self, entries): | 2644 def import_journals(self, entries): |
| 2645 """Import a class's journal. | 2645 """Import a class's journal. |
| 2646 | 2646 |
| 2647 Uses setjournal() to set the journal for each item.""" | 2647 Uses setjournal() to set the journal for each item. |
| 2648 Strategy for import: Sort first by id, then import journals for | |
| 2649 each id, this way the memory footprint is a lot smaller than the | |
| 2650 initial implementation which stored everything in a big hash by | |
| 2651 id and then proceeded to import journals for each id.""" | |
| 2648 properties = self.getprops() | 2652 properties = self.getprops() |
| 2649 d = {} | 2653 a = [] |
| 2650 for l in entries: | 2654 for l in entries: |
| 2655 # first element in sorted list is the (numeric) id | |
| 2656 # in python2.4 and up we would use sorted with a key... | |
| 2657 a.append ((int (l [0].strip ("'")), l)) | |
| 2658 a.sort () | |
| 2659 | |
| 2660 | |
| 2661 last = 0 | |
| 2662 r = [] | |
| 2663 for n, l in a: | |
| 2651 nodeid, jdate, user, action, params = map(eval, l) | 2664 nodeid, jdate, user, action, params = map(eval, l) |
| 2652 r = d.setdefault(nodeid, []) | 2665 assert (str(n) == nodeid) |
| 2666 if n != last: | |
| 2667 if r: | |
| 2668 self.db.setjournal(self.classname, nodeid, r) | |
| 2669 last = n | |
| 2670 r = [] | |
| 2671 | |
| 2653 if action == 'set': | 2672 if action == 'set': |
| 2654 for propname, value in params.iteritems(): | 2673 for propname, value in params.iteritems(): |
| 2655 prop = properties[propname] | 2674 prop = properties[propname] |
| 2656 if value is None: | 2675 if value is None: |
| 2657 pass | 2676 pass |
| 2666 params[propname] = value | 2685 params[propname] = value |
| 2667 elif action == 'create' and params: | 2686 elif action == 'create' and params: |
| 2668 # old tracker with data stored in the create! | 2687 # old tracker with data stored in the create! |
| 2669 params = {} | 2688 params = {} |
| 2670 r.append((nodeid, date.Date(jdate), user, action, params)) | 2689 r.append((nodeid, date.Date(jdate), user, action, params)) |
| 2671 | 2690 if r: |
| 2672 for nodeid, l in d.iteritems(): | 2691 self.db.setjournal(self.classname, nodeid, r) |
| 2673 self.db.setjournal(self.classname, nodeid, l) | |
| 2674 | 2692 |
| 2675 class FileClass(hyperdb.FileClass, Class): | 2693 class FileClass(hyperdb.FileClass, Class): |
| 2676 """This class defines a large chunk of data. To support this, it has a | 2694 """This class defines a large chunk of data. To support this, it has a |
| 2677 mandatory String property "content" which is typically saved off | 2695 mandatory String property "content" which is typically saved off |
| 2678 externally to the hyperdb. | 2696 externally to the hyperdb. |