Mercurial > p > roundup > code
comparison roundup/admin.py @ 7668:5b41018617f2
fix: out of memory error when importing under postgresql
If you try importing more than 20k items under postgresql you can run
out of memory:
psycopg2.errors.OutOfMemory: out of shared memory
HINT: You might need to increase max_locks_per_transaction.
Tuning memory may help, it's unknown at this point.
This checkin forces a commit to the postgres database after 10,000
rows have been added. This clears out the savepoints for each row and
starts a new transaction.
back_postgresql.py:
Implement commit mechanism in checkpoint_data(). Add two class level
attributes for tracking the number of savepoints and the limit when
the commit should happen.
roundup_admin.py:
implement pragma and dynamically create the config item
RDBMS_SAVEPOINT_LIMIT used by checkpoint_data.
Also fixed formatting of descriptions when using pragma list in
verbose mode.
admin_guide.txt, upgrading.txt:
Document change and use of pragma savepoint_limit in roundup-admin
for changing the default of 10,000.
test/db_test_base.py:
add some more asserts. In existing testAdminImportExport, set the
savepoint limit to 5 to test setting method and so that the commit
code will be run by existing tests. This provides coverage, but
does not actually test that the commit is done every 5 savepoints
8-(. The verification of every 5 savepoints was done manually
using a pdb breakpoint just before the commit.
acknowledgements.txt:
Added 2.4.0 section mentioning Norbert as he has done a ton of
testing with much larger datasets than I can test with.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Thu, 19 Oct 2023 16:11:25 -0400 |
| parents | 4de48eadf5f4 |
| children | 4dda4a9dfe0b |
comparison
equal
deleted
inserted
replaced
| 7667:08e4399c3ae4 | 7668:5b41018617f2 |
|---|---|
| 33 import sys | 33 import sys |
| 34 | 34 |
| 35 from roundup import date, hyperdb, init, password, token_r | 35 from roundup import date, hyperdb, init, password, token_r |
| 36 from roundup import __version__ as roundup_version | 36 from roundup import __version__ as roundup_version |
| 37 import roundup.instance | 37 import roundup.instance |
| 38 from roundup.configuration import (CoreConfig, NoConfigError, OptionUnsetError, | 38 from roundup.configuration import (CoreConfig, NoConfigError, Option, |
| 39 OptionValueError, ParsingOptionError, UserConfig) | 39 OptionUnsetError, OptionValueError, |
| 40 ParsingOptionError, UserConfig) | |
| 40 from roundup.i18n import _, get_translation | 41 from roundup.i18n import _, get_translation |
| 41 from roundup.exceptions import UsageError | 42 from roundup.exceptions import UsageError |
| 42 from roundup.anypy.my_input import my_input | 43 from roundup.anypy.my_input import my_input |
| 43 from roundup.anypy.strings import repr_export | 44 from roundup.anypy.strings import repr_export |
| 44 | 45 |
| 106 self.settings = { | 107 self.settings = { |
| 107 'display_header': False, | 108 'display_header': False, |
| 108 'display_protected': False, | 109 'display_protected': False, |
| 109 'indexer_backend': "as set in config.ini", | 110 'indexer_backend': "as set in config.ini", |
| 110 '_reopen_tracker': False, | 111 '_reopen_tracker': False, |
| 112 'savepoint_limit': 10000, | |
| 111 'show_retired': "no", | 113 'show_retired': "no", |
| 112 '_retired_val': False, | 114 '_retired_val': False, |
| 113 'verbose': False, | 115 'verbose': False, |
| 114 '_inttest': 3, | 116 '_inttest': 3, |
| 115 '_floattest': 3.5, | 117 '_floattest': 3.5, |
| 116 } | 118 } |
| 117 self.settings_help = { | 119 self.settings_help = { |
| 118 'display_header': | 120 'display_header': |
| 119 _("Have 'display designator[,designator*]' show header inside " | 121 _("Have 'display designator[,designator*]' show header inside\n" |
| 120 " []'s before items. Includes retired/active status."), | 122 " []'s before items. Includes retired/active status.\n"), |
| 121 | 123 |
| 122 'display_protected': | 124 'display_protected': |
| 123 _("Have 'display designator' and 'specification class' show " | 125 _("Have 'display designator' and 'specification class' show\n" |
| 124 "protected fields: creator, id etc."), | 126 " protected fields: creator, id etc.\n"), |
| 125 | 127 |
| 126 'indexer_backend': | 128 'indexer_backend': |
| 127 _("Set indexer to use when running 'reindex' NYI"), | 129 _("Set indexer to use when running 'reindex' NYI\n"), |
| 128 | 130 |
| 129 '_reopen_tracker': | 131 '_reopen_tracker': |
| 130 _("Force reopening of tracker when running each command."), | 132 _("Force reopening of tracker when running each command.\n"), |
| 131 | 133 |
| 132 'show_retired': _("Show retired items in table, list etc. One of 'no', 'only', 'both'"), | 134 'savepoint_limit': |
| 133 '_retired_val': _("internal mapping for show_retired."), | 135 _("set the number of rows imported before a database commit is\n" |
| 134 'verbose': _("Enable verbose output: tracing, descriptions..."), | 136 " done. Used only for imports on PostgreSQL.\n"), |
| 135 | 137 'show_retired': _("Show retired items in table, list etc. " |
| 136 '_inttest': "Integer valued setting. For testing only.", | 138 "One of 'no', 'only', 'both'\n"), |
| 137 '_floattest': "Float valued setting. For testing only.", | 139 '_retired_val': _("internal mapping for show_retired.\n"), |
| 140 'verbose': _("Enable verbose output: tracing, descriptions...\n"), | |
| 141 | |
| 142 '_inttest': "Integer valued setting. For testing only.\n", | |
| 143 '_floattest': "Float valued setting. For testing only.\n", | |
| 138 } | 144 } |
| 139 | 145 |
| 140 def get_class(self, classname): | 146 def get_class(self, classname): |
| 141 """Get the class - raise an exception if it doesn't exist. | 147 """Get the class - raise an exception if it doesn't exist. |
| 142 """ | 148 """ |
| 1047 raise UsageError(_('Not enough arguments supplied')) | 1053 raise UsageError(_('Not enough arguments supplied')) |
| 1048 | 1054 |
| 1049 if hasattr(csv, 'field_size_limit'): | 1055 if hasattr(csv, 'field_size_limit'): |
| 1050 csv.field_size_limit(self.db.config.CSV_FIELD_SIZE) | 1056 csv.field_size_limit(self.db.config.CSV_FIELD_SIZE) |
| 1051 | 1057 |
| 1058 # default value is 10000, only go through this if default | |
| 1059 # is different. | |
| 1060 if self.settings['savepoint_limit'] != 10000: | |
| 1061 self.db.config.add_option(Option(self.db.config, | |
| 1062 "rdbms", "savepoint_limit")) | |
| 1063 self.db.config.options["RDBMS_SAVEPOINT_LIMIT"].set( | |
| 1064 self.settings['savepoint_limit']) | |
| 1065 | |
| 1052 # directory to import from | 1066 # directory to import from |
| 1053 dir = args[0] | 1067 dir = args[0] |
| 1054 | 1068 |
| 1055 class colon_separated(csv.excel): | 1069 class colon_separated(csv.excel): |
| 1056 delimiter = ':' | 1070 delimiter = ':' |
