comparison roundup/admin.py @ 7668:5b41018617f2

fix: out of memory error when importing under postgresql If you try importing more than 20k items under postgresql you can run out of memory: psycopg2.errors.OutOfMemory: out of shared memory HINT: You might need to increase max_locks_per_transaction. Tuning memory may help, it's unknown at this point. This checkin forces a commit to the postgres database after 10,000 rows have been added. This clears out the savepoints for each row and starts a new transaction. back_postgresql.py: Implement commit mechanism in checkpoint_data(). Add two class level attributes for tracking the number of savepoints and the limit when the commit should happen. roundup_admin.py: implement pragma and dynamically create the config item RDBMS_SAVEPOINT_LIMIT used by checkpoint_data. Also fixed formatting of descriptions when using pragma list in verbose mode. admin_guide.txt, upgrading.txt: Document change and use of pragma savepoint_limit in roundup-admin for changing the default of 10,000. test/db_test_base.py: add some more asserts. In existing testAdminImportExport, set the savepoint limit to 5 to test setting method and so that the commit code will be run by existing tests. This provides coverage, but does not actually test that the commit is done every 5 savepoints 8-(. The verification of every 5 savepoints was done manually using a pdb breakpoint just before the commit. acknowledgements.txt: Added 2.4.0 section mentioning Norbert as he has done a ton of testing with much larger datasets than I can test with.
author John Rouillard <rouilj@ieee.org>
date Thu, 19 Oct 2023 16:11:25 -0400
parents 4de48eadf5f4
children 4dda4a9dfe0b
comparison
equal deleted inserted replaced
7667:08e4399c3ae4 7668:5b41018617f2
33 import sys 33 import sys
34 34
35 from roundup import date, hyperdb, init, password, token_r 35 from roundup import date, hyperdb, init, password, token_r
36 from roundup import __version__ as roundup_version 36 from roundup import __version__ as roundup_version
37 import roundup.instance 37 import roundup.instance
38 from roundup.configuration import (CoreConfig, NoConfigError, OptionUnsetError, 38 from roundup.configuration import (CoreConfig, NoConfigError, Option,
39 OptionValueError, ParsingOptionError, UserConfig) 39 OptionUnsetError, OptionValueError,
40 ParsingOptionError, UserConfig)
40 from roundup.i18n import _, get_translation 41 from roundup.i18n import _, get_translation
41 from roundup.exceptions import UsageError 42 from roundup.exceptions import UsageError
42 from roundup.anypy.my_input import my_input 43 from roundup.anypy.my_input import my_input
43 from roundup.anypy.strings import repr_export 44 from roundup.anypy.strings import repr_export
44 45
106 self.settings = { 107 self.settings = {
107 'display_header': False, 108 'display_header': False,
108 'display_protected': False, 109 'display_protected': False,
109 'indexer_backend': "as set in config.ini", 110 'indexer_backend': "as set in config.ini",
110 '_reopen_tracker': False, 111 '_reopen_tracker': False,
112 'savepoint_limit': 10000,
111 'show_retired': "no", 113 'show_retired': "no",
112 '_retired_val': False, 114 '_retired_val': False,
113 'verbose': False, 115 'verbose': False,
114 '_inttest': 3, 116 '_inttest': 3,
115 '_floattest': 3.5, 117 '_floattest': 3.5,
116 } 118 }
117 self.settings_help = { 119 self.settings_help = {
118 'display_header': 120 'display_header':
119 _("Have 'display designator[,designator*]' show header inside " 121 _("Have 'display designator[,designator*]' show header inside\n"
120 " []'s before items. Includes retired/active status."), 122 " []'s before items. Includes retired/active status.\n"),
121 123
122 'display_protected': 124 'display_protected':
123 _("Have 'display designator' and 'specification class' show " 125 _("Have 'display designator' and 'specification class' show\n"
124 "protected fields: creator, id etc."), 126 " protected fields: creator, id etc.\n"),
125 127
126 'indexer_backend': 128 'indexer_backend':
127 _("Set indexer to use when running 'reindex' NYI"), 129 _("Set indexer to use when running 'reindex' NYI\n"),
128 130
129 '_reopen_tracker': 131 '_reopen_tracker':
130 _("Force reopening of tracker when running each command."), 132 _("Force reopening of tracker when running each command.\n"),
131 133
132 'show_retired': _("Show retired items in table, list etc. One of 'no', 'only', 'both'"), 134 'savepoint_limit':
133 '_retired_val': _("internal mapping for show_retired."), 135 _("set the number of rows imported before a database commit is\n"
134 'verbose': _("Enable verbose output: tracing, descriptions..."), 136 " done. Used only for imports on PostgreSQL.\n"),
135 137 'show_retired': _("Show retired items in table, list etc. "
136 '_inttest': "Integer valued setting. For testing only.", 138 "One of 'no', 'only', 'both'\n"),
137 '_floattest': "Float valued setting. For testing only.", 139 '_retired_val': _("internal mapping for show_retired.\n"),
140 'verbose': _("Enable verbose output: tracing, descriptions...\n"),
141
142 '_inttest': "Integer valued setting. For testing only.\n",
143 '_floattest': "Float valued setting. For testing only.\n",
138 } 144 }
139 145
140 def get_class(self, classname): 146 def get_class(self, classname):
141 """Get the class - raise an exception if it doesn't exist. 147 """Get the class - raise an exception if it doesn't exist.
142 """ 148 """
1047 raise UsageError(_('Not enough arguments supplied')) 1053 raise UsageError(_('Not enough arguments supplied'))
1048 1054
1049 if hasattr(csv, 'field_size_limit'): 1055 if hasattr(csv, 'field_size_limit'):
1050 csv.field_size_limit(self.db.config.CSV_FIELD_SIZE) 1056 csv.field_size_limit(self.db.config.CSV_FIELD_SIZE)
1051 1057
1058 # default value is 10000, only go through this if default
1059 # is different.
1060 if self.settings['savepoint_limit'] != 10000:
1061 self.db.config.add_option(Option(self.db.config,
1062 "rdbms", "savepoint_limit"))
1063 self.db.config.options["RDBMS_SAVEPOINT_LIMIT"].set(
1064 self.settings['savepoint_limit'])
1065
1052 # directory to import from 1066 # directory to import from
1053 dir = args[0] 1067 dir = args[0]
1054 1068
1055 class colon_separated(csv.excel): 1069 class colon_separated(csv.excel):
1056 delimiter = ':' 1070 delimiter = ':'

Roundup Issue Tracker: http://roundup-tracker.org/