view test/test_anydbm.py @ 7668:5b41018617f2

fix: out of memory error when importing under postgresql If you try importing more than 20k items under postgresql you can run out of memory: psycopg2.errors.OutOfMemory: out of shared memory HINT: You might need to increase max_locks_per_transaction. Tuning memory may help, it's unknown at this point. This checkin forces a commit to the postgres database after 10,000 rows have been added. This clears out the savepoints for each row and starts a new transaction. back_postgresql.py: Implement commit mechanism in checkpoint_data(). Add two class level attributes for tracking the number of savepoints and the limit when the commit should happen. roundup_admin.py: implement pragma and dynamically create the config item RDBMS_SAVEPOINT_LIMIT used by checkpoint_data. Also fixed formatting of descriptions when using pragma list in verbose mode. admin_guide.txt, upgrading.txt: Document change and use of pragma savepoint_limit in roundup-admin for changing the default of 10,000. test/db_test_base.py: add some more asserts. In existing testAdminImportExport, set the savepoint limit to 5 to test setting method and so that the commit code will be run by existing tests. This provides coverage, but does not actually test that the commit is done every 5 savepoints 8-(. The verification of every 5 savepoints was done manually using a pdb breakpoint just before the commit. acknowledgements.txt: Added 2.4.0 section mentioning Norbert as he has done a ton of testing with much larger datasets than I can test with.
author John Rouillard <rouilj@ieee.org>
date Thu, 19 Oct 2023 16:11:25 -0400
parents 3f60a71b0812
children
line wrap: on
line source

#
# Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
# This module is free software, and you may redistribute it and/or modify
# under the same terms as Python, so long as this copyright message and
# disclaimer are retained in their original form.
#
# IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
# OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

import unittest, os, shutil, time
from roundup.backends import get_backend

from .db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config
from .db_test_base import HTMLItemTest, SpecialActionTest
from .rest_common import TestCase as RestTestCase

from roundup.anypy import strings

class anydbmOpener:
    module = get_backend('anydbm')

    def nuke_database(self):
        shutil.rmtree(config.DATABASE)


class anydbmDBTest(anydbmOpener, DBTest, unittest.TestCase):
    pass


class anydbmROTest(anydbmOpener, ROTest, unittest.TestCase):
    pass


class anydbmSchemaTest(anydbmOpener, SchemaTest, unittest.TestCase):
    pass


class anydbmClassicInitTest(ClassicInitTest, unittest.TestCase):
    backend = 'anydbm'


class anydbmHTMLItemTest(HTMLItemTest, unittest.TestCase):
    backend = 'anydbm'


from .session_common import SessionTest
class anydbmSessionTest(anydbmOpener, SessionTest, unittest.TestCase):
    s2b = lambda x,y: strings.s2b(y)

    def get_ts(self):
        return (self.sessions.get('random_session', '__timestamp'),)

    def testDbType(self):
        self.assertIn("back_anydbm", repr(self.db))
        self.assertIn("roundup.backends.sessions_dbm.Sessions", repr(self.db.Session))
        
class anydbmSpecialActionTestCase(anydbmOpener, SpecialActionTest,
                                  unittest.TestCase):
    backend = 'anydbm'

class anydbmRestTest (RestTestCase, unittest.TestCase):
    backend = 'anydbm'

# vim: set filetype=python ts=4 sw=4 et si

Roundup Issue Tracker: http://roundup-tracker.org/