Mercurial > p > roundup > code
changeset 6930:a96a239db0d9
Set all sqlite db's to WAL mode on creation
Complete work done on 6917:fba76e0bba98.
WAL journaling mode now enabled on primary db as well as session db's
when using sqlite for session databases.
Made change to WAL mode more robust for testing by bracketing
`pragma journal_mode=wal` with commit() calls. Normally the conversion
would occur when a new session was opened, but this commits the changes
explicitly to make all the tests stable.
Also added doc on this.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Thu, 08 Sep 2022 01:12:12 -0400 |
| parents | 581b09777eba |
| children | 83fa81f084bc |
| files | CHANGES.txt doc/upgrading.txt roundup/backends/back_sqlite.py roundup/backends/sessions_sqlite.py test/test_sqlite.py |
| diffstat | 5 files changed, 78 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/CHANGES.txt Wed Sep 07 10:55:31 2022 -0400 +++ b/CHANGES.txt Thu Sep 08 01:12:12 2022 -0400 @@ -59,6 +59,9 @@ - sqlite native-fts backend now uses the stopwords list in config.ini to filter words from queries. (Stopwords are still indexed so that phrase/proximity searches still work.) (John Rouillard) +- sqlite databases use WAL mode when *created* to improve read + concurrency. Existing sqlite database still use rollback journal + mode. See upgrading.txt for details. (John Rouillard) 2022-07-13 2.2.0
--- a/doc/upgrading.txt Wed Sep 07 10:55:31 2022 -0400 +++ b/doc/upgrading.txt Thu Sep 08 01:12:12 2022 -0400 @@ -124,6 +124,51 @@ .. _Using Redis for Session Databases: admin_guide.html#using-redis-for-session-databases +New SQLite databases created with WAL mode journaling (optional) +---------------------------------------------------------------- + +By default, SQLite databases use a rollback journal when +writing an update. The rollback journal stores a copy of the +data from before the update. One downside of this is that +all reads have to be suspended while a write is +occurring. SQLite has an alternate way of insuring ACID +compliance by using a WAL (write ahead log) journal. + +Version 2.3.0 of Roundup, creates new SQLite databases using +WAL journaling. With WAL, a writer does not block readers +and readers do not block writing an update. This keeps +Roundup accessible even under a heavy write load (e.g. when +bulk loading data or automated updates via REST). + +If you want to convert your existing SQLite db to WAL mode: + + 1. check the current journal mode on your database + using:: + + sqlite3 <tracker_home>/db/db "pragma journal_mode;" + + 2. If it returns ``delete``, change it to WAL mode using:: + + sqlite3 <tracker_home>/db/db "pragma journal_mode=WAL;" + + 3. verify by running the command in step 1 again and you + should get ``wal``. + +If you are using SQLite for session and otk databases, +perform the same steps replacing ``db`` with ``db-session`` +and ``db-otk``. + +If you find WAL mode is not working for you, you can set the +journal method to a rollback journal (``delete`` mode) by +using step 2 and replacing ``wal`` with ``delete``. (Note: +SQLite supports other journaling modes, but only ``wal`` and +``delete`` persist. Roundup doesn't set a journaling mode +when it opens the database, so options such as ``truncate`` +are not used.) + +For details on WAL mode see `<https://www.sqlite.org/wal.html>`_ +and `<https://www.sqlite.org/pragma.html#pragma_journal_mode>`_. + .. index:: Upgrading; 2.1.0 to 2.2.0 Migrating from 2.1.0 to 2.2.0
--- a/roundup/backends/back_sqlite.py Wed Sep 07 10:55:31 2022 -0400 +++ b/roundup/backends/back_sqlite.py Thu Sep 08 01:12:12 2022 -0400 @@ -204,7 +204,9 @@ self.create_version_2_tables() self._add_fts5_table() # Set journal mode to WAL. - self.conn.execute('pragma journal_mode=wal') + self.sql_commit() # close out rollback journal/transaction + self.sql('pragma journal_mode=wal') # set wal + self.sql_commit() # close out rollback and commit wal change def create_version_2_tables(self): self.sql('create table otks (otk_key varchar, '
--- a/roundup/backends/sessions_sqlite.py Wed Sep 07 10:55:31 2022 -0400 +++ b/roundup/backends/sessions_sqlite.py Thu Sep 08 01:12:12 2022 -0400 @@ -36,7 +36,10 @@ %(name)s_value TEXT, %(name)s_time REAL)''' % {"name": self.name}) self.sql('CREATE INDEX %(name)s_key_idx ON ' '%(name)ss(%(name)s_key)' % {"name": self.name}) - self.commit() + # Set journal mode to WAL. + self.commit() # close out rollback journal/transaction + self.sql('pragma journal_mode=wal') # set wal + self.commit() # close out rollback and commit wal change def sql(self, sql, args=None, cursor=None): """ Execute the sql with the optional args.
--- a/test/test_sqlite.py Wed Sep 07 10:55:31 2022 -0400 +++ b/test/test_sqlite.py Thu Sep 08 01:12:12 2022 -0400 @@ -33,6 +33,18 @@ def nuke_database(self): shutil.rmtree(config.DATABASE) + def testWalMode(self): + """verify that all sqlite db's are in WAL mode + and not journal mode + """ + if not hasattr(self, 'db'): + self.skipTest("test has no database open") + + for db in [self.db]: + print("testing db", str(db)) + db.sql('pragma journal_mode;') + self.assertEqual(db.cursor.fetchone()['journal_mode'], 'wal') + class sqliteDBTest(sqliteOpener, DBTest, unittest.TestCase): @@ -198,6 +210,15 @@ self.assertIn("roundlite", repr(self.db)) self.assertIn("roundup.backends.sessions_sqlite.Sessions", repr(self.db.Session)) + def testWalMode(self): + """verify that all sqlite db's are in WAL mode + and not Rollback mode + """ + for db in [self.db, self.db.Session, self.db.Otk]: + print("testing db", str(db)) + db.sql('pragma journal_mode;') + self.assertEqual(db.cursor.fetchone()['journal_mode'], 'wal') + class anydbmSessionTest(sqliteOpener, SessionTest, unittest.TestCase): s2b = lambda x,y : y @@ -227,6 +248,7 @@ def testDbType(self): self.assertIn("roundlite", repr(self.db)) self.assertIn("roundup.backends.sessions_dbm.Sessions", repr(self.db.Session)) - + + class sqliteRestTest (RestTestCase, unittest.TestCase): backend = 'sqlite'
