Mercurial > p > roundup > code
view roundup/backends/indexer_sqlite_fts.py @ 6628:2bb6d7baa47d
"Comment" out the meta data - will not process under 1.7.5 sphinx
Apparently field names with : fail on 1.7.5 sphinx which is the
virtual env version on sourceforge. It works on my 1.6.7 python2
install.
Looks like I need to add sphinxext-opengraph to get this to work.
However that is python3 only so need to spin up new virtualenv etc.
Looks like no python3 on sourceforge which may be an issue.
On sourceforge in /home/project-web/roundup/src/docbuilder these
packages are used and must be scp'ed as pip has no network access
outside of sourceforge:
Babel-2.6.0-py2.py3-none-any.whl
Jinja2-2.10-py2.py3-none-any.whl
MarkupSafe-1.0.tar.gz
Pygments-2.2.0-py2.py3-none-any.whl
Sphinx-1.7.5
Sphinx-1.7.5-py2.py3-none-any.whl
Sphinx-1.7.5.tar.gz
alabaster-0.7.11-py2.py3-none-any.whl
certifi-2018.4.16-py2.py3-none-any.whl
chardet-3.0.4-py2.py3-none-any.whl
docutils-0.14-py2-none-any.whl
idna-2.7-py2.py3-none-any.whl
imagesize-1.0.0-py2.py3-none-any.whl
packaging-17.1-py2.py3-none-any.whl
pip-10.0.1
pip-10.0.1.tar.gz
pyparsing-2.2.0-py2.py3-none-any.whl
pytz-2018.5-py2.py3-none-any.whl
requests-2.19.1-py2.py3-none-any.whl
setuptools-39.2.0-py2.py3-none-any.whl
six-1.11.0-py2.py3-none-any.whl
snowballstemmer-1.2.1-py2.py3-none-any.whl
sphinxcontrib_websupport-1.1.0-py2.py3-none-any.whl
typing-3.6.4-py2-none-any.whl
urllib3-1.23-py2.py3-none-any.whl
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Sun, 27 Mar 2022 13:57:04 -0400 |
| parents | 91ab3e0ffcd0 |
| children | 9ff091537f43 |
line wrap: on
line source
""" This implements the full-text indexer using fts5 in sqlite. The table consists of (Class, propname, itemid) instances as columns along with a textblob column. The textblob column is searched using MATCH and the instances returned. sqlite test commands to manage schema version change required by this update. -- check length before and after select length(schema) from schema; -- reset from version 7 (with fts index) to version 6 update schema set schema = (select replace(schema, '''version'': 7','''version'': 6') as new_schema from schema); -- check version. Good thing it's at the front of the schema select substr(schema,0,15) from schema; {'version': 6, """ from roundup.backends.indexer_common import Indexer as IndexerBase from roundup.i18n import _ from roundup.cgi.exceptions import IndexerQueryError try: import sqlite3 as sqlite if sqlite.sqlite_version_info < (3, 9, 0): raise ValueError('sqlite minimum version for FTS5 is 3.9.0+ ' '- %s found' % sqlite.sqlite_version) except ImportError: raise ValueError('Unable to import sqlite3 to support FTS.') class Indexer(IndexerBase): def __init__(self, db): IndexerBase.__init__(self, db) self.db = db self.reindex = 0 self.query_language = True def close(self): """close the indexing database""" # just nuke the circular reference self.db = None def save_index(self): """Save the changes to the index.""" # not necessary - the RDBMS connection will handle this for us pass def force_reindex(self): """Force a reindexing of the database. This essentially empties the __fts table and sets a flag so that the databases are reindexed""" self.reindex = 1 def should_reindex(self): """returns True if the indexes need to be rebuilt""" return self.reindex def add_text(self, identifier, text, mime_type='text/plain'): """ "identifier" is (classname, itemid, property) """ if mime_type != 'text/plain': return # Ensure all elements of the identifier are strings 'cos the itemid # column is varchar even if item ids may be numbers elsewhere in the # code. ugh. identifier = tuple(map(str, identifier)) # removed pre-processing of text that incudes only words with: # self.minlength <= len(word) <= self.maxlength # Not sure if that is correct. # first, find the rowid of the (classname, itemid, property) a = self.db.arg # arg is the token for positional parameters sql = 'select rowid from __fts where _class=%s and '\ '_itemid=%s and _prop=%s' % (a, a, a) self.db.cursor.execute(sql, identifier) r = self.db.cursor.fetchone() if not r: # not previously indexed sql = 'insert into __fts (_class, _itemid, _prop, _textblob)'\ ' values (%s, %s, %s, %s)' % (a, a, a, a) self.db.cursor.execute(sql, identifier + (text,)) else: id = int(r[0]) sql = 'update __fts set _textblob=%s where rowid=%s' % \ (a, a) self.db.cursor.execute(sql, (text, id)) def find(self, wordlist): """look up all the words in the wordlist. For testing wordlist is actually a list. In production, wordlist is a list of a single string that is a sqlite MATCH query. https://www.sqlite.org/fts5.html#full_text_query_syntax """ if not wordlist: return [] a = self.db.arg # arg is the token for positional parameters # removed filtering of word in wordlist to include only # words with: self.minlength <= len(word) <= self.maxlength sql = 'select _class, _itemid, _prop from __fts '\ 'where _textblob MATCH %s' % a try: # tests supply a multi element word list. Join them. self.db.cursor.execute(sql, (" ".join(wordlist),)) except sqlite.OperationalError as e: if 'no such column' in e.args[0]: raise IndexerQueryError( _("Search failed. Try quoting any terms that " "include a '-' and retry the search.")) else: raise IndexerQueryError(e.args[0].replace("fts5:", "Query error:")) return self.db.cursor.fetchall()
