Mercurial > p > roundup > code
changeset 3046:a7045bad20de maint-0.8
have RDBMS full-text indexer do AND searching [SF#1055435]
| author | Richard Jones <richard@users.sourceforge.net> |
|---|---|
| date | Tue, 04 Jan 2005 01:33:04 +0000 |
| parents | f859d9393bf0 |
| children | e9538444d00b |
| files | CHANGES.txt roundup/backends/back_mysql.py roundup/backends/back_postgresql.py roundup/backends/back_sqlite.py roundup/backends/indexer_rdbms.py test/db_test_base.py |
| diffstat | 6 files changed, 63 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/CHANGES.txt Tue Jan 04 01:08:52 2005 +0000 +++ b/CHANGES.txt Tue Jan 04 01:33:04 2005 +0000 @@ -73,6 +73,7 @@ - fix indexer searching with no valid words (sf bug 1086787) - updated searching / indexing docs - fix "(list)" popup when list is one item long (sf bug 1064716) +- have RDBMS full-text indexer do AND searching (sf bug 1055435) 2004-10-26 0.7.9
--- a/roundup/backends/back_mysql.py Tue Jan 04 01:08:52 2005 +0000 +++ b/roundup/backends/back_mysql.py Tue Jan 04 01:33:04 2005 +0000 @@ -105,6 +105,9 @@ class Database(Database): arg = '%s' + # used by some code to switch styles of query + implements_intersect = 0 + # Backend for MySQL to use. # InnoDB is faster, but if you're running <4.0.16 then you'll need to # use BDB to pass all unit tests.
--- a/roundup/backends/back_postgresql.py Tue Jan 04 01:08:52 2005 +0000 +++ b/roundup/backends/back_postgresql.py Tue Jan 04 01:33:04 2005 +0000 @@ -81,6 +81,9 @@ class Database(rdbms_common.Database): arg = '%s' + # used by some code to switch styles of query + implements_intersect = 1 + def sql_open_connection(self): db = rdbms_common.connection_dict(self.config, 'database') self.config.logging.getLogger('hyperdb').info('open database %r'%(
--- a/roundup/backends/back_sqlite.py Tue Jan 04 01:08:52 2005 +0000 +++ b/roundup/backends/back_sqlite.py Tue Jan 04 01:33:04 2005 +0000 @@ -1,4 +1,4 @@ -# $Id: back_sqlite.py,v 1.36.2.1 2004-12-14 23:23:35 richard Exp $ +# $Id: back_sqlite.py,v 1.36.2.2 2005-01-04 01:33:03 richard Exp $ '''Implements a backend for SQLite. See https://pysqlite.sourceforge.net/ for pysqlite info @@ -25,6 +25,10 @@ class Database(rdbms_common.Database): # char to use for positional arguments arg = '%s' + + # used by some code to switch styles of query + implements_intersect = 1 + hyperdb_to_sql_datatypes = { hyperdb.String : 'VARCHAR(255)', hyperdb.Date : 'VARCHAR(30)',
--- a/roundup/backends/indexer_rdbms.py Tue Jan 04 01:08:52 2005 +0000 +++ b/roundup/backends/indexer_rdbms.py Tue Jan 04 01:33:04 2005 +0000 @@ -79,16 +79,53 @@ l = [word.upper() for word in wordlist if 26 > len(word) > 2] - a = ','.join([self.db.arg] * len(l)) - sql = 'select distinct(_textid) from __words where _word in (%s)'%a - self.db.cursor.execute(sql, tuple(l)) - r = self.db.cursor.fetchall() - if not r: + if not l: return {} - a = ','.join([self.db.arg] * len(r)) - sql = 'select _class, _itemid, _prop from __textids '\ - 'where _textid in (%s)'%a - self.db.cursor.execute(sql, tuple([int(id) for (id,) in r])) + + if self.db.implements_intersect: + # simple AND search + sql = 'select distinct(_textid) from __words where _word=%s'%self.db.arg + sql = '\nINTERSECT\n'.join([sql]*len(l)) + self.db.cursor.execute(sql, tuple(l)) + r = self.db.cursor.fetchall() + if not r: + return {} + a = ','.join([self.db.arg] * len(r)) + sql = 'select _class, _itemid, _prop from __textids '\ + 'where _textid in (%s)'%a + self.db.cursor.execute(sql, tuple([int(id) for (id,) in r])) + + else: + # A more complex version for MySQL since it doesn't implement INTERSECT + + # Construct SQL statement to join __words table to itself + # multiple times. + sql = """select distinct(__words1._textid) + from __words as __words1 %s + where __words1._word=%s %s""" + + join_tmpl = ' left join __words as __words%d using (_textid) \n' + match_tmpl = ' and __words%d._word=%s \n' + + join_list = [] + match_list = [] + for n in xrange(len(l) - 1): + join_list.append(join_tmpl % (n + 2)) + match_list.append(match_tmpl % (n + 2, self.db.arg)) + + sql = sql%(' '.join(join_list), self.db.arg, ' '.join(match_list)) + self.db.cursor.execute(sql, l) + + r = map(lambda x: x[0], self.db.cursor.fetchall()) + if not r: + return {} + + a = ','.join([self.db.arg] * len(r)) + sql = 'select _class, _itemid, _prop from __textids '\ + 'where _textid in (%s)'%a + + self.db.cursor.execute(sql, tuple(map(int, r))) + # self.search_index has the results as {some id: identifier} ... # sigh r = {}
--- a/test/db_test_base.py Tue Jan 04 01:08:52 2005 +0000 +++ b/test/db_test_base.py Tue Jan 04 01:33:04 2005 +0000 @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: db_test_base.py,v 1.55.2.1 2005-01-03 03:23:38 richard Exp $ +# $Id: db_test_base.py,v 1.55.2.2 2005-01-04 01:33:04 richard Exp $ import unittest, os, shutil, errno, imp, sys, time, pprint @@ -707,6 +707,10 @@ self.assertEquals(self.db.indexer.search(['flebble'], self.db.issue), {i1: {}, i2: {}}) + # test AND'ing of search terms + self.assertEquals(self.db.indexer.search(['frooz', 'flebble'], + self.db.issue), {i2: {}}) + # unindexed stopword self.assertEquals(self.db.indexer.search(['the'], self.db.issue), {})
